import * as fs from 'fs'; import * as path from 'path'; export type SampledPoint = { x: number; y: number; z: number; error?: number; r?: number; g?: number; b?: number }; export type SampledPointcloudBatch = { buffer: Buffer; pointCount: number; hasColor: boolean; hasError: boolean; fileStats: Array<{ name: string; sizeGB: number; sampled: number }>; }; /** 与 fileRoutes 批量采样一致:随机 seek 文本点云行 */ export async function randomSeekSample( filePath: string, fileSize: number, k: number, ): Promise { if (fileSize === 0 || k === 0) return []; const PROBE_BYTES = 64 * 1024; const probeBuf = Buffer.alloc(Math.min(PROBE_BYTES, fileSize)); const probeFd = fs.openSync(filePath, 'r'); fs.readSync(probeFd, probeBuf, 0, probeBuf.length, 0); fs.closeSync(probeFd); const probeLines = probeBuf.toString('utf8').split('\n').filter((l) => { const t = l.trim(); return t && !t.startsWith('#') && !t.startsWith('//'); }); const sampleCount = Math.min(probeLines.length, 200); const avgLineBytes = sampleCount > 0 ? probeLines.slice(0, sampleCount).reduce((s, l) => s + Buffer.byteLength(l) + 1, 0) / sampleCount : 40; console.log( `[SeekSampler] ${path.basename(filePath)}: size=${(fileSize / 1e9).toFixed(2)}GB ` + `avgLine=${avgLineBytes.toFixed(1)}B quota=${k}`, ); const startOffset = Math.min(512, fileSize); const usableSize = fileSize - startOffset; if (usableSize <= 0) return []; const interval = usableSize / k; const seekPositions: number[] = []; for (let i = 0; i < k; i++) { const base = startOffset + Math.floor(i * interval); const jitter = Math.floor(Math.random() * Math.min(interval * 0.5, 1024)); seekPositions.push(Math.min(base + jitter, fileSize - 2)); } seekPositions.sort((a, b) => a - b); const points: SampledPoint[] = []; const fd = fs.openSync(filePath, 'r'); const lineBuf = Buffer.alloc(1024); try { for (const pos of seekPositions) { if (points.length >= k) break; const bytesRead = fs.readSync(fd, lineBuf, 0, lineBuf.length, pos); if (bytesRead < 4) continue; const chunk = lineBuf.toString('utf8', 0, bytesRead); const nl1 = chunk.indexOf('\n'); if (nl1 === -1) continue; const afterFirst = chunk.slice(nl1 + 1); const nl2 = afterFirst.indexOf('\n'); const line = (nl2 === -1 ? afterFirst : afterFirst.slice(0, nl2)).trim(); if (!line || line.startsWith('#') || line.startsWith('//')) continue; const parts = line.split(/[\s,\t]+/); const x = parseFloat(parts[0]); const y = parseFloat(parts[1]); const z = parseFloat(parts[2]); if (isNaN(x) || isNaN(y) || isNaN(z)) continue; const parsedError = parts.length >= 4 ? parseFloat(parts[3]) : NaN; const point: SampledPoint = { x, y, z }; if (Number.isFinite(parsedError)) point.error = parsedError; points.push(point); } } finally { fs.closeSync(fd); } return points; } // ─── PLY 格式解析与采样 ─────────────────────────────────────────────────────── /** PLY 属性类型 → 字节数 */ const PLY_PROP_SIZES: Record = { char: 1, int8: 1, uchar: 1, uint8: 1, short: 2, int16: 2, ushort: 2, uint16: 2, int: 4, int32: 4, uint: 4, uint32: 4, float: 4, float32: 4, double: 8, float64: 8, }; interface PlyHeader { /** 'ascii' | 'binary_little_endian' | 'binary_big_endian' */ format: string; /** 顶点数 */ vertexCount: number; /** 顶点元素各属性,顺序与文件一致 */ properties: Array<{ name: string; type: string; byteSize: number }>; /** 'end_header\n' 之后的第一个字节偏移量(即数据区起始位置) */ dataOffset: number; } /** * 读取 PLY 文件头(总是 ASCII,以 "end_header\n" 结尾)。 * 最多读取 16 KB,足以容纳任何合理的 PLY 头。 */ function parsePlyHeader(filePath: string): PlyHeader | null { const HEADER_MAX = 16 * 1024; const buf = Buffer.alloc(HEADER_MAX); const fd = fs.openSync(filePath, 'r'); const bytesRead = fs.readSync(fd, buf, 0, HEADER_MAX, 0); fs.closeSync(fd); const raw = buf.toString('ascii', 0, bytesRead); // 必须以 "ply\n" 开头 if (!raw.startsWith('ply\n') && !raw.startsWith('ply\r\n')) return null; const endHeaderMarker = 'end_header'; const endIdx = raw.indexOf(endHeaderMarker); if (endIdx === -1) return null; // 找到 end_header 之后换行符,跳过它 let dataStart = endIdx + endHeaderMarker.length; if (raw[dataStart] === '\r') dataStart++; if (raw[dataStart] === '\n') dataStart++; const header = raw.slice(0, endIdx); const lines = header.split(/\r?\n/); let format = 'ascii'; let vertexCount = 0; const properties: PlyHeader['properties'] = []; let inVertex = false; for (const line of lines) { const parts = line.trim().split(/\s+/); if (parts[0] === 'format') { format = parts[1] ?? 'ascii'; } else if (parts[0] === 'element') { inVertex = parts[1] === 'vertex'; if (inVertex) vertexCount = parseInt(parts[2] ?? '0', 10); } else if (parts[0] === 'property' && inVertex) { // skip "property list ..." (face element) if (parts[1] === 'list') { inVertex = false; continue; } const type = parts[1] ?? 'float'; const name = parts[2] ?? ''; const byteSize = PLY_PROP_SIZES[type] ?? 4; properties.push({ name, type, byteSize }); } } if (vertexCount <= 0 || properties.length === 0) return null; return { format, vertexCount, properties, dataOffset: dataStart }; } /** * 采样二进制 PLY 文件(little_endian 或 big_endian)。 * 策略:均匀间隔 + 小量随机抖动(与 randomSeekSample 一致)。 */ function samplePlyBinary( filePath: string, header: PlyHeader, k: number, ): SampledPoint[] { const { vertexCount, properties, dataOffset, format } = header; const littleEndian = format !== 'binary_big_endian'; const bytesPerVertex = properties.reduce((s, p) => s + p.byteSize, 0); // 找 x, y, z,以及 red/green/blue(uchar 0-255) let xOff = -1, yOff = -1, zOff = -1; let rOff = -1, gOff = -1, bOff = -1; let errorOff = -1; let xType = 'float', yType = 'float', zType = 'float'; let rType = 'uchar', gType = 'uchar', bType = 'uchar'; let errorType = 'float'; let byteOff = 0; for (const p of properties) { const n = p.name.toLowerCase(); if (n === 'x') { xOff = byteOff; xType = p.type; } else if (n === 'y') { yOff = byteOff; yType = p.type; } else if (n === 'z') { zOff = byteOff; zType = p.type; } else if (n === 'red') { rOff = byteOff; rType = p.type; } else if (n === 'green') { gOff = byteOff; gType = p.type; } else if (n === 'blue') { bOff = byteOff; bType = p.type; } else if (n === 'error' || n === 'deviation') { errorOff = byteOff; errorType = p.type; } byteOff += p.byteSize; } if (xOff === -1 || yOff === -1 || zOff === -1) { console.warn(`[PLY] ${path.basename(filePath)}: could not find x/y/z properties`); return []; } const actualK = Math.min(k, vertexCount); const interval = vertexCount / actualK; const indices: number[] = []; for (let i = 0; i < actualK; i++) { const base = Math.floor(i * interval); const jitter = Math.floor(Math.random() * Math.max(1, interval * 0.5)); indices.push(Math.min(base + jitter, vertexCount - 1)); } const recBuf = Buffer.alloc(bytesPerVertex); const fd = fs.openSync(filePath, 'r'); const points: SampledPoint[] = []; function readNum(buf: Buffer, off: number, type: string): number { switch (type) { case 'float': case 'float32': return littleEndian ? buf.readFloatLE(off) : buf.readFloatBE(off); case 'double': case 'float64': return littleEndian ? buf.readDoubleLE(off) : buf.readDoubleBE(off); case 'int': case 'int32': return littleEndian ? buf.readInt32LE(off) : buf.readInt32BE(off); case 'uint': case 'uint32': return littleEndian ? buf.readUInt32LE(off) : buf.readUInt32BE(off); case 'short': case 'int16': return littleEndian ? buf.readInt16LE(off) : buf.readInt16BE(off); case 'ushort': case 'uint16': return littleEndian ? buf.readUInt16LE(off) : buf.readUInt16BE(off); case 'char': case 'int8': return buf.readInt8(off); case 'uchar': case 'uint8': return buf.readUInt8(off); default: return littleEndian ? buf.readFloatLE(off) : buf.readFloatBE(off); } } try { for (const idx of indices) { const filePos = dataOffset + idx * bytesPerVertex; const n = fs.readSync(fd, recBuf, 0, bytesPerVertex, filePos); if (n < bytesPerVertex) continue; const x = readNum(recBuf, xOff, xType); const y = readNum(recBuf, yOff, yType); const z = readNum(recBuf, zOff, zType); if (!isFinite(x) || !isFinite(y) || !isFinite(z)) continue; const pt: SampledPoint = { x, y, z }; if (errorOff >= 0) { const error = readNum(recBuf, errorOff, errorType); if (Number.isFinite(error)) pt.error = error; } if (rOff >= 0 && gOff >= 0 && bOff >= 0) { pt.r = readNum(recBuf, rOff, rType); pt.g = readNum(recBuf, gOff, gType); pt.b = readNum(recBuf, bOff, bType); } points.push(pt); } } finally { fs.closeSync(fd); } console.log( `[PLY-Binary] ${path.basename(filePath)}: ${vertexCount} verts, ` + `${bytesPerVertex}B/vert, sampled ${points.length}`, ); return points; } /** * 采样 ASCII PLY 文件。 * 复用 randomSeekSample 的均匀 seek 策略,跳过 PLY 头,从数据区随机取行。 */ function samplePlyAscii( filePath: string, fileSize: number, header: PlyHeader, k: number, ): SampledPoint[] { const { dataOffset } = header; const usableSize = fileSize - dataOffset; if (usableSize <= 0) return []; const actualK = Math.min(k, header.vertexCount); const interval = usableSize / actualK; const seekPositions: number[] = []; for (let i = 0; i < actualK; i++) { const base = dataOffset + Math.floor(i * interval); const jitter = Math.floor(Math.random() * Math.min(interval * 0.5, 1024)); seekPositions.push(Math.min(base + jitter, fileSize - 2)); } const points: SampledPoint[] = []; const fd = fs.openSync(filePath, 'r'); const lineBuf = Buffer.alloc(512); try { for (const pos of seekPositions) { if (points.length >= actualK) break; const bytesRead = fs.readSync(fd, lineBuf, 0, lineBuf.length, pos); if (bytesRead < 4) continue; const chunk = lineBuf.toString('utf8', 0, bytesRead); const nl1 = chunk.indexOf('\n'); if (nl1 === -1) continue; const afterFirst = chunk.slice(nl1 + 1); const nl2 = afterFirst.indexOf('\n'); const line = (nl2 === -1 ? afterFirst : afterFirst.slice(0, nl2)).trim(); if (!line) continue; const parts = line.split(/\s+/); // 按 header properties 顺序找 x/y/z/red/green/blue 的列索引 let xi = -1, yi = -1, zi = -1, ri = -1, gi = -1, bi = -1, errorI = -1; header.properties.forEach((p, i) => { const n = p.name.toLowerCase(); if (n === 'x') xi = i; else if (n === 'y') yi = i; else if (n === 'z') zi = i; else if (n === 'red') ri = i; else if (n === 'green') gi = i; else if (n === 'blue') bi = i; else if (n === 'error' || n === 'deviation') errorI = i; }); const x = parseFloat(parts[xi >= 0 ? xi : 0]); const y = parseFloat(parts[yi >= 0 ? yi : 1]); const z = parseFloat(parts[zi >= 0 ? zi : 2]); if (!isFinite(x) || !isFinite(y) || !isFinite(z)) continue; const pt: SampledPoint = { x, y, z }; if (errorI >= 0) { const error = parseFloat(parts[errorI]); if (Number.isFinite(error)) pt.error = error; } if (ri >= 0 && gi >= 0 && bi >= 0 && parts[ri] !== undefined) { pt.r = parseFloat(parts[ri]); pt.g = parseFloat(parts[gi]); pt.b = parseFloat(parts[bi]); } points.push(pt); } } finally { fs.closeSync(fd); } console.log( `[PLY-ASCII] ${path.basename(filePath)}: sampled ${points.length} / ${header.vertexCount} verts`, ); return points; } /** * 通用采样入口:根据扩展名分发到对应采样器。 * 支持 .ply(ASCII + binary),.txt / .xyz / .pts(纯文本行)。 */ export async function samplePointcloudFile( filePath: string, fileSize: number, k: number, ): Promise { if (fileSize === 0 || k === 0) return []; const ext = path.extname(filePath).toLowerCase(); if (ext === '.ply') { const header = parsePlyHeader(filePath); if (!header) { console.warn(`[PLY] ${path.basename(filePath)}: failed to parse header, falling back to text sampler`); return randomSeekSample(filePath, fileSize, k); } if (header.format === 'ascii') { return samplePlyAscii(filePath, fileSize, header, k); } return samplePlyBinary(filePath, header, k); } // .txt / .xyz / .pts / etc. — existing text line sampler return randomSeekSample(filePath, fileSize, k); } /** * 对一组已校验的点云文件按文件大小分配总采样配额,并编码为浏览器消费的二进制协议。 * 每点固定包含 x/y/z/error;任一文件带颜色时,所有点再包含归一化后的 r/g/b。 */ export async function samplePointcloudFiles( filePaths: string[], requestedTotalPoints: number, maxTotalPoints = 2_000_000, ): Promise { const totalPoints = Number.isFinite(requestedTotalPoints) ? Math.max(0, Math.min(Math.floor(requestedTotalPoints), maxTotalPoints)) : 0; const fileInfos = filePaths.map((filePath) => ({ filePath, name: path.basename(filePath), size: fs.statSync(filePath).size, })); const totalBytes = fileInfos.reduce((sum, file) => sum + file.size, 0); // floor 后把剩余名额依次补给文件,保证所有配额之和严格不超过总预算。 const quotas = fileInfos.map((file) => totalBytes > 0 ? Math.floor((file.size / totalBytes) * totalPoints) : 0); let remainder = totalPoints - quotas.reduce((sum, quota) => sum + quota, 0); for (let index = 0; remainder > 0 && fileInfos.length > 0; index = (index + 1) % fileInfos.length) { quotas[index] += 1; remainder -= 1; } const sampledPerFile = await Promise.all( fileInfos.map((file, index) => samplePointcloudFile(file.filePath, file.size, quotas[index])), ); const pointCount = sampledPerFile.reduce((sum, points) => sum + points.length, 0); const hasError = sampledPerFile.some((points) => points.some((point) => Number.isFinite(point.error))); const hasColor = sampledPerFile.some((points) => points.some((point) => point.r !== undefined)); const bytesPerPoint = hasColor ? 28 : 16; const buffer = Buffer.allocUnsafe(pointCount * bytesPerPoint); let offset = 0; for (const points of sampledPerFile) { for (const point of points) { buffer.writeFloatLE(point.x, offset); buffer.writeFloatLE(point.y, offset + 4); buffer.writeFloatLE(point.z, offset + 8); buffer.writeFloatLE(point.error ?? Number.NaN, offset + 12); if (hasColor) { buffer.writeFloatLE((point.r ?? 128) / 255, offset + 16); buffer.writeFloatLE((point.g ?? 128) / 255, offset + 20); buffer.writeFloatLE((point.b ?? 128) / 255, offset + 24); } offset += bytesPerPoint; } } return { buffer, pointCount, hasColor, hasError, fileStats: fileInfos.map((file, index) => ({ name: file.name, sizeGB: +(file.size / 1e9).toFixed(3), sampled: sampledPerFile[index].length, })), }; }