434 lines
16 KiB
TypeScript
434 lines
16 KiB
TypeScript
import * as fs from 'fs';
|
||
import * as path from 'path';
|
||
|
||
export type SampledPoint = { x: number; y: number; z: number; error?: number; r?: number; g?: number; b?: number };
|
||
|
||
export type SampledPointcloudBatch = {
|
||
buffer: Buffer;
|
||
pointCount: number;
|
||
hasColor: boolean;
|
||
hasError: boolean;
|
||
fileStats: Array<{ name: string; sizeGB: number; sampled: number }>;
|
||
};
|
||
|
||
/** 与 fileRoutes 批量采样一致：随机 seek 文本点云行 */
|
||
export async function randomSeekSample(
|
||
filePath: string,
|
||
fileSize: number,
|
||
k: number,
|
||
): Promise<SampledPoint[]> {
|
||
if (fileSize === 0 || k === 0) return [];
|
||
|
||
const PROBE_BYTES = 64 * 1024;
|
||
const probeBuf = Buffer.alloc(Math.min(PROBE_BYTES, fileSize));
|
||
const probeFd = fs.openSync(filePath, 'r');
|
||
fs.readSync(probeFd, probeBuf, 0, probeBuf.length, 0);
|
||
fs.closeSync(probeFd);
|
||
|
||
const probeLines = probeBuf.toString('utf8').split('\n').filter((l) => {
|
||
const t = l.trim();
|
||
return t && !t.startsWith('#') && !t.startsWith('//');
|
||
});
|
||
const sampleCount = Math.min(probeLines.length, 200);
|
||
const avgLineBytes =
|
||
sampleCount > 0
|
||
? probeLines.slice(0, sampleCount).reduce((s, l) => s + Buffer.byteLength(l) + 1, 0) / sampleCount
|
||
: 40;
|
||
|
||
console.log(
|
||
`[SeekSampler] ${path.basename(filePath)}: size=${(fileSize / 1e9).toFixed(2)}GB ` +
|
||
`avgLine=${avgLineBytes.toFixed(1)}B quota=${k}`,
|
||
);
|
||
|
||
const startOffset = Math.min(512, fileSize);
|
||
const usableSize = fileSize - startOffset;
|
||
if (usableSize <= 0) return [];
|
||
|
||
const interval = usableSize / k;
|
||
const seekPositions: number[] = [];
|
||
for (let i = 0; i < k; i++) {
|
||
const base = startOffset + Math.floor(i * interval);
|
||
const jitter = Math.floor(Math.random() * Math.min(interval * 0.5, 1024));
|
||
seekPositions.push(Math.min(base + jitter, fileSize - 2));
|
||
}
|
||
seekPositions.sort((a, b) => a - b);
|
||
|
||
const points: SampledPoint[] = [];
|
||
const fd = fs.openSync(filePath, 'r');
|
||
const lineBuf = Buffer.alloc(1024);
|
||
|
||
try {
|
||
for (const pos of seekPositions) {
|
||
if (points.length >= k) break;
|
||
const bytesRead = fs.readSync(fd, lineBuf, 0, lineBuf.length, pos);
|
||
if (bytesRead < 4) continue;
|
||
const chunk = lineBuf.toString('utf8', 0, bytesRead);
|
||
const nl1 = chunk.indexOf('\n');
|
||
if (nl1 === -1) continue;
|
||
const afterFirst = chunk.slice(nl1 + 1);
|
||
const nl2 = afterFirst.indexOf('\n');
|
||
const line = (nl2 === -1 ? afterFirst : afterFirst.slice(0, nl2)).trim();
|
||
if (!line || line.startsWith('#') || line.startsWith('//')) continue;
|
||
const parts = line.split(/[\s,\t]+/);
|
||
const x = parseFloat(parts[0]);
|
||
const y = parseFloat(parts[1]);
|
||
const z = parseFloat(parts[2]);
|
||
if (isNaN(x) || isNaN(y) || isNaN(z)) continue;
|
||
const parsedError = parts.length >= 4 ? parseFloat(parts[3]) : NaN;
|
||
const point: SampledPoint = { x, y, z };
|
||
if (Number.isFinite(parsedError)) point.error = parsedError;
|
||
points.push(point);
|
||
}
|
||
} finally {
|
||
fs.closeSync(fd);
|
||
}
|
||
return points;
|
||
}
|
||
|
||
// ─── PLY 格式解析与采样 ───────────────────────────────────────────────────────
|
||
|
||
/** PLY 属性类型 → 字节数 */
|
||
const PLY_PROP_SIZES: Record<string, number> = {
|
||
char: 1, int8: 1,
|
||
uchar: 1, uint8: 1,
|
||
short: 2, int16: 2,
|
||
ushort: 2, uint16: 2,
|
||
int: 4, int32: 4,
|
||
uint: 4, uint32: 4,
|
||
float: 4, float32: 4,
|
||
double: 8, float64: 8,
|
||
};
|
||
|
||
interface PlyHeader {
|
||
/** 'ascii' | 'binary_little_endian' | 'binary_big_endian' */
|
||
format: string;
|
||
/** 顶点数 */
|
||
vertexCount: number;
|
||
/** 顶点元素各属性，顺序与文件一致 */
|
||
properties: Array<{ name: string; type: string; byteSize: number }>;
|
||
/** 'end_header\n' 之后的第一个字节偏移量（即数据区起始位置） */
|
||
dataOffset: number;
|
||
}
|
||
|
||
/**
|
||
* 读取 PLY 文件头（总是 ASCII，以 "end_header\n" 结尾）。
|
||
* 最多读取 16 KB，足以容纳任何合理的 PLY 头。
|
||
*/
|
||
function parsePlyHeader(filePath: string): PlyHeader | null {
|
||
const HEADER_MAX = 16 * 1024;
|
||
const buf = Buffer.alloc(HEADER_MAX);
|
||
const fd = fs.openSync(filePath, 'r');
|
||
const bytesRead = fs.readSync(fd, buf, 0, HEADER_MAX, 0);
|
||
fs.closeSync(fd);
|
||
|
||
const raw = buf.toString('ascii', 0, bytesRead);
|
||
|
||
// 必须以 "ply\n" 开头
|
||
if (!raw.startsWith('ply\n') && !raw.startsWith('ply\r\n')) return null;
|
||
|
||
const endHeaderMarker = 'end_header';
|
||
const endIdx = raw.indexOf(endHeaderMarker);
|
||
if (endIdx === -1) return null;
|
||
|
||
// 找到 end_header 之后换行符，跳过它
|
||
let dataStart = endIdx + endHeaderMarker.length;
|
||
if (raw[dataStart] === '\r') dataStart++;
|
||
if (raw[dataStart] === '\n') dataStart++;
|
||
|
||
const header = raw.slice(0, endIdx);
|
||
const lines = header.split(/\r?\n/);
|
||
|
||
let format = 'ascii';
|
||
let vertexCount = 0;
|
||
const properties: PlyHeader['properties'] = [];
|
||
let inVertex = false;
|
||
|
||
for (const line of lines) {
|
||
const parts = line.trim().split(/\s+/);
|
||
if (parts[0] === 'format') {
|
||
format = parts[1] ?? 'ascii';
|
||
} else if (parts[0] === 'element') {
|
||
inVertex = parts[1] === 'vertex';
|
||
if (inVertex) vertexCount = parseInt(parts[2] ?? '0', 10);
|
||
} else if (parts[0] === 'property' && inVertex) {
|
||
// skip "property list ..." (face element)
|
||
if (parts[1] === 'list') { inVertex = false; continue; }
|
||
const type = parts[1] ?? 'float';
|
||
const name = parts[2] ?? '';
|
||
const byteSize = PLY_PROP_SIZES[type] ?? 4;
|
||
properties.push({ name, type, byteSize });
|
||
}
|
||
}
|
||
|
||
if (vertexCount <= 0 || properties.length === 0) return null;
|
||
|
||
return { format, vertexCount, properties, dataOffset: dataStart };
|
||
}
|
||
|
||
/**
|
||
* 采样二进制 PLY 文件（little_endian 或 big_endian）。
|
||
* 策略：均匀间隔 + 小量随机抖动（与 randomSeekSample 一致）。
|
||
*/
|
||
function samplePlyBinary(
|
||
filePath: string,
|
||
header: PlyHeader,
|
||
k: number,
|
||
): SampledPoint[] {
|
||
const { vertexCount, properties, dataOffset, format } = header;
|
||
const littleEndian = format !== 'binary_big_endian';
|
||
|
||
const bytesPerVertex = properties.reduce((s, p) => s + p.byteSize, 0);
|
||
|
||
// 找 x, y, z，以及 red/green/blue（uchar 0-255）
|
||
let xOff = -1, yOff = -1, zOff = -1;
|
||
let rOff = -1, gOff = -1, bOff = -1;
|
||
let errorOff = -1;
|
||
let xType = 'float', yType = 'float', zType = 'float';
|
||
let rType = 'uchar', gType = 'uchar', bType = 'uchar';
|
||
let errorType = 'float';
|
||
let byteOff = 0;
|
||
for (const p of properties) {
|
||
const n = p.name.toLowerCase();
|
||
if (n === 'x') { xOff = byteOff; xType = p.type; }
|
||
else if (n === 'y') { yOff = byteOff; yType = p.type; }
|
||
else if (n === 'z') { zOff = byteOff; zType = p.type; }
|
||
else if (n === 'red') { rOff = byteOff; rType = p.type; }
|
||
else if (n === 'green') { gOff = byteOff; gType = p.type; }
|
||
else if (n === 'blue') { bOff = byteOff; bType = p.type; }
|
||
else if (n === 'error' || n === 'deviation') { errorOff = byteOff; errorType = p.type; }
|
||
byteOff += p.byteSize;
|
||
}
|
||
|
||
if (xOff === -1 || yOff === -1 || zOff === -1) {
|
||
console.warn(`[PLY] ${path.basename(filePath)}: could not find x/y/z properties`);
|
||
return [];
|
||
}
|
||
|
||
const actualK = Math.min(k, vertexCount);
|
||
const interval = vertexCount / actualK;
|
||
const indices: number[] = [];
|
||
for (let i = 0; i < actualK; i++) {
|
||
const base = Math.floor(i * interval);
|
||
const jitter = Math.floor(Math.random() * Math.max(1, interval * 0.5));
|
||
indices.push(Math.min(base + jitter, vertexCount - 1));
|
||
}
|
||
|
||
const recBuf = Buffer.alloc(bytesPerVertex);
|
||
const fd = fs.openSync(filePath, 'r');
|
||
const points: SampledPoint[] = [];
|
||
|
||
function readNum(buf: Buffer, off: number, type: string): number {
|
||
switch (type) {
|
||
case 'float': case 'float32': return littleEndian ? buf.readFloatLE(off) : buf.readFloatBE(off);
|
||
case 'double': case 'float64': return littleEndian ? buf.readDoubleLE(off) : buf.readDoubleBE(off);
|
||
case 'int': case 'int32': return littleEndian ? buf.readInt32LE(off) : buf.readInt32BE(off);
|
||
case 'uint': case 'uint32': return littleEndian ? buf.readUInt32LE(off) : buf.readUInt32BE(off);
|
||
case 'short': case 'int16': return littleEndian ? buf.readInt16LE(off) : buf.readInt16BE(off);
|
||
case 'ushort': case 'uint16': return littleEndian ? buf.readUInt16LE(off) : buf.readUInt16BE(off);
|
||
case 'char': case 'int8': return buf.readInt8(off);
|
||
case 'uchar': case 'uint8': return buf.readUInt8(off);
|
||
default: return littleEndian ? buf.readFloatLE(off) : buf.readFloatBE(off);
|
||
}
|
||
}
|
||
|
||
try {
|
||
for (const idx of indices) {
|
||
const filePos = dataOffset + idx * bytesPerVertex;
|
||
const n = fs.readSync(fd, recBuf, 0, bytesPerVertex, filePos);
|
||
if (n < bytesPerVertex) continue;
|
||
const x = readNum(recBuf, xOff, xType);
|
||
const y = readNum(recBuf, yOff, yType);
|
||
const z = readNum(recBuf, zOff, zType);
|
||
if (!isFinite(x) || !isFinite(y) || !isFinite(z)) continue;
|
||
const pt: SampledPoint = { x, y, z };
|
||
if (errorOff >= 0) {
|
||
const error = readNum(recBuf, errorOff, errorType);
|
||
if (Number.isFinite(error)) pt.error = error;
|
||
}
|
||
if (rOff >= 0 && gOff >= 0 && bOff >= 0) {
|
||
pt.r = readNum(recBuf, rOff, rType);
|
||
pt.g = readNum(recBuf, gOff, gType);
|
||
pt.b = readNum(recBuf, bOff, bType);
|
||
}
|
||
points.push(pt);
|
||
}
|
||
} finally {
|
||
fs.closeSync(fd);
|
||
}
|
||
|
||
console.log(
|
||
`[PLY-Binary] ${path.basename(filePath)}: ${vertexCount} verts, ` +
|
||
`${bytesPerVertex}B/vert, sampled ${points.length}`,
|
||
);
|
||
return points;
|
||
}
|
||
|
||
/**
|
||
* 采样 ASCII PLY 文件。
|
||
* 复用 randomSeekSample 的均匀 seek 策略，跳过 PLY 头，从数据区随机取行。
|
||
*/
|
||
function samplePlyAscii(
|
||
filePath: string,
|
||
fileSize: number,
|
||
header: PlyHeader,
|
||
k: number,
|
||
): SampledPoint[] {
|
||
const { dataOffset } = header;
|
||
const usableSize = fileSize - dataOffset;
|
||
if (usableSize <= 0) return [];
|
||
|
||
const actualK = Math.min(k, header.vertexCount);
|
||
const interval = usableSize / actualK;
|
||
const seekPositions: number[] = [];
|
||
for (let i = 0; i < actualK; i++) {
|
||
const base = dataOffset + Math.floor(i * interval);
|
||
const jitter = Math.floor(Math.random() * Math.min(interval * 0.5, 1024));
|
||
seekPositions.push(Math.min(base + jitter, fileSize - 2));
|
||
}
|
||
|
||
const points: SampledPoint[] = [];
|
||
const fd = fs.openSync(filePath, 'r');
|
||
const lineBuf = Buffer.alloc(512);
|
||
|
||
try {
|
||
for (const pos of seekPositions) {
|
||
if (points.length >= actualK) break;
|
||
const bytesRead = fs.readSync(fd, lineBuf, 0, lineBuf.length, pos);
|
||
if (bytesRead < 4) continue;
|
||
const chunk = lineBuf.toString('utf8', 0, bytesRead);
|
||
const nl1 = chunk.indexOf('\n');
|
||
if (nl1 === -1) continue;
|
||
const afterFirst = chunk.slice(nl1 + 1);
|
||
const nl2 = afterFirst.indexOf('\n');
|
||
const line = (nl2 === -1 ? afterFirst : afterFirst.slice(0, nl2)).trim();
|
||
if (!line) continue;
|
||
const parts = line.split(/\s+/);
|
||
// 按 header properties 顺序找 x/y/z/red/green/blue 的列索引
|
||
let xi = -1, yi = -1, zi = -1, ri = -1, gi = -1, bi = -1, errorI = -1;
|
||
header.properties.forEach((p, i) => {
|
||
const n = p.name.toLowerCase();
|
||
if (n === 'x') xi = i; else if (n === 'y') yi = i; else if (n === 'z') zi = i;
|
||
else if (n === 'red') ri = i; else if (n === 'green') gi = i; else if (n === 'blue') bi = i;
|
||
else if (n === 'error' || n === 'deviation') errorI = i;
|
||
});
|
||
const x = parseFloat(parts[xi >= 0 ? xi : 0]);
|
||
const y = parseFloat(parts[yi >= 0 ? yi : 1]);
|
||
const z = parseFloat(parts[zi >= 0 ? zi : 2]);
|
||
if (!isFinite(x) || !isFinite(y) || !isFinite(z)) continue;
|
||
const pt: SampledPoint = { x, y, z };
|
||
if (errorI >= 0) {
|
||
const error = parseFloat(parts[errorI]);
|
||
if (Number.isFinite(error)) pt.error = error;
|
||
}
|
||
if (ri >= 0 && gi >= 0 && bi >= 0 && parts[ri] !== undefined) {
|
||
pt.r = parseFloat(parts[ri]);
|
||
pt.g = parseFloat(parts[gi]);
|
||
pt.b = parseFloat(parts[bi]);
|
||
}
|
||
points.push(pt);
|
||
}
|
||
} finally {
|
||
fs.closeSync(fd);
|
||
}
|
||
|
||
console.log(
|
||
`[PLY-ASCII] ${path.basename(filePath)}: sampled ${points.length} / ${header.vertexCount} verts`,
|
||
);
|
||
return points;
|
||
}
|
||
|
||
/**
|
||
* 通用采样入口：根据扩展名分发到对应采样器。
|
||
* 支持 .ply（ASCII + binary），.txt / .xyz / .pts（纯文本行）。
|
||
*/
|
||
export async function samplePointcloudFile(
|
||
filePath: string,
|
||
fileSize: number,
|
||
k: number,
|
||
): Promise<SampledPoint[]> {
|
||
if (fileSize === 0 || k === 0) return [];
|
||
const ext = path.extname(filePath).toLowerCase();
|
||
|
||
if (ext === '.ply') {
|
||
const header = parsePlyHeader(filePath);
|
||
if (!header) {
|
||
console.warn(`[PLY] ${path.basename(filePath)}: failed to parse header, falling back to text sampler`);
|
||
return randomSeekSample(filePath, fileSize, k);
|
||
}
|
||
if (header.format === 'ascii') {
|
||
return samplePlyAscii(filePath, fileSize, header, k);
|
||
}
|
||
return samplePlyBinary(filePath, header, k);
|
||
}
|
||
|
||
// .txt / .xyz / .pts / etc. — existing text line sampler
|
||
return randomSeekSample(filePath, fileSize, k);
|
||
}
|
||
|
||
/**
|
||
* 对一组已校验的点云文件按文件大小分配总采样配额，并编码为浏览器消费的二进制协议。
|
||
* 每点固定包含 x/y/z/error；任一文件带颜色时，所有点再包含归一化后的 r/g/b。
|
||
*/
|
||
export async function samplePointcloudFiles(
|
||
filePaths: string[],
|
||
requestedTotalPoints: number,
|
||
maxTotalPoints = 2_000_000,
|
||
): Promise<SampledPointcloudBatch> {
|
||
const totalPoints = Number.isFinite(requestedTotalPoints)
|
||
? Math.max(0, Math.min(Math.floor(requestedTotalPoints), maxTotalPoints))
|
||
: 0;
|
||
const fileInfos = filePaths.map((filePath) => ({
|
||
filePath,
|
||
name: path.basename(filePath),
|
||
size: fs.statSync(filePath).size,
|
||
}));
|
||
const totalBytes = fileInfos.reduce((sum, file) => sum + file.size, 0);
|
||
|
||
// floor 后把剩余名额依次补给文件，保证所有配额之和严格不超过总预算。
|
||
const quotas = fileInfos.map((file) => totalBytes > 0
|
||
? Math.floor((file.size / totalBytes) * totalPoints)
|
||
: 0);
|
||
let remainder = totalPoints - quotas.reduce((sum, quota) => sum + quota, 0);
|
||
for (let index = 0; remainder > 0 && fileInfos.length > 0; index = (index + 1) % fileInfos.length) {
|
||
quotas[index] += 1;
|
||
remainder -= 1;
|
||
}
|
||
|
||
const sampledPerFile = await Promise.all(
|
||
fileInfos.map((file, index) => samplePointcloudFile(file.filePath, file.size, quotas[index])),
|
||
);
|
||
const pointCount = sampledPerFile.reduce((sum, points) => sum + points.length, 0);
|
||
const hasError = sampledPerFile.some((points) => points.some((point) => Number.isFinite(point.error)));
|
||
const hasColor = sampledPerFile.some((points) => points.some((point) => point.r !== undefined));
|
||
const bytesPerPoint = hasColor ? 28 : 16;
|
||
const buffer = Buffer.allocUnsafe(pointCount * bytesPerPoint);
|
||
let offset = 0;
|
||
|
||
for (const points of sampledPerFile) {
|
||
for (const point of points) {
|
||
buffer.writeFloatLE(point.x, offset);
|
||
buffer.writeFloatLE(point.y, offset + 4);
|
||
buffer.writeFloatLE(point.z, offset + 8);
|
||
buffer.writeFloatLE(point.error ?? Number.NaN, offset + 12);
|
||
if (hasColor) {
|
||
buffer.writeFloatLE((point.r ?? 128) / 255, offset + 16);
|
||
buffer.writeFloatLE((point.g ?? 128) / 255, offset + 20);
|
||
buffer.writeFloatLE((point.b ?? 128) / 255, offset + 24);
|
||
}
|
||
offset += bytesPerPoint;
|
||
}
|
||
}
|
||
|
||
return {
|
||
buffer,
|
||
pointCount,
|
||
hasColor,
|
||
hasError,
|
||
fileStats: fileInfos.map((file, index) => ({
|
||
name: file.name,
|
||
sizeGB: +(file.size / 1e9).toFixed(3),
|
||
sampled: sampledPerFile[index].length,
|
||
})),
|
||
};
|
||
}
|