feat: initial commit - massive-cloud point cloud processing system

2026-05-27 09:51:42 +00:00
commit 16811100d1
391 changed files with 78338 additions and 0 deletions
--- a/server/pointcloudSample.ts
+++ b/server/pointcloudSample.ts
@@ -0,0 +1,323 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+export type SampledPoint = { x: number; y: number; z: number; error: number };
+
+/** 与 fileRoutes 批量采样一致：随机 seek 文本点云行 */
+export async function randomSeekSample(
+  filePath: string,
+  fileSize: number,
+  k: number,
+): Promise<SampledPoint[]> {
+  if (fileSize === 0 || k === 0) return [];
+
+  const PROBE_BYTES = 64 * 1024;
+  const probeBuf = Buffer.alloc(Math.min(PROBE_BYTES, fileSize));
+  const probeFd = fs.openSync(filePath, 'r');
+  fs.readSync(probeFd, probeBuf, 0, probeBuf.length, 0);
+  fs.closeSync(probeFd);
+
+  const probeLines = probeBuf.toString('utf8').split('\n').filter((l) => {
+    const t = l.trim();
+    return t && !t.startsWith('#') && !t.startsWith('//');
+  });
+  const sampleCount = Math.min(probeLines.length, 200);
+  const avgLineBytes =
+    sampleCount > 0
+      ? probeLines.slice(0, sampleCount).reduce((s, l) => s + Buffer.byteLength(l) + 1, 0) / sampleCount
+      : 40;
+
+  console.log(
+    `[SeekSampler] ${path.basename(filePath)}: size=${(fileSize / 1e9).toFixed(2)}GB ` +
+      `avgLine=${avgLineBytes.toFixed(1)}B quota=${k}`,
+  );
+
+  const startOffset = Math.min(512, fileSize);
+  const usableSize = fileSize - startOffset;
+  if (usableSize <= 0) return [];
+
+  const interval = usableSize / k;
+  const seekPositions: number[] = [];
+  for (let i = 0; i < k; i++) {
+    const base = startOffset + Math.floor(i * interval);
+    const jitter = Math.floor(Math.random() * Math.min(interval * 0.5, 1024));
+    seekPositions.push(Math.min(base + jitter, fileSize - 2));
+  }
+  seekPositions.sort((a, b) => a - b);
+
+  const points: SampledPoint[] = [];
+  const fd = fs.openSync(filePath, 'r');
+  const lineBuf = Buffer.alloc(1024);
+
+  try {
+    for (const pos of seekPositions) {
+      if (points.length >= k) break;
+      const bytesRead = fs.readSync(fd, lineBuf, 0, lineBuf.length, pos);
+      if (bytesRead < 4) continue;
+      const chunk = lineBuf.toString('utf8', 0, bytesRead);
+      const nl1 = chunk.indexOf('\n');
+      if (nl1 === -1) continue;
+      const afterFirst = chunk.slice(nl1 + 1);
+      const nl2 = afterFirst.indexOf('\n');
+      const line = (nl2 === -1 ? afterFirst : afterFirst.slice(0, nl2)).trim();
+      if (!line || line.startsWith('#') || line.startsWith('//')) continue;
+      const parts = line.split(/[\s,\t]+/);
+      const x = parseFloat(parts[0]);
+      const y = parseFloat(parts[1]);
+      const z = parseFloat(parts[2]);
+      if (isNaN(x) || isNaN(y) || isNaN(z)) continue;
+      let error = parts.length >= 4 ? parseFloat(parts[3]) : NaN;
+      if (isNaN(error)) error = Math.abs(Math.sin(x * 0.5) * Math.cos(z * 0.5)) * 1.5;
+      points.push({ x, y, z, error });
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+  return points;
+}
+
+// ─── PLY 格式解析与采样 ───────────────────────────────────────────────────────
+
+/** PLY 属性类型 → 字节数 */
+const PLY_PROP_SIZES: Record<string, number> = {
+  char: 1, int8: 1,
+  uchar: 1, uint8: 1,
+  short: 2, int16: 2,
+  ushort: 2, uint16: 2,
+  int: 4, int32: 4,
+  uint: 4, uint32: 4,
+  float: 4, float32: 4,
+  double: 8, float64: 8,
+};
+
+interface PlyHeader {
+  /** 'ascii' | 'binary_little_endian' | 'binary_big_endian' */
+  format: string;
+  /** 顶点数 */
+  vertexCount: number;
+  /** 顶点元素各属性，顺序与文件一致 */
+  properties: Array<{ name: string; type: string; byteSize: number }>;
+  /** 'end_header\n' 之后的第一个字节偏移量（即数据区起始位置） */
+  dataOffset: number;
+}
+
+/**
+ * 读取 PLY 文件头（总是 ASCII，以 "end_header\n" 结尾）。
+ * 最多读取 16 KB，足以容纳任何合理的 PLY 头。
+ */
+function parsePlyHeader(filePath: string): PlyHeader | null {
+  const HEADER_MAX = 16 * 1024;
+  const buf = Buffer.alloc(HEADER_MAX);
+  const fd = fs.openSync(filePath, 'r');
+  const bytesRead = fs.readSync(fd, buf, 0, HEADER_MAX, 0);
+  fs.closeSync(fd);
+
+  const raw = buf.toString('ascii', 0, bytesRead);
+
+  // 必须以 "ply\n" 开头
+  if (!raw.startsWith('ply\n') && !raw.startsWith('ply\r\n')) return null;
+
+  const endHeaderMarker = 'end_header';
+  const endIdx = raw.indexOf(endHeaderMarker);
+  if (endIdx === -1) return null;
+
+  // 找到 end_header 之后换行符，跳过它
+  let dataStart = endIdx + endHeaderMarker.length;
+  if (raw[dataStart] === '\r') dataStart++;
+  if (raw[dataStart] === '\n') dataStart++;
+
+  const header = raw.slice(0, endIdx);
+  const lines = header.split(/\r?\n/);
+
+  let format = 'ascii';
+  let vertexCount = 0;
+  const properties: PlyHeader['properties'] = [];
+  let inVertex = false;
+
+  for (const line of lines) {
+    const parts = line.trim().split(/\s+/);
+    if (parts[0] === 'format') {
+      format = parts[1] ?? 'ascii';
+    } else if (parts[0] === 'element') {
+      inVertex = parts[1] === 'vertex';
+      if (inVertex) vertexCount = parseInt(parts[2] ?? '0', 10);
+    } else if (parts[0] === 'property' && inVertex) {
+      // skip "property list ..." (face element)
+      if (parts[1] === 'list') { inVertex = false; continue; }
+      const type = parts[1] ?? 'float';
+      const name = parts[2] ?? '';
+      const byteSize = PLY_PROP_SIZES[type] ?? 4;
+      properties.push({ name, type, byteSize });
+    }
+  }
+
+  if (vertexCount <= 0 || properties.length === 0) return null;
+
+  return { format, vertexCount, properties, dataOffset: dataStart };
+}
+
+/**
+ * 采样二进制 PLY 文件（little_endian 或 big_endian）。
+ * 策略：均匀间隔 + 小量随机抖动（与 randomSeekSample 一致）。
+ */
+function samplePlyBinary(
+  filePath: string,
+  header: PlyHeader,
+  k: number,
+): SampledPoint[] {
+  const { vertexCount, properties, dataOffset, format } = header;
+  const littleEndian = format !== 'binary_big_endian';
+
+  const bytesPerVertex = properties.reduce((s, p) => s + p.byteSize, 0);
+
+  // 找 x, y, z（可能叫 x/y/z 或 X/Y/Z；允许宽松匹配）
+  let xOff = -1, yOff = -1, zOff = -1;
+  let xType = 'float', yType = 'float', zType = 'float';
+  let byteOff = 0;
+  for (const p of properties) {
+    const n = p.name.toLowerCase();
+    if (n === 'x') { xOff = byteOff; xType = p.type; }
+    else if (n === 'y') { yOff = byteOff; yType = p.type; }
+    else if (n === 'z') { zOff = byteOff; zType = p.type; }
+    byteOff += p.byteSize;
+  }
+
+  if (xOff === -1 || yOff === -1 || zOff === -1) {
+    console.warn(`[PLY] ${path.basename(filePath)}: could not find x/y/z properties`);
+    return [];
+  }
+
+  const actualK = Math.min(k, vertexCount);
+  const interval = vertexCount / actualK;
+  const indices: number[] = [];
+  for (let i = 0; i < actualK; i++) {
+    const base = Math.floor(i * interval);
+    const jitter = Math.floor(Math.random() * Math.max(1, interval * 0.5));
+    indices.push(Math.min(base + jitter, vertexCount - 1));
+  }
+
+  const recBuf = Buffer.alloc(bytesPerVertex);
+  const fd = fs.openSync(filePath, 'r');
+  const points: SampledPoint[] = [];
+
+  function readNum(buf: Buffer, off: number, type: string): number {
+    switch (type) {
+      case 'float': case 'float32': return littleEndian ? buf.readFloatLE(off) : buf.readFloatBE(off);
+      case 'double': case 'float64': return littleEndian ? buf.readDoubleLE(off) : buf.readDoubleBE(off);
+      case 'int': case 'int32': return littleEndian ? buf.readInt32LE(off) : buf.readInt32BE(off);
+      case 'uint': case 'uint32': return littleEndian ? buf.readUInt32LE(off) : buf.readUInt32BE(off);
+      case 'short': case 'int16': return littleEndian ? buf.readInt16LE(off) : buf.readInt16BE(off);
+      case 'ushort': case 'uint16': return littleEndian ? buf.readUInt16LE(off) : buf.readUInt16BE(off);
+      case 'char': case 'int8': return buf.readInt8(off);
+      case 'uchar': case 'uint8': return buf.readUInt8(off);
+      default: return littleEndian ? buf.readFloatLE(off) : buf.readFloatBE(off);
+    }
+  }
+
+  try {
+    for (const idx of indices) {
+      const filePos = dataOffset + idx * bytesPerVertex;
+      const n = fs.readSync(fd, recBuf, 0, bytesPerVertex, filePos);
+      if (n < bytesPerVertex) continue;
+      const x = readNum(recBuf, xOff, xType);
+      const y = readNum(recBuf, yOff, yType);
+      const z = readNum(recBuf, zOff, zType);
+      if (!isFinite(x) || !isFinite(y) || !isFinite(z)) continue;
+      const error = Math.abs(Math.sin(x * 0.001) * Math.cos(z * 0.001)) * 1.5;
+      points.push({ x, y, z, error });
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+
+  console.log(
+    `[PLY-Binary] ${path.basename(filePath)}: ${vertexCount} verts, ` +
+    `${bytesPerVertex}B/vert, sampled ${points.length}`,
+  );
+  return points;
+}
+
+/**
+ * 采样 ASCII PLY 文件。
+ * 复用 randomSeekSample 的均匀 seek 策略，跳过 PLY 头，从数据区随机取行。
+ */
+function samplePlyAscii(
+  filePath: string,
+  fileSize: number,
+  header: PlyHeader,
+  k: number,
+): SampledPoint[] {
+  const { dataOffset } = header;
+  const usableSize = fileSize - dataOffset;
+  if (usableSize <= 0) return [];
+
+  const actualK = Math.min(k, header.vertexCount);
+  const interval = usableSize / actualK;
+  const seekPositions: number[] = [];
+  for (let i = 0; i < actualK; i++) {
+    const base = dataOffset + Math.floor(i * interval);
+    const jitter = Math.floor(Math.random() * Math.min(interval * 0.5, 1024));
+    seekPositions.push(Math.min(base + jitter, fileSize - 2));
+  }
+
+  const points: SampledPoint[] = [];
+  const fd = fs.openSync(filePath, 'r');
+  const lineBuf = Buffer.alloc(512);
+
+  try {
+    for (const pos of seekPositions) {
+      if (points.length >= actualK) break;
+      const bytesRead = fs.readSync(fd, lineBuf, 0, lineBuf.length, pos);
+      if (bytesRead < 4) continue;
+      const chunk = lineBuf.toString('utf8', 0, bytesRead);
+      const nl1 = chunk.indexOf('\n');
+      if (nl1 === -1) continue;
+      const afterFirst = chunk.slice(nl1 + 1);
+      const nl2 = afterFirst.indexOf('\n');
+      const line = (nl2 === -1 ? afterFirst : afterFirst.slice(0, nl2)).trim();
+      if (!line) continue;
+      const parts = line.split(/\s+/);
+      const x = parseFloat(parts[0]);
+      const y = parseFloat(parts[1]);
+      const z = parseFloat(parts[2]);
+      if (!isFinite(x) || !isFinite(y) || !isFinite(z)) continue;
+      const error = Math.abs(Math.sin(x * 0.001) * Math.cos(z * 0.001)) * 1.5;
+      points.push({ x, y, z, error });
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+
+  console.log(
+    `[PLY-ASCII] ${path.basename(filePath)}: sampled ${points.length} / ${header.vertexCount} verts`,
+  );
+  return points;
+}
+
+/**
+ * 通用采样入口：根据扩展名分发到对应采样器。
+ * 支持 .ply（ASCII + binary），.txt / .xyz / .pts（纯文本行）。
+ */
+export async function samplePointcloudFile(
+  filePath: string,
+  fileSize: number,
+  k: number,
+): Promise<SampledPoint[]> {
+  if (fileSize === 0 || k === 0) return [];
+  const ext = path.extname(filePath).toLowerCase();
+
+  if (ext === '.ply') {
+    const header = parsePlyHeader(filePath);
+    if (!header) {
+      console.warn(`[PLY] ${path.basename(filePath)}: failed to parse header, falling back to text sampler`);
+      return randomSeekSample(filePath, fileSize, k);
+    }
+    if (header.format === 'ascii') {
+      return samplePlyAscii(filePath, fileSize, header, k);
+    }
+    return samplePlyBinary(filePath, header, k);
+  }
+
+  // .txt / .xyz / .pts / etc. — existing text line sampler
+  return randomSeekSample(filePath, fileSize, k);
+}