Compare commits
3 Commits
29a69793db
...
ac1387debc
| Author | SHA1 | Date | |
|---|---|---|---|
| ac1387debc | |||
| d33b280a93 | |||
| 9f4482bea1 |
39
.agents/skills/package-source-sha1/SKILL.md
Normal file
39
.agents/skills/package-source-sha1/SKILL.md
Normal file
@@ -0,0 +1,39 @@
|
||||
---
|
||||
name: package-source-sha1
|
||||
description: 为 massive-cloud-v3 生成可复现的纯源码 ZIP 包、内容清单和 40 位 SHA-1。用户要求打包整个项目、交付源码包、排除点云/CAD/数据库/构建产物/密钥,或计算项目 ZIP 哈希时使用。
|
||||
---
|
||||
|
||||
# 打包源码并计算 SHA-1
|
||||
|
||||
使用 `scripts/package_source.py`,不要临时拼装 `find`、`tar` 或 `zip` 命令。
|
||||
|
||||
## 工作流
|
||||
|
||||
1. 根据项目 `AGENTS.md` 查看开发服务器 `/home/lipan/massive-cloud-v3` 的实时工作树和 `git status`。
|
||||
2. 先运行 `--dry-run` 检查纳入、排除数量和排除原因;发现异常大文件或新的数据格式时,先更新脚本规则。
|
||||
3. 运行正式打包。脚本只从 `git ls-files --cached --others --exclude-standard` 取候选文件,因此包含已跟踪文件和未忽略的新文件,也包含尚未提交的实际代码改动。
|
||||
4. 将 ZIP、`.sha1` 和 `.manifest.json` 一起交付。明确说明包是否包含未提交改动,并报告基线提交。
|
||||
5. 用 `unzip -t` 验证 ZIP;再独立运行 `sha1sum` 或 `shasum -a 1`,确认与 `.sha1` 一致且为 40 位小写十六进制。
|
||||
|
||||
## 固定规则
|
||||
|
||||
- 保留源码、测试、SQL、配置模板、依赖清单/锁文件、容器配置、脚本和文档。
|
||||
- 排除 Git 历史、被 `.gitignore` 忽略的内容、真实 `.env`、密钥、日志、缓存、依赖目录和构建产物。
|
||||
- 排除点云、CAD/网格、数据库、表格结果、归档包、模型权重等非源码文件。
|
||||
- 不要使用 `git archive`:它会遗漏工作区中尚未提交的实际代码。
|
||||
- ZIP 使用固定时间戳、固定顺序和规范权限;相同文件内容应生成相同哈希。
|
||||
|
||||
## 命令
|
||||
|
||||
```bash
|
||||
python3 scripts/package_source.py \
|
||||
--source /home/lipan/massive-cloud-v3 \
|
||||
--output /tmp/massive-cloud-v3-source.zip \
|
||||
--dry-run
|
||||
|
||||
python3 scripts/package_source.py \
|
||||
--source /home/lipan/massive-cloud-v3 \
|
||||
--output /tmp/massive-cloud-v3-source.zip
|
||||
```
|
||||
|
||||
不要把产物写进被打包的源码目录;优先使用 `/tmp`,再复制到交付位置。
|
||||
4
.agents/skills/package-source-sha1/agents/openai.yaml
Normal file
4
.agents/skills/package-source-sha1/agents/openai.yaml
Normal file
@@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "源码包 SHA-1"
|
||||
short_description: "可复现地打包项目纯源码并自动生成 SHA-1 校验值"
|
||||
default_prompt: "使用 $package-source-sha1 打包当前项目纯源码并返回 ZIP 的 SHA-1。"
|
||||
176
.agents/skills/package-source-sha1/scripts/package_source.py
Normal file
176
.agents/skills/package-source-sha1/scripts/package_source.py
Normal file
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a reproducible source-only ZIP and its SHA-1/manifest files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
EXCLUDED_SUFFIXES = {
|
||||
# Point clouds, CAD and meshes
|
||||
".ply", ".pcd", ".las", ".laz", ".e57", ".xyz", ".pts",
|
||||
".stp", ".step", ".iges", ".igs", ".obj", ".stl", ".glb", ".gltf",
|
||||
# Databases and generated tabular results
|
||||
".db", ".sqlite", ".sqlite3", ".mdb", ".accdb", ".xlsx", ".xls",
|
||||
# Logs, caches, backups and generated archives
|
||||
".log", ".pyc", ".pyo", ".bak", ".tmp", ".zip", ".7z", ".rar",
|
||||
".tar", ".gz", ".bz2", ".xz",
|
||||
# Model weights / binary artifacts
|
||||
".pt", ".pth", ".onnx", ".ckpt", ".safetensors", ".bin",
|
||||
# Private-key/container formats
|
||||
".pem", ".key", ".p12", ".pfx", ".jks",
|
||||
}
|
||||
|
||||
EXCLUDED_DIRS = {
|
||||
".git", ".manus-logs", ".pnpm-store", "node_modules", "dist", "build",
|
||||
"coverage", "results", "result", "outputs", "output", "__pycache__",
|
||||
".pytest_cache", ".mypy_cache", ".ruff_cache", ".cache", "data", "datasets",
|
||||
}
|
||||
|
||||
EXCLUDED_NAMES = {".env", "credentials.json", "secrets.json", "=1.10", "=3.1"}
|
||||
FIXED_ZIP_TIME = (1980, 1, 1, 0, 0, 0)
|
||||
ZIP_ROOT = "massive-cloud-v3-source"
|
||||
|
||||
|
||||
def git_output(root: Path, *args: str) -> bytes:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), *args], check=True, stdout=subprocess.PIPE
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def candidates(root: Path) -> list[str]:
|
||||
raw = git_output(root, "ls-files", "--cached", "--others", "--exclude-standard", "-z")
|
||||
return sorted(os.fsdecode(item) for item in raw.split(b"\0") if item)
|
||||
|
||||
|
||||
def exclusion_reason(relative: str) -> str | None:
|
||||
path = PurePosixPath(relative)
|
||||
lowered_parts = tuple(part.lower() for part in path.parts)
|
||||
name = path.name.lower()
|
||||
|
||||
if any(part in EXCLUDED_DIRS for part in lowered_parts[:-1]):
|
||||
return "excluded-directory"
|
||||
if name in EXCLUDED_NAMES:
|
||||
return "secret-or-artifact-name"
|
||||
if name.startswith(".env.") and name not in {".env.example", ".env.template", ".env.sample"}:
|
||||
return "environment-secret"
|
||||
if path.suffix.lower() in EXCLUDED_SUFFIXES:
|
||||
return "non-source-suffix"
|
||||
return None
|
||||
|
||||
|
||||
def sha1_file(path: Path) -> str:
|
||||
digest = hashlib.sha1()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def git_text(root: Path, *args: str) -> str:
|
||||
return git_output(root, *args).decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def build_manifest(root: Path, included: list[str], excluded: list[dict[str, str]]) -> dict:
|
||||
return {
|
||||
"format": "massive-cloud-source-package-v1",
|
||||
"source_root": str(root),
|
||||
"git_branch": git_text(root, "branch", "--show-current"),
|
||||
"git_commit": git_text(root, "rev-parse", "HEAD"),
|
||||
"git_status": git_text(root, "status", "--short"),
|
||||
"included_count": len(included),
|
||||
"included_files": included,
|
||||
"excluded_count": len(excluded),
|
||||
"excluded_files": excluded,
|
||||
"rules": {
|
||||
"candidate_source": "git ls-files --cached --others --exclude-standard",
|
||||
"excluded_directories": sorted(EXCLUDED_DIRS),
|
||||
"excluded_names": sorted(EXCLUDED_NAMES),
|
||||
"excluded_suffixes": sorted(EXCLUDED_SUFFIXES),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_zip(root: Path, output: Path, included: list[str]) -> None:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive:
|
||||
for relative in included:
|
||||
source = root / relative
|
||||
if not source.is_file() or source.is_symlink():
|
||||
continue
|
||||
info = zipfile.ZipInfo(f"{ZIP_ROOT}/{PurePosixPath(relative)}", FIXED_ZIP_TIME)
|
||||
mode = source.stat().st_mode
|
||||
normalized_mode = 0o755 if mode & stat.S_IXUSR else 0o644
|
||||
info.create_system = 3
|
||||
info.external_attr = (stat.S_IFREG | normalized_mode) << 16
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
with source.open("rb") as handle:
|
||||
archive.writestr(info, handle.read(), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source", type=Path, required=True, help="Git working tree")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output .zip path")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print selection summary only")
|
||||
args = parser.parse_args()
|
||||
|
||||
root = args.source.resolve()
|
||||
output = args.output.resolve()
|
||||
if output.suffix.lower() != ".zip":
|
||||
parser.error("--output must end with .zip")
|
||||
if not (root / ".git").exists():
|
||||
parser.error(f"not a Git working tree: {root}")
|
||||
try:
|
||||
output.relative_to(root)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
parser.error("write the package outside the source tree")
|
||||
|
||||
selected: list[str] = []
|
||||
excluded: list[dict[str, str]] = []
|
||||
for relative in candidates(root):
|
||||
reason = exclusion_reason(relative)
|
||||
if reason:
|
||||
excluded.append({"path": relative, "reason": reason})
|
||||
elif (root / relative).is_file() and not (root / relative).is_symlink():
|
||||
selected.append(relative)
|
||||
|
||||
reasons = Counter(item["reason"] for item in excluded)
|
||||
print(f"source: {root}")
|
||||
print(f"included: {len(selected)} files")
|
||||
print(f"excluded: {len(excluded)} files ({dict(sorted(reasons.items()))})")
|
||||
print(f"git commit: {git_text(root, 'rev-parse', 'HEAD')}")
|
||||
status = git_text(root, "status", "--short")
|
||||
print("working tree: dirty" if status else "working tree: clean")
|
||||
if args.dry_run:
|
||||
return 0
|
||||
|
||||
manifest = build_manifest(root, selected, excluded)
|
||||
write_zip(root, output, selected)
|
||||
digest = sha1_file(output)
|
||||
manifest.update({"zip_file": output.name, "zip_size": output.stat().st_size, "sha1": digest})
|
||||
|
||||
sha_path = output.with_suffix(output.suffix + ".sha1")
|
||||
manifest_path = output.with_suffix(output.suffix + ".manifest.json")
|
||||
sha_path.write_text(f"{digest} {output.name}\n", encoding="ascii")
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"zip: {output}")
|
||||
print(f"sha1: {digest}")
|
||||
print(f"manifest: {manifest_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1890,21 +1890,32 @@ export default function PointCloudViewer({
|
||||
setPlyOverlayStatus({ status: 'loading', message: '正在加载 PLY 叠加层' });
|
||||
(async () => {
|
||||
try {
|
||||
// 用 /sample/batch 接口按 filePath 采样,最多 30 万点
|
||||
// 支持多文件(各面板 PLY),合并采样
|
||||
// 通过受限的分析结果接口加载相对 PLY 路径,支持多文件合并采样
|
||||
const plyFiles: string[] = Array.isArray(overlay.files) && overlay.files.length > 0
|
||||
? overlay.files
|
||||
: [overlay.file];
|
||||
const resp = await fetch('/api/files/sample/batch', {
|
||||
const resp = await fetch('/api/files/sample/analysis', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
signal: ac.signal,
|
||||
body: JSON.stringify({
|
||||
filePaths: plyFiles.map(f => `/data/tasks/${taskId}/analysis/${resultId}/${f}`),
|
||||
taskId,
|
||||
resultId,
|
||||
files: plyFiles,
|
||||
totalPoints: 300_000,
|
||||
}),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
if (!resp.ok) {
|
||||
const payload = await resp.json().catch(() => null) as { error?: string } | null;
|
||||
const detail = payload?.error;
|
||||
if (resp.status === 404) {
|
||||
throw new Error(detail ?? '分析结果或 PLY 文件不存在');
|
||||
}
|
||||
if (resp.status === 400 || resp.status === 403) {
|
||||
throw new Error(detail ?? 'PLY 文件参数或访问权限无效');
|
||||
}
|
||||
throw new Error(detail ?? `服务端采样失败(HTTP ${resp.status})`);
|
||||
}
|
||||
const hasColor = resp.headers.get('X-Has-Color') === '1';
|
||||
const stride = hasColor ? 7 : 4;
|
||||
const buf = await resp.arrayBuffer();
|
||||
@@ -1950,7 +1961,10 @@ export default function PointCloudViewer({
|
||||
if (e instanceof DOMException && e.name === 'AbortError') return;
|
||||
console.warn('[PlyOverlay] load failed:', e);
|
||||
setPlyOverlayData(null);
|
||||
setPlyOverlayStatus({ status: 'error', message: '加载 PLY 叠加层失败' });
|
||||
setPlyOverlayStatus({
|
||||
status: 'error',
|
||||
message: e instanceof Error ? e.message : '服务端采样失败,请稍后重试',
|
||||
});
|
||||
}
|
||||
})();
|
||||
return () => ac.abort();
|
||||
|
||||
@@ -240,6 +240,19 @@ export default function TaskManager({
|
||||
const activeInspectionObject = inspectionObjects.find(
|
||||
object => object.id === activeTask?.inspectionObjectId
|
||||
);
|
||||
|
||||
// 打开新建任务表单时默认选择首个可用检测对象;若列表异步刷新,
|
||||
// 则仅在当前选择失效时回退到第一项,避免覆盖用户的手动选择。
|
||||
useEffect(() => {
|
||||
if (!adding) return;
|
||||
setNewTaskInspectionObjectId(currentId => {
|
||||
if (inspectionObjects.some(object => object.id === currentId)) {
|
||||
return currentId;
|
||||
}
|
||||
return inspectionObjects[0]?.id ?? null;
|
||||
});
|
||||
}, [adding, inspectionObjects]);
|
||||
|
||||
const createTask = trpc.inspectionTask.create.useMutation({
|
||||
onSuccess: result => {
|
||||
pendingDefaultSelection.current = {
|
||||
@@ -692,6 +705,7 @@ export default function TaskManager({
|
||||
<button
|
||||
className="industrial-btn flex shrink-0 items-center gap-0.5 whitespace-nowrap px-2 py-0.5 text-[10px]"
|
||||
onClick={() => {
|
||||
setNewTaskInspectionObjectId(inspectionObjects[0]?.id ?? null);
|
||||
setAdding(true);
|
||||
onStartCreate?.();
|
||||
}}
|
||||
|
||||
126
server/analysisPlySample.test.ts
Normal file
126
server/analysisPlySample.test.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
import express from 'express';
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { AddressInfo } from 'node:net';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
authenticated: true,
|
||||
resultTaskId: 7,
|
||||
}));
|
||||
|
||||
vi.mock('./_core/auth.js', () => ({
|
||||
authenticateRequest: vi.fn(async () => mocks.authenticated
|
||||
? { id: 1, role: 'user', mustChangePassword: 0 }
|
||||
: null),
|
||||
}));
|
||||
|
||||
vi.mock('./db', () => ({
|
||||
getTaskAnalysisResultById: vi.fn(async (id: number) => id === 11
|
||||
? { id, taskId: mocks.resultTaskId }
|
||||
: undefined),
|
||||
listTaskPointcloudFiles: vi.fn(async () => []),
|
||||
}));
|
||||
|
||||
const dataDir = mkdtempSync(join(tmpdir(), 'analysis-ply-route-'));
|
||||
process.env.DATA_DIR = dataDir;
|
||||
|
||||
function writePly(relativePath: string, withColor = false): void {
|
||||
const resultDir = join(dataDir, 'tasks', '7', 'analysis', '11');
|
||||
const filePath = join(resultDir, relativePath);
|
||||
mkdirSync(join(filePath, '..'), { recursive: true });
|
||||
const properties = withColor
|
||||
? ['x', 'y', 'z', 'red', 'green', 'blue']
|
||||
: ['x', 'y', 'z'];
|
||||
const header = Buffer.from([
|
||||
'ply',
|
||||
'format binary_little_endian 1.0',
|
||||
'element vertex 2',
|
||||
...properties.map((name) => `property float ${name}`),
|
||||
'end_header',
|
||||
'',
|
||||
].join('\n'), 'ascii');
|
||||
const values = withColor
|
||||
? [[1, 2, 3, 255, 0, 128], [4, 5, 6, 0, 255, 64]]
|
||||
: [[7, 8, 9], [10, 11, 12]];
|
||||
const body = Buffer.alloc(values.length * properties.length * 4);
|
||||
values.flat().forEach((value, index) => body.writeFloatLE(value, index * 4));
|
||||
writeFileSync(filePath, Buffer.concat([header, body]));
|
||||
}
|
||||
|
||||
describe('POST /api/files/sample/analysis', () => {
|
||||
let origin: string;
|
||||
let closeServer: () => Promise<void>;
|
||||
|
||||
beforeAll(async () => {
|
||||
writePly('plain.ply');
|
||||
writePly('nested/color.ply', true);
|
||||
const { fileRouter } = await import('./fileRoutes');
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use('/api/files', fileRouter);
|
||||
const server = app.listen(0);
|
||||
await new Promise<void>((resolve) => server.once('listening', resolve));
|
||||
origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
closeServer = () => new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await closeServer();
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mocks.authenticated = true;
|
||||
mocks.resultTaskId = 7;
|
||||
});
|
||||
|
||||
async function post(body: unknown) {
|
||||
return fetch(`${origin}/api/files/sample/analysis`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
it('采样单个无色 PLY 并返回 16 字节步长', async () => {
|
||||
const response = await post({ taskId: 7, resultId: 11, files: ['plain.ply'], totalPoints: 2 });
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('X-Point-Count')).toBe('2');
|
||||
expect(response.headers.get('X-Has-Color')).toBe('0');
|
||||
expect((await response.arrayBuffer()).byteLength).toBe(2 * 16);
|
||||
});
|
||||
|
||||
it('采样多个 PLY,彩色响应使用 28 字节步长且不超预算', async () => {
|
||||
const response = await post({
|
||||
taskId: 7,
|
||||
resultId: 11,
|
||||
files: ['plain.ply', 'nested/color.ply'],
|
||||
totalPoints: 3,
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('X-Point-Count')).toBe('3');
|
||||
expect(response.headers.get('X-Has-Color')).toBe('1');
|
||||
expect((await response.arrayBuffer()).byteLength).toBe(3 * 28);
|
||||
});
|
||||
|
||||
it('拒绝未登录访问', async () => {
|
||||
mocks.authenticated = false;
|
||||
expect((await post({ taskId: 7, resultId: 11, files: ['plain.ply'] })).status).toBe(401);
|
||||
});
|
||||
|
||||
it('拒绝错误的任务与结果组合', async () => {
|
||||
mocks.resultTaskId = 8;
|
||||
expect((await post({ taskId: 7, resultId: 11, files: ['plain.ply'] })).status).toBe(404);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['路径穿越', '../plain.ply', 403],
|
||||
['绝对路径', '/tmp/plain.ply', 400],
|
||||
['非 PLY 文件', 'result.json', 400],
|
||||
['缺失文件', 'missing.ply', 404],
|
||||
])('拒绝%s', async (_label, file, status) => {
|
||||
expect((await post({ taskId: 7, resultId: 11, files: [file] })).status).toBe(status);
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import multer from 'multer';
|
||||
import * as db from './db';
|
||||
import { samplePointcloudFile, type SampledPoint } from './pointcloudSample.js';
|
||||
import { samplePointcloudFiles } from './pointcloudSample.js';
|
||||
import { forceReload as reloadPluginRegistry } from './plugins/registry.js';
|
||||
import { authenticateRequest } from './_core/auth.js';
|
||||
import { inspectPointcloudMetadata } from './pointcloudMetadata.js';
|
||||
@@ -88,7 +88,19 @@ function formatFileSize(bytes: number): string {
|
||||
|
||||
function safeResolve(base: string, filename: string): string | null {
|
||||
const resolved = path.resolve(base, decodeURIComponent(filename));
|
||||
return resolved.startsWith(path.resolve(base)) ? resolved : null;
|
||||
const relative = path.relative(path.resolve(base), resolved);
|
||||
return relative !== '' && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)
|
||||
? resolved
|
||||
: null;
|
||||
}
|
||||
|
||||
function sendSampledPointcloudBatch(res: Parameters<RequestHandler>[1], batch: Awaited<ReturnType<typeof samplePointcloudFiles>>) {
|
||||
res.setHeader('Content-Type', 'application/octet-stream');
|
||||
res.setHeader('X-Point-Count', batch.pointCount);
|
||||
res.setHeader('X-Has-Color', batch.hasColor ? '1' : '0');
|
||||
res.setHeader('X-Has-Error', batch.hasError ? '1' : '0');
|
||||
res.setHeader('X-File-Stats', JSON.stringify(batch.fileStats));
|
||||
res.send(batch.buffer);
|
||||
}
|
||||
|
||||
// ─── 车型 CAD 文件上传 ────────────────────────────────────────────────────────
|
||||
@@ -333,7 +345,6 @@ router.post('/sample/batch', requireFileRead, async (req, res) => {
|
||||
return res.send(empty);
|
||||
}
|
||||
|
||||
const budget = Math.min(totalPoints, 2000000);
|
||||
const supportedExts = ['.ply', '.txt', '.xyz', '.pts'];
|
||||
|
||||
const fileInfos = paths.map((fp) => ({
|
||||
@@ -348,66 +359,100 @@ router.post('/sample/batch', requireFileRead, async (req, res) => {
|
||||
|
||||
console.log(
|
||||
`[FileRoutes] Batch sample: ${fileInfos.length} files ` +
|
||||
`(${(totalBytes / 1e9).toFixed(2)} GB), budget=${budget}`
|
||||
`(${(totalBytes / 1e9).toFixed(2)} GB), budget=${totalPoints}`
|
||||
);
|
||||
|
||||
try {
|
||||
const quotaMap = new Map<string, number>();
|
||||
for (const f of supportedFiles) {
|
||||
const quota = totalBytes > 0
|
||||
? Math.max(100, Math.round((f.size / totalBytes) * budget))
|
||||
: Math.floor(budget / supportedFiles.length);
|
||||
quotaMap.set(f.filePath, quota);
|
||||
}
|
||||
|
||||
const sampledPerFile = await Promise.all(
|
||||
fileInfos.map((f) => {
|
||||
if (!supportedExts.includes(f.ext) || f.size === 0) return Promise.resolve([] as SampledPoint[]);
|
||||
return samplePointcloudFile(f.filePath, f.size, quotaMap.get(f.filePath) ?? 0);
|
||||
})
|
||||
);
|
||||
|
||||
const totalSampled = sampledPerFile.reduce((s, pts) => s + pts.length, 0);
|
||||
// 如果任意点有颜色,则输出 28 字节/点(x,y,z,error,r,g,b 各 float32);否则 16 字节
|
||||
const hasError = sampledPerFile.some((pts) => pts.some((p) => Number.isFinite(p.error)));
|
||||
const hasColor = sampledPerFile.some((pts) => pts.some((p) => p.r !== undefined));
|
||||
const bytesPerPoint = hasColor ? 28 : 16;
|
||||
const binBuf = Buffer.allocUnsafe(totalSampled * bytesPerPoint);
|
||||
let offset = 0;
|
||||
for (const pts of sampledPerFile) {
|
||||
for (const p of pts) {
|
||||
binBuf.writeFloatLE(p.x, offset);
|
||||
binBuf.writeFloatLE(p.y, offset + 4);
|
||||
binBuf.writeFloatLE(p.z, offset + 8);
|
||||
binBuf.writeFloatLE(p.error ?? Number.NaN, offset + 12);
|
||||
if (hasColor) {
|
||||
binBuf.writeFloatLE((p.r ?? 128) / 255, offset + 16);
|
||||
binBuf.writeFloatLE((p.g ?? 128) / 255, offset + 20);
|
||||
binBuf.writeFloatLE((p.b ?? 128) / 255, offset + 24);
|
||||
}
|
||||
offset += bytesPerPoint;
|
||||
}
|
||||
}
|
||||
|
||||
const fileStats = fileInfos.map((f, i) => ({
|
||||
name: f.name,
|
||||
sizeGB: +(f.size / 1e9).toFixed(3),
|
||||
sampled: sampledPerFile[i].length,
|
||||
}));
|
||||
|
||||
console.log(`[FileRoutes] Batch done: ${totalSampled} pts, ${(binBuf.length / 1e6).toFixed(1)} MB`);
|
||||
res.setHeader('Content-Type', 'application/octet-stream');
|
||||
res.setHeader('X-Point-Count', totalSampled);
|
||||
res.setHeader('X-Has-Color', hasColor ? '1' : '0');
|
||||
res.setHeader('X-Has-Error', hasError ? '1' : '0');
|
||||
res.setHeader('X-File-Stats', JSON.stringify(fileStats));
|
||||
res.send(binBuf);
|
||||
const batch = await samplePointcloudFiles(supportedFiles.map((file) => file.filePath), totalPoints);
|
||||
console.log(`[FileRoutes] Batch done: ${batch.pointCount} pts, ${(batch.buffer.length / 1e6).toFixed(1)} MB`);
|
||||
sendSampledPointcloudBatch(res, batch);
|
||||
} catch (err) {
|
||||
console.error('[FileRoutes] Batch sampling error:', err);
|
||||
res.status(500).json({ error: 'Failed to sample files' });
|
||||
}
|
||||
});
|
||||
|
||||
// ─── 分析结果 PLY 批量采样接口 ───────────────────────────────────────────────
|
||||
// POST /api/files/sample/analysis
|
||||
// Body: { taskId: number, resultId: number, files: string[], totalPoints?: number }
|
||||
|
||||
router.post('/sample/analysis', requireFileRead, async (req, res) => {
|
||||
const { taskId, resultId, files, totalPoints = 300_000 } = req.body as {
|
||||
taskId?: number;
|
||||
resultId?: number;
|
||||
files?: unknown;
|
||||
totalPoints?: number;
|
||||
};
|
||||
if (!Number.isSafeInteger(taskId) || Number(taskId) <= 0 ||
|
||||
!Number.isSafeInteger(resultId) || Number(resultId) <= 0 ||
|
||||
!Number.isSafeInteger(totalPoints) || totalPoints <= 0) {
|
||||
return res.status(400).json({ error: '分析结果采样参数无效' });
|
||||
}
|
||||
if (!Array.isArray(files) || files.length === 0 ||
|
||||
files.some((file) => typeof file !== 'string' || file.length === 0)) {
|
||||
return res.status(400).json({ error: '请提供 PLY 文件列表' });
|
||||
}
|
||||
|
||||
const validTaskId = taskId as number;
|
||||
const validResultId = resultId as number;
|
||||
const validTotalPoints = totalPoints as number;
|
||||
|
||||
const row = await db.getTaskAnalysisResultById(validResultId);
|
||||
if (!row || row.taskId !== validTaskId) {
|
||||
return res.status(404).json({ error: '分析结果不存在或不属于当前任务' });
|
||||
}
|
||||
|
||||
const baseDir = path.join(TASKS_DIR, String(validTaskId), 'analysis', String(validResultId));
|
||||
let realBaseDir: string;
|
||||
try {
|
||||
realBaseDir = fs.realpathSync(baseDir);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return res.status(404).json({ error: '分析结果目录不存在' });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const resolvedFiles: string[] = [];
|
||||
for (const file of files as string[]) {
|
||||
if (path.isAbsolute(file) || path.extname(file).toLowerCase() !== '.ply') {
|
||||
return res.status(400).json({ error: '只允许读取分析结果目录中的相对 PLY 路径' });
|
||||
}
|
||||
let filePath: string | null;
|
||||
try {
|
||||
filePath = safeResolve(baseDir, file);
|
||||
} catch {
|
||||
return res.status(400).json({ error: 'PLY 文件路径编码无效' });
|
||||
}
|
||||
if (!filePath) return res.status(403).json({ error: 'PLY 文件路径越界' });
|
||||
let stat: fs.Stats;
|
||||
try {
|
||||
stat = fs.lstatSync(filePath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return res.status(404).json({ error: `PLY 文件不存在: ${file}` });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!stat.isFile() || stat.isSymbolicLink()) {
|
||||
return res.status(400).json({ error: `PLY 路径不是普通文件: ${file}` });
|
||||
}
|
||||
const realFilePath = fs.realpathSync(filePath);
|
||||
const realRelative = path.relative(realBaseDir, realFilePath);
|
||||
if (realRelative === '' || realRelative === '..' || realRelative.startsWith(`..${path.sep}`) || path.isAbsolute(realRelative)) {
|
||||
return res.status(403).json({ error: 'PLY 文件路径越界' });
|
||||
}
|
||||
resolvedFiles.push(realFilePath);
|
||||
}
|
||||
|
||||
try {
|
||||
const batch = await samplePointcloudFiles(resolvedFiles, validTotalPoints, 300_000);
|
||||
sendSampledPointcloudBatch(res, batch);
|
||||
} catch (error) {
|
||||
console.error('[FileRoutes] Analysis PLY sampling error:', error);
|
||||
return res.status(500).json({ error: '分析结果 PLY 采样失败' });
|
||||
}
|
||||
});
|
||||
|
||||
// ─── 平滑热力分析结果(异步任务产物,仅读取已落盘的顶点色)──────────────────────
|
||||
// GET /api/files/read/analysis/heatmap/:taskId/:resultId
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { samplePointcloudFile } from './pointcloudSample';
|
||||
import { samplePointcloudFile, samplePointcloudFiles } from './pointcloudSample';
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
@@ -58,3 +58,32 @@ describe('samplePointcloudFile', () => {
|
||||
expect(points.map((point) => point.error)).toEqual([0.25, -0.5]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('samplePointcloudFiles', () => {
|
||||
it('合并彩色与无色 PLY,并使用 28 字节彩色步长', async () => {
|
||||
const plain = writeBinaryPly(['x', 'y', 'z'], [[1, 2, 3], [4, 5, 6]]);
|
||||
const colored = writeBinaryPly(
|
||||
['x', 'y', 'z', 'red', 'green', 'blue'],
|
||||
[[7, 8, 9, 255, 128, 0], [10, 11, 12, 0, 64, 255]],
|
||||
);
|
||||
|
||||
const result = await samplePointcloudFiles([plain, colored], 4, 10);
|
||||
|
||||
expect(result.pointCount).toBe(4);
|
||||
expect(result.hasColor).toBe(true);
|
||||
expect(result.hasError).toBe(false);
|
||||
expect(result.buffer.byteLength).toBe(4 * 28);
|
||||
expect(result.fileStats.map((file) => file.sampled)).toEqual([2, 2]);
|
||||
});
|
||||
|
||||
it('限制总采样数量,无色 PLY 使用 16 字节步长', async () => {
|
||||
const first = writeBinaryPly(['x', 'y', 'z'], Array.from({ length: 10 }, (_, i) => [i, i, i]));
|
||||
const second = writeBinaryPly(['x', 'y', 'z'], Array.from({ length: 10 }, (_, i) => [i + 10, i, i]));
|
||||
|
||||
const result = await samplePointcloudFiles([first, second], 100, 6);
|
||||
|
||||
expect(result.pointCount).toBe(6);
|
||||
expect(result.hasColor).toBe(false);
|
||||
expect(result.buffer.byteLength).toBe(6 * 16);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,14 @@ import * as path from 'path';
|
||||
|
||||
export type SampledPoint = { x: number; y: number; z: number; error?: number; r?: number; g?: number; b?: number };
|
||||
|
||||
export type SampledPointcloudBatch = {
|
||||
buffer: Buffer;
|
||||
pointCount: number;
|
||||
hasColor: boolean;
|
||||
hasError: boolean;
|
||||
fileStats: Array<{ name: string; sizeGB: number; sampled: number }>;
|
||||
};
|
||||
|
||||
/** 与 fileRoutes 批量采样一致:随机 seek 文本点云行 */
|
||||
export async function randomSeekSample(
|
||||
filePath: string,
|
||||
@@ -356,3 +364,70 @@ export async function samplePointcloudFile(
|
||||
// .txt / .xyz / .pts / etc. — existing text line sampler
|
||||
return randomSeekSample(filePath, fileSize, k);
|
||||
}
|
||||
|
||||
/**
|
||||
* 对一组已校验的点云文件按文件大小分配总采样配额,并编码为浏览器消费的二进制协议。
|
||||
* 每点固定包含 x/y/z/error;任一文件带颜色时,所有点再包含归一化后的 r/g/b。
|
||||
*/
|
||||
export async function samplePointcloudFiles(
|
||||
filePaths: string[],
|
||||
requestedTotalPoints: number,
|
||||
maxTotalPoints = 2_000_000,
|
||||
): Promise<SampledPointcloudBatch> {
|
||||
const totalPoints = Number.isFinite(requestedTotalPoints)
|
||||
? Math.max(0, Math.min(Math.floor(requestedTotalPoints), maxTotalPoints))
|
||||
: 0;
|
||||
const fileInfos = filePaths.map((filePath) => ({
|
||||
filePath,
|
||||
name: path.basename(filePath),
|
||||
size: fs.statSync(filePath).size,
|
||||
}));
|
||||
const totalBytes = fileInfos.reduce((sum, file) => sum + file.size, 0);
|
||||
|
||||
// floor 后把剩余名额依次补给文件,保证所有配额之和严格不超过总预算。
|
||||
const quotas = fileInfos.map((file) => totalBytes > 0
|
||||
? Math.floor((file.size / totalBytes) * totalPoints)
|
||||
: 0);
|
||||
let remainder = totalPoints - quotas.reduce((sum, quota) => sum + quota, 0);
|
||||
for (let index = 0; remainder > 0 && fileInfos.length > 0; index = (index + 1) % fileInfos.length) {
|
||||
quotas[index] += 1;
|
||||
remainder -= 1;
|
||||
}
|
||||
|
||||
const sampledPerFile = await Promise.all(
|
||||
fileInfos.map((file, index) => samplePointcloudFile(file.filePath, file.size, quotas[index])),
|
||||
);
|
||||
const pointCount = sampledPerFile.reduce((sum, points) => sum + points.length, 0);
|
||||
const hasError = sampledPerFile.some((points) => points.some((point) => Number.isFinite(point.error)));
|
||||
const hasColor = sampledPerFile.some((points) => points.some((point) => point.r !== undefined));
|
||||
const bytesPerPoint = hasColor ? 28 : 16;
|
||||
const buffer = Buffer.allocUnsafe(pointCount * bytesPerPoint);
|
||||
let offset = 0;
|
||||
|
||||
for (const points of sampledPerFile) {
|
||||
for (const point of points) {
|
||||
buffer.writeFloatLE(point.x, offset);
|
||||
buffer.writeFloatLE(point.y, offset + 4);
|
||||
buffer.writeFloatLE(point.z, offset + 8);
|
||||
buffer.writeFloatLE(point.error ?? Number.NaN, offset + 12);
|
||||
if (hasColor) {
|
||||
buffer.writeFloatLE((point.r ?? 128) / 255, offset + 16);
|
||||
buffer.writeFloatLE((point.g ?? 128) / 255, offset + 20);
|
||||
buffer.writeFloatLE((point.b ?? 128) / 255, offset + 24);
|
||||
}
|
||||
offset += bytesPerPoint;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
buffer,
|
||||
pointCount,
|
||||
hasColor,
|
||||
hasError,
|
||||
fileStats: fileInfos.map((file, index) => ({
|
||||
name: file.name,
|
||||
sizeGB: +(file.size / 1e9).toFixed(3),
|
||||
sampled: sampledPerFile[index].length,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user