Compare commits
3 Commits
29a69793db
...
ac1387debc
| Author | SHA1 | Date | |
|---|---|---|---|
| ac1387debc | |||
| d33b280a93 | |||
| 9f4482bea1 |
39
.agents/skills/package-source-sha1/SKILL.md
Normal file
39
.agents/skills/package-source-sha1/SKILL.md
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
---
|
||||||
|
name: package-source-sha1
|
||||||
|
description: 为 massive-cloud-v3 生成可复现的纯源码 ZIP 包、内容清单和 40 位 SHA-1。用户要求打包整个项目、交付源码包、排除点云/CAD/数据库/构建产物/密钥,或计算项目 ZIP 哈希时使用。
|
||||||
|
---
|
||||||
|
|
||||||
|
# 打包源码并计算 SHA-1
|
||||||
|
|
||||||
|
使用 `scripts/package_source.py`,不要临时拼装 `find`、`tar` 或 `zip` 命令。
|
||||||
|
|
||||||
|
## 工作流
|
||||||
|
|
||||||
|
1. 根据项目 `AGENTS.md` 查看开发服务器 `/home/lipan/massive-cloud-v3` 的实时工作树和 `git status`。
|
||||||
|
2. 先运行 `--dry-run` 检查纳入、排除数量和排除原因;发现异常大文件或新的数据格式时,先更新脚本规则。
|
||||||
|
3. 运行正式打包。脚本只从 `git ls-files --cached --others --exclude-standard` 取候选文件,因此包含已跟踪文件和未忽略的新文件,也包含尚未提交的实际代码改动。
|
||||||
|
4. 将 ZIP、`.sha1` 和 `.manifest.json` 一起交付。明确说明包是否包含未提交改动,并报告基线提交。
|
||||||
|
5. 用 `unzip -t` 验证 ZIP;再独立运行 `sha1sum` 或 `shasum -a 1`,确认与 `.sha1` 一致且为 40 位小写十六进制。
|
||||||
|
|
||||||
|
## 固定规则
|
||||||
|
|
||||||
|
- 保留源码、测试、SQL、配置模板、依赖清单/锁文件、容器配置、脚本和文档。
|
||||||
|
- 排除 Git 历史、被 `.gitignore` 忽略的内容、真实 `.env`、密钥、日志、缓存、依赖目录和构建产物。
|
||||||
|
- 排除点云、CAD/网格、数据库、表格结果、归档包、模型权重等非源码文件。
|
||||||
|
- 不要使用 `git archive`:它会遗漏工作区中尚未提交的实际代码。
|
||||||
|
- ZIP 使用固定时间戳、固定顺序和规范权限;相同文件内容应生成相同哈希。
|
||||||
|
|
||||||
|
## 命令
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/package_source.py \
|
||||||
|
--source /home/lipan/massive-cloud-v3 \
|
||||||
|
--output /tmp/massive-cloud-v3-source.zip \
|
||||||
|
--dry-run
|
||||||
|
|
||||||
|
python3 scripts/package_source.py \
|
||||||
|
--source /home/lipan/massive-cloud-v3 \
|
||||||
|
--output /tmp/massive-cloud-v3-source.zip
|
||||||
|
```
|
||||||
|
|
||||||
|
不要把产物写进被打包的源码目录;优先使用 `/tmp`,再复制到交付位置。
|
||||||
4
.agents/skills/package-source-sha1/agents/openai.yaml
Normal file
4
.agents/skills/package-source-sha1/agents/openai.yaml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
interface:
|
||||||
|
display_name: "源码包 SHA-1"
|
||||||
|
short_description: "可复现地打包项目纯源码并自动生成 SHA-1 校验值"
|
||||||
|
default_prompt: "使用 $package-source-sha1 打包当前项目纯源码并返回 ZIP 的 SHA-1。"
|
||||||
176
.agents/skills/package-source-sha1/scripts/package_source.py
Normal file
176
.agents/skills/package-source-sha1/scripts/package_source.py
Normal file
@@ -0,0 +1,176 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Create a reproducible source-only ZIP and its SHA-1/manifest files."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
|
||||||
|
|
||||||
|
EXCLUDED_SUFFIXES = {
|
||||||
|
# Point clouds, CAD and meshes
|
||||||
|
".ply", ".pcd", ".las", ".laz", ".e57", ".xyz", ".pts",
|
||||||
|
".stp", ".step", ".iges", ".igs", ".obj", ".stl", ".glb", ".gltf",
|
||||||
|
# Databases and generated tabular results
|
||||||
|
".db", ".sqlite", ".sqlite3", ".mdb", ".accdb", ".xlsx", ".xls",
|
||||||
|
# Logs, caches, backups and generated archives
|
||||||
|
".log", ".pyc", ".pyo", ".bak", ".tmp", ".zip", ".7z", ".rar",
|
||||||
|
".tar", ".gz", ".bz2", ".xz",
|
||||||
|
# Model weights / binary artifacts
|
||||||
|
".pt", ".pth", ".onnx", ".ckpt", ".safetensors", ".bin",
|
||||||
|
# Private-key/container formats
|
||||||
|
".pem", ".key", ".p12", ".pfx", ".jks",
|
||||||
|
}
|
||||||
|
|
||||||
|
EXCLUDED_DIRS = {
|
||||||
|
".git", ".manus-logs", ".pnpm-store", "node_modules", "dist", "build",
|
||||||
|
"coverage", "results", "result", "outputs", "output", "__pycache__",
|
||||||
|
".pytest_cache", ".mypy_cache", ".ruff_cache", ".cache", "data", "datasets",
|
||||||
|
}
|
||||||
|
|
||||||
|
EXCLUDED_NAMES = {".env", "credentials.json", "secrets.json", "=1.10", "=3.1"}
|
||||||
|
FIXED_ZIP_TIME = (1980, 1, 1, 0, 0, 0)
|
||||||
|
ZIP_ROOT = "massive-cloud-v3-source"
|
||||||
|
|
||||||
|
|
||||||
|
def git_output(root: Path, *args: str) -> bytes:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), *args], check=True, stdout=subprocess.PIPE
|
||||||
|
)
|
||||||
|
return result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def candidates(root: Path) -> list[str]:
|
||||||
|
raw = git_output(root, "ls-files", "--cached", "--others", "--exclude-standard", "-z")
|
||||||
|
return sorted(os.fsdecode(item) for item in raw.split(b"\0") if item)
|
||||||
|
|
||||||
|
|
||||||
|
def exclusion_reason(relative: str) -> str | None:
|
||||||
|
path = PurePosixPath(relative)
|
||||||
|
lowered_parts = tuple(part.lower() for part in path.parts)
|
||||||
|
name = path.name.lower()
|
||||||
|
|
||||||
|
if any(part in EXCLUDED_DIRS for part in lowered_parts[:-1]):
|
||||||
|
return "excluded-directory"
|
||||||
|
if name in EXCLUDED_NAMES:
|
||||||
|
return "secret-or-artifact-name"
|
||||||
|
if name.startswith(".env.") and name not in {".env.example", ".env.template", ".env.sample"}:
|
||||||
|
return "environment-secret"
|
||||||
|
if path.suffix.lower() in EXCLUDED_SUFFIXES:
|
||||||
|
return "non-source-suffix"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def sha1_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha1()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def git_text(root: Path, *args: str) -> str:
|
||||||
|
return git_output(root, *args).decode("utf-8", "replace").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def build_manifest(root: Path, included: list[str], excluded: list[dict[str, str]]) -> dict:
|
||||||
|
return {
|
||||||
|
"format": "massive-cloud-source-package-v1",
|
||||||
|
"source_root": str(root),
|
||||||
|
"git_branch": git_text(root, "branch", "--show-current"),
|
||||||
|
"git_commit": git_text(root, "rev-parse", "HEAD"),
|
||||||
|
"git_status": git_text(root, "status", "--short"),
|
||||||
|
"included_count": len(included),
|
||||||
|
"included_files": included,
|
||||||
|
"excluded_count": len(excluded),
|
||||||
|
"excluded_files": excluded,
|
||||||
|
"rules": {
|
||||||
|
"candidate_source": "git ls-files --cached --others --exclude-standard",
|
||||||
|
"excluded_directories": sorted(EXCLUDED_DIRS),
|
||||||
|
"excluded_names": sorted(EXCLUDED_NAMES),
|
||||||
|
"excluded_suffixes": sorted(EXCLUDED_SUFFIXES),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_zip(root: Path, output: Path, included: list[str]) -> None:
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive:
|
||||||
|
for relative in included:
|
||||||
|
source = root / relative
|
||||||
|
if not source.is_file() or source.is_symlink():
|
||||||
|
continue
|
||||||
|
info = zipfile.ZipInfo(f"{ZIP_ROOT}/{PurePosixPath(relative)}", FIXED_ZIP_TIME)
|
||||||
|
mode = source.stat().st_mode
|
||||||
|
normalized_mode = 0o755 if mode & stat.S_IXUSR else 0o644
|
||||||
|
info.create_system = 3
|
||||||
|
info.external_attr = (stat.S_IFREG | normalized_mode) << 16
|
||||||
|
info.compress_type = zipfile.ZIP_DEFLATED
|
||||||
|
with source.open("rb") as handle:
|
||||||
|
archive.writestr(info, handle.read(), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--source", type=Path, required=True, help="Git working tree")
|
||||||
|
parser.add_argument("--output", type=Path, required=True, help="Output .zip path")
|
||||||
|
parser.add_argument("--dry-run", action="store_true", help="Print selection summary only")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
root = args.source.resolve()
|
||||||
|
output = args.output.resolve()
|
||||||
|
if output.suffix.lower() != ".zip":
|
||||||
|
parser.error("--output must end with .zip")
|
||||||
|
if not (root / ".git").exists():
|
||||||
|
parser.error(f"not a Git working tree: {root}")
|
||||||
|
try:
|
||||||
|
output.relative_to(root)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
parser.error("write the package outside the source tree")
|
||||||
|
|
||||||
|
selected: list[str] = []
|
||||||
|
excluded: list[dict[str, str]] = []
|
||||||
|
for relative in candidates(root):
|
||||||
|
reason = exclusion_reason(relative)
|
||||||
|
if reason:
|
||||||
|
excluded.append({"path": relative, "reason": reason})
|
||||||
|
elif (root / relative).is_file() and not (root / relative).is_symlink():
|
||||||
|
selected.append(relative)
|
||||||
|
|
||||||
|
reasons = Counter(item["reason"] for item in excluded)
|
||||||
|
print(f"source: {root}")
|
||||||
|
print(f"included: {len(selected)} files")
|
||||||
|
print(f"excluded: {len(excluded)} files ({dict(sorted(reasons.items()))})")
|
||||||
|
print(f"git commit: {git_text(root, 'rev-parse', 'HEAD')}")
|
||||||
|
status = git_text(root, "status", "--short")
|
||||||
|
print("working tree: dirty" if status else "working tree: clean")
|
||||||
|
if args.dry_run:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
manifest = build_manifest(root, selected, excluded)
|
||||||
|
write_zip(root, output, selected)
|
||||||
|
digest = sha1_file(output)
|
||||||
|
manifest.update({"zip_file": output.name, "zip_size": output.stat().st_size, "sha1": digest})
|
||||||
|
|
||||||
|
sha_path = output.with_suffix(output.suffix + ".sha1")
|
||||||
|
manifest_path = output.with_suffix(output.suffix + ".manifest.json")
|
||||||
|
sha_path.write_text(f"{digest} {output.name}\n", encoding="ascii")
|
||||||
|
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||||
|
print(f"zip: {output}")
|
||||||
|
print(f"sha1: {digest}")
|
||||||
|
print(f"manifest: {manifest_path}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -1890,21 +1890,32 @@ export default function PointCloudViewer({
|
|||||||
setPlyOverlayStatus({ status: 'loading', message: '正在加载 PLY 叠加层' });
|
setPlyOverlayStatus({ status: 'loading', message: '正在加载 PLY 叠加层' });
|
||||||
(async () => {
|
(async () => {
|
||||||
try {
|
try {
|
||||||
// 用 /sample/batch 接口按 filePath 采样,最多 30 万点
|
// 通过受限的分析结果接口加载相对 PLY 路径,支持多文件合并采样
|
||||||
// 支持多文件(各面板 PLY),合并采样
|
|
||||||
const plyFiles: string[] = Array.isArray(overlay.files) && overlay.files.length > 0
|
const plyFiles: string[] = Array.isArray(overlay.files) && overlay.files.length > 0
|
||||||
? overlay.files
|
? overlay.files
|
||||||
: [overlay.file];
|
: [overlay.file];
|
||||||
const resp = await fetch('/api/files/sample/batch', {
|
const resp = await fetch('/api/files/sample/analysis', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
signal: ac.signal,
|
signal: ac.signal,
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
filePaths: plyFiles.map(f => `/data/tasks/${taskId}/analysis/${resultId}/${f}`),
|
taskId,
|
||||||
|
resultId,
|
||||||
|
files: plyFiles,
|
||||||
totalPoints: 300_000,
|
totalPoints: 300_000,
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
if (!resp.ok) {
|
||||||
|
const payload = await resp.json().catch(() => null) as { error?: string } | null;
|
||||||
|
const detail = payload?.error;
|
||||||
|
if (resp.status === 404) {
|
||||||
|
throw new Error(detail ?? '分析结果或 PLY 文件不存在');
|
||||||
|
}
|
||||||
|
if (resp.status === 400 || resp.status === 403) {
|
||||||
|
throw new Error(detail ?? 'PLY 文件参数或访问权限无效');
|
||||||
|
}
|
||||||
|
throw new Error(detail ?? `服务端采样失败(HTTP ${resp.status})`);
|
||||||
|
}
|
||||||
const hasColor = resp.headers.get('X-Has-Color') === '1';
|
const hasColor = resp.headers.get('X-Has-Color') === '1';
|
||||||
const stride = hasColor ? 7 : 4;
|
const stride = hasColor ? 7 : 4;
|
||||||
const buf = await resp.arrayBuffer();
|
const buf = await resp.arrayBuffer();
|
||||||
@@ -1950,7 +1961,10 @@ export default function PointCloudViewer({
|
|||||||
if (e instanceof DOMException && e.name === 'AbortError') return;
|
if (e instanceof DOMException && e.name === 'AbortError') return;
|
||||||
console.warn('[PlyOverlay] load failed:', e);
|
console.warn('[PlyOverlay] load failed:', e);
|
||||||
setPlyOverlayData(null);
|
setPlyOverlayData(null);
|
||||||
setPlyOverlayStatus({ status: 'error', message: '加载 PLY 叠加层失败' });
|
setPlyOverlayStatus({
|
||||||
|
status: 'error',
|
||||||
|
message: e instanceof Error ? e.message : '服务端采样失败,请稍后重试',
|
||||||
|
});
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
return () => ac.abort();
|
return () => ac.abort();
|
||||||
|
|||||||
@@ -240,6 +240,19 @@ export default function TaskManager({
|
|||||||
const activeInspectionObject = inspectionObjects.find(
|
const activeInspectionObject = inspectionObjects.find(
|
||||||
object => object.id === activeTask?.inspectionObjectId
|
object => object.id === activeTask?.inspectionObjectId
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// 打开新建任务表单时默认选择首个可用检测对象;若列表异步刷新,
|
||||||
|
// 则仅在当前选择失效时回退到第一项,避免覆盖用户的手动选择。
|
||||||
|
useEffect(() => {
|
||||||
|
if (!adding) return;
|
||||||
|
setNewTaskInspectionObjectId(currentId => {
|
||||||
|
if (inspectionObjects.some(object => object.id === currentId)) {
|
||||||
|
return currentId;
|
||||||
|
}
|
||||||
|
return inspectionObjects[0]?.id ?? null;
|
||||||
|
});
|
||||||
|
}, [adding, inspectionObjects]);
|
||||||
|
|
||||||
const createTask = trpc.inspectionTask.create.useMutation({
|
const createTask = trpc.inspectionTask.create.useMutation({
|
||||||
onSuccess: result => {
|
onSuccess: result => {
|
||||||
pendingDefaultSelection.current = {
|
pendingDefaultSelection.current = {
|
||||||
@@ -692,6 +705,7 @@ export default function TaskManager({
|
|||||||
<button
|
<button
|
||||||
className="industrial-btn flex shrink-0 items-center gap-0.5 whitespace-nowrap px-2 py-0.5 text-[10px]"
|
className="industrial-btn flex shrink-0 items-center gap-0.5 whitespace-nowrap px-2 py-0.5 text-[10px]"
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
|
setNewTaskInspectionObjectId(inspectionObjects[0]?.id ?? null);
|
||||||
setAdding(true);
|
setAdding(true);
|
||||||
onStartCreate?.();
|
onStartCreate?.();
|
||||||
}}
|
}}
|
||||||
|
|||||||
126
server/analysisPlySample.test.ts
Normal file
126
server/analysisPlySample.test.ts
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
import express from 'express';
|
||||||
|
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import type { AddressInfo } from 'node:net';
|
||||||
|
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
const mocks = vi.hoisted(() => ({
|
||||||
|
authenticated: true,
|
||||||
|
resultTaskId: 7,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./_core/auth.js', () => ({
|
||||||
|
authenticateRequest: vi.fn(async () => mocks.authenticated
|
||||||
|
? { id: 1, role: 'user', mustChangePassword: 0 }
|
||||||
|
: null),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./db', () => ({
|
||||||
|
getTaskAnalysisResultById: vi.fn(async (id: number) => id === 11
|
||||||
|
? { id, taskId: mocks.resultTaskId }
|
||||||
|
: undefined),
|
||||||
|
listTaskPointcloudFiles: vi.fn(async () => []),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const dataDir = mkdtempSync(join(tmpdir(), 'analysis-ply-route-'));
|
||||||
|
process.env.DATA_DIR = dataDir;
|
||||||
|
|
||||||
|
function writePly(relativePath: string, withColor = false): void {
|
||||||
|
const resultDir = join(dataDir, 'tasks', '7', 'analysis', '11');
|
||||||
|
const filePath = join(resultDir, relativePath);
|
||||||
|
mkdirSync(join(filePath, '..'), { recursive: true });
|
||||||
|
const properties = withColor
|
||||||
|
? ['x', 'y', 'z', 'red', 'green', 'blue']
|
||||||
|
: ['x', 'y', 'z'];
|
||||||
|
const header = Buffer.from([
|
||||||
|
'ply',
|
||||||
|
'format binary_little_endian 1.0',
|
||||||
|
'element vertex 2',
|
||||||
|
...properties.map((name) => `property float ${name}`),
|
||||||
|
'end_header',
|
||||||
|
'',
|
||||||
|
].join('\n'), 'ascii');
|
||||||
|
const values = withColor
|
||||||
|
? [[1, 2, 3, 255, 0, 128], [4, 5, 6, 0, 255, 64]]
|
||||||
|
: [[7, 8, 9], [10, 11, 12]];
|
||||||
|
const body = Buffer.alloc(values.length * properties.length * 4);
|
||||||
|
values.flat().forEach((value, index) => body.writeFloatLE(value, index * 4));
|
||||||
|
writeFileSync(filePath, Buffer.concat([header, body]));
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('POST /api/files/sample/analysis', () => {
|
||||||
|
let origin: string;
|
||||||
|
let closeServer: () => Promise<void>;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
writePly('plain.ply');
|
||||||
|
writePly('nested/color.ply', true);
|
||||||
|
const { fileRouter } = await import('./fileRoutes');
|
||||||
|
const app = express();
|
||||||
|
app.use(express.json());
|
||||||
|
app.use('/api/files', fileRouter);
|
||||||
|
const server = app.listen(0);
|
||||||
|
await new Promise<void>((resolve) => server.once('listening', resolve));
|
||||||
|
origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||||
|
closeServer = () => new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await closeServer();
|
||||||
|
rmSync(dataDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
mocks.authenticated = true;
|
||||||
|
mocks.resultTaskId = 7;
|
||||||
|
});
|
||||||
|
|
||||||
|
async function post(body: unknown) {
|
||||||
|
return fetch(`${origin}/api/files/sample/analysis`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
it('采样单个无色 PLY 并返回 16 字节步长', async () => {
|
||||||
|
const response = await post({ taskId: 7, resultId: 11, files: ['plain.ply'], totalPoints: 2 });
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(response.headers.get('X-Point-Count')).toBe('2');
|
||||||
|
expect(response.headers.get('X-Has-Color')).toBe('0');
|
||||||
|
expect((await response.arrayBuffer()).byteLength).toBe(2 * 16);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('采样多个 PLY,彩色响应使用 28 字节步长且不超预算', async () => {
|
||||||
|
const response = await post({
|
||||||
|
taskId: 7,
|
||||||
|
resultId: 11,
|
||||||
|
files: ['plain.ply', 'nested/color.ply'],
|
||||||
|
totalPoints: 3,
|
||||||
|
});
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(response.headers.get('X-Point-Count')).toBe('3');
|
||||||
|
expect(response.headers.get('X-Has-Color')).toBe('1');
|
||||||
|
expect((await response.arrayBuffer()).byteLength).toBe(3 * 28);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('拒绝未登录访问', async () => {
|
||||||
|
mocks.authenticated = false;
|
||||||
|
expect((await post({ taskId: 7, resultId: 11, files: ['plain.ply'] })).status).toBe(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('拒绝错误的任务与结果组合', async () => {
|
||||||
|
mocks.resultTaskId = 8;
|
||||||
|
expect((await post({ taskId: 7, resultId: 11, files: ['plain.ply'] })).status).toBe(404);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
['路径穿越', '../plain.ply', 403],
|
||||||
|
['绝对路径', '/tmp/plain.ply', 400],
|
||||||
|
['非 PLY 文件', 'result.json', 400],
|
||||||
|
['缺失文件', 'missing.ply', 404],
|
||||||
|
])('拒绝%s', async (_label, file, status) => {
|
||||||
|
expect((await post({ taskId: 7, resultId: 11, files: [file] })).status).toBe(status);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -3,7 +3,7 @@ import * as fs from 'fs';
|
|||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import multer from 'multer';
|
import multer from 'multer';
|
||||||
import * as db from './db';
|
import * as db from './db';
|
||||||
import { samplePointcloudFile, type SampledPoint } from './pointcloudSample.js';
|
import { samplePointcloudFiles } from './pointcloudSample.js';
|
||||||
import { forceReload as reloadPluginRegistry } from './plugins/registry.js';
|
import { forceReload as reloadPluginRegistry } from './plugins/registry.js';
|
||||||
import { authenticateRequest } from './_core/auth.js';
|
import { authenticateRequest } from './_core/auth.js';
|
||||||
import { inspectPointcloudMetadata } from './pointcloudMetadata.js';
|
import { inspectPointcloudMetadata } from './pointcloudMetadata.js';
|
||||||
@@ -88,7 +88,19 @@ function formatFileSize(bytes: number): string {
|
|||||||
|
|
||||||
function safeResolve(base: string, filename: string): string | null {
|
function safeResolve(base: string, filename: string): string | null {
|
||||||
const resolved = path.resolve(base, decodeURIComponent(filename));
|
const resolved = path.resolve(base, decodeURIComponent(filename));
|
||||||
return resolved.startsWith(path.resolve(base)) ? resolved : null;
|
const relative = path.relative(path.resolve(base), resolved);
|
||||||
|
return relative !== '' && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)
|
||||||
|
? resolved
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendSampledPointcloudBatch(res: Parameters<RequestHandler>[1], batch: Awaited<ReturnType<typeof samplePointcloudFiles>>) {
|
||||||
|
res.setHeader('Content-Type', 'application/octet-stream');
|
||||||
|
res.setHeader('X-Point-Count', batch.pointCount);
|
||||||
|
res.setHeader('X-Has-Color', batch.hasColor ? '1' : '0');
|
||||||
|
res.setHeader('X-Has-Error', batch.hasError ? '1' : '0');
|
||||||
|
res.setHeader('X-File-Stats', JSON.stringify(batch.fileStats));
|
||||||
|
res.send(batch.buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── 车型 CAD 文件上传 ────────────────────────────────────────────────────────
|
// ─── 车型 CAD 文件上传 ────────────────────────────────────────────────────────
|
||||||
@@ -333,7 +345,6 @@ router.post('/sample/batch', requireFileRead, async (req, res) => {
|
|||||||
return res.send(empty);
|
return res.send(empty);
|
||||||
}
|
}
|
||||||
|
|
||||||
const budget = Math.min(totalPoints, 2000000);
|
|
||||||
const supportedExts = ['.ply', '.txt', '.xyz', '.pts'];
|
const supportedExts = ['.ply', '.txt', '.xyz', '.pts'];
|
||||||
|
|
||||||
const fileInfos = paths.map((fp) => ({
|
const fileInfos = paths.map((fp) => ({
|
||||||
@@ -348,66 +359,100 @@ router.post('/sample/batch', requireFileRead, async (req, res) => {
|
|||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`[FileRoutes] Batch sample: ${fileInfos.length} files ` +
|
`[FileRoutes] Batch sample: ${fileInfos.length} files ` +
|
||||||
`(${(totalBytes / 1e9).toFixed(2)} GB), budget=${budget}`
|
`(${(totalBytes / 1e9).toFixed(2)} GB), budget=${totalPoints}`
|
||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const quotaMap = new Map<string, number>();
|
const batch = await samplePointcloudFiles(supportedFiles.map((file) => file.filePath), totalPoints);
|
||||||
for (const f of supportedFiles) {
|
console.log(`[FileRoutes] Batch done: ${batch.pointCount} pts, ${(batch.buffer.length / 1e6).toFixed(1)} MB`);
|
||||||
const quota = totalBytes > 0
|
sendSampledPointcloudBatch(res, batch);
|
||||||
? Math.max(100, Math.round((f.size / totalBytes) * budget))
|
|
||||||
: Math.floor(budget / supportedFiles.length);
|
|
||||||
quotaMap.set(f.filePath, quota);
|
|
||||||
}
|
|
||||||
|
|
||||||
const sampledPerFile = await Promise.all(
|
|
||||||
fileInfos.map((f) => {
|
|
||||||
if (!supportedExts.includes(f.ext) || f.size === 0) return Promise.resolve([] as SampledPoint[]);
|
|
||||||
return samplePointcloudFile(f.filePath, f.size, quotaMap.get(f.filePath) ?? 0);
|
|
||||||
})
|
|
||||||
);
|
|
||||||
|
|
||||||
const totalSampled = sampledPerFile.reduce((s, pts) => s + pts.length, 0);
|
|
||||||
// 如果任意点有颜色,则输出 28 字节/点(x,y,z,error,r,g,b 各 float32);否则 16 字节
|
|
||||||
const hasError = sampledPerFile.some((pts) => pts.some((p) => Number.isFinite(p.error)));
|
|
||||||
const hasColor = sampledPerFile.some((pts) => pts.some((p) => p.r !== undefined));
|
|
||||||
const bytesPerPoint = hasColor ? 28 : 16;
|
|
||||||
const binBuf = Buffer.allocUnsafe(totalSampled * bytesPerPoint);
|
|
||||||
let offset = 0;
|
|
||||||
for (const pts of sampledPerFile) {
|
|
||||||
for (const p of pts) {
|
|
||||||
binBuf.writeFloatLE(p.x, offset);
|
|
||||||
binBuf.writeFloatLE(p.y, offset + 4);
|
|
||||||
binBuf.writeFloatLE(p.z, offset + 8);
|
|
||||||
binBuf.writeFloatLE(p.error ?? Number.NaN, offset + 12);
|
|
||||||
if (hasColor) {
|
|
||||||
binBuf.writeFloatLE((p.r ?? 128) / 255, offset + 16);
|
|
||||||
binBuf.writeFloatLE((p.g ?? 128) / 255, offset + 20);
|
|
||||||
binBuf.writeFloatLE((p.b ?? 128) / 255, offset + 24);
|
|
||||||
}
|
|
||||||
offset += bytesPerPoint;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const fileStats = fileInfos.map((f, i) => ({
|
|
||||||
name: f.name,
|
|
||||||
sizeGB: +(f.size / 1e9).toFixed(3),
|
|
||||||
sampled: sampledPerFile[i].length,
|
|
||||||
}));
|
|
||||||
|
|
||||||
console.log(`[FileRoutes] Batch done: ${totalSampled} pts, ${(binBuf.length / 1e6).toFixed(1)} MB`);
|
|
||||||
res.setHeader('Content-Type', 'application/octet-stream');
|
|
||||||
res.setHeader('X-Point-Count', totalSampled);
|
|
||||||
res.setHeader('X-Has-Color', hasColor ? '1' : '0');
|
|
||||||
res.setHeader('X-Has-Error', hasError ? '1' : '0');
|
|
||||||
res.setHeader('X-File-Stats', JSON.stringify(fileStats));
|
|
||||||
res.send(binBuf);
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('[FileRoutes] Batch sampling error:', err);
|
console.error('[FileRoutes] Batch sampling error:', err);
|
||||||
res.status(500).json({ error: 'Failed to sample files' });
|
res.status(500).json({ error: 'Failed to sample files' });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ─── 分析结果 PLY 批量采样接口 ───────────────────────────────────────────────
|
||||||
|
// POST /api/files/sample/analysis
|
||||||
|
// Body: { taskId: number, resultId: number, files: string[], totalPoints?: number }
|
||||||
|
|
||||||
|
router.post('/sample/analysis', requireFileRead, async (req, res) => {
|
||||||
|
const { taskId, resultId, files, totalPoints = 300_000 } = req.body as {
|
||||||
|
taskId?: number;
|
||||||
|
resultId?: number;
|
||||||
|
files?: unknown;
|
||||||
|
totalPoints?: number;
|
||||||
|
};
|
||||||
|
if (!Number.isSafeInteger(taskId) || Number(taskId) <= 0 ||
|
||||||
|
!Number.isSafeInteger(resultId) || Number(resultId) <= 0 ||
|
||||||
|
!Number.isSafeInteger(totalPoints) || totalPoints <= 0) {
|
||||||
|
return res.status(400).json({ error: '分析结果采样参数无效' });
|
||||||
|
}
|
||||||
|
if (!Array.isArray(files) || files.length === 0 ||
|
||||||
|
files.some((file) => typeof file !== 'string' || file.length === 0)) {
|
||||||
|
return res.status(400).json({ error: '请提供 PLY 文件列表' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const validTaskId = taskId as number;
|
||||||
|
const validResultId = resultId as number;
|
||||||
|
const validTotalPoints = totalPoints as number;
|
||||||
|
|
||||||
|
const row = await db.getTaskAnalysisResultById(validResultId);
|
||||||
|
if (!row || row.taskId !== validTaskId) {
|
||||||
|
return res.status(404).json({ error: '分析结果不存在或不属于当前任务' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseDir = path.join(TASKS_DIR, String(validTaskId), 'analysis', String(validResultId));
|
||||||
|
let realBaseDir: string;
|
||||||
|
try {
|
||||||
|
realBaseDir = fs.realpathSync(baseDir);
|
||||||
|
} catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||||
|
return res.status(404).json({ error: '分析结果目录不存在' });
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const resolvedFiles: string[] = [];
|
||||||
|
for (const file of files as string[]) {
|
||||||
|
if (path.isAbsolute(file) || path.extname(file).toLowerCase() !== '.ply') {
|
||||||
|
return res.status(400).json({ error: '只允许读取分析结果目录中的相对 PLY 路径' });
|
||||||
|
}
|
||||||
|
let filePath: string | null;
|
||||||
|
try {
|
||||||
|
filePath = safeResolve(baseDir, file);
|
||||||
|
} catch {
|
||||||
|
return res.status(400).json({ error: 'PLY 文件路径编码无效' });
|
||||||
|
}
|
||||||
|
if (!filePath) return res.status(403).json({ error: 'PLY 文件路径越界' });
|
||||||
|
let stat: fs.Stats;
|
||||||
|
try {
|
||||||
|
stat = fs.lstatSync(filePath);
|
||||||
|
} catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||||
|
return res.status(404).json({ error: `PLY 文件不存在: ${file}` });
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
||||||
|
return res.status(400).json({ error: `PLY 路径不是普通文件: ${file}` });
|
||||||
|
}
|
||||||
|
const realFilePath = fs.realpathSync(filePath);
|
||||||
|
const realRelative = path.relative(realBaseDir, realFilePath);
|
||||||
|
if (realRelative === '' || realRelative === '..' || realRelative.startsWith(`..${path.sep}`) || path.isAbsolute(realRelative)) {
|
||||||
|
return res.status(403).json({ error: 'PLY 文件路径越界' });
|
||||||
|
}
|
||||||
|
resolvedFiles.push(realFilePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const batch = await samplePointcloudFiles(resolvedFiles, validTotalPoints, 300_000);
|
||||||
|
sendSampledPointcloudBatch(res, batch);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[FileRoutes] Analysis PLY sampling error:', error);
|
||||||
|
return res.status(500).json({ error: '分析结果 PLY 采样失败' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// ─── 平滑热力分析结果(异步任务产物,仅读取已落盘的顶点色)──────────────────────
|
// ─── 平滑热力分析结果(异步任务产物,仅读取已落盘的顶点色)──────────────────────
|
||||||
// GET /api/files/read/analysis/heatmap/:taskId/:resultId
|
// GET /api/files/read/analysis/heatmap/:taskId/:resultId
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|||||||
import { tmpdir } from 'node:os';
|
import { tmpdir } from 'node:os';
|
||||||
import { join } from 'node:path';
|
import { join } from 'node:path';
|
||||||
import { afterEach, describe, expect, it } from 'vitest';
|
import { afterEach, describe, expect, it } from 'vitest';
|
||||||
import { samplePointcloudFile } from './pointcloudSample';
|
import { samplePointcloudFile, samplePointcloudFiles } from './pointcloudSample';
|
||||||
|
|
||||||
const temporaryDirectories: string[] = [];
|
const temporaryDirectories: string[] = [];
|
||||||
|
|
||||||
@@ -58,3 +58,32 @@ describe('samplePointcloudFile', () => {
|
|||||||
expect(points.map((point) => point.error)).toEqual([0.25, -0.5]);
|
expect(points.map((point) => point.error)).toEqual([0.25, -0.5]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('samplePointcloudFiles', () => {
|
||||||
|
it('合并彩色与无色 PLY,并使用 28 字节彩色步长', async () => {
|
||||||
|
const plain = writeBinaryPly(['x', 'y', 'z'], [[1, 2, 3], [4, 5, 6]]);
|
||||||
|
const colored = writeBinaryPly(
|
||||||
|
['x', 'y', 'z', 'red', 'green', 'blue'],
|
||||||
|
[[7, 8, 9, 255, 128, 0], [10, 11, 12, 0, 64, 255]],
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await samplePointcloudFiles([plain, colored], 4, 10);
|
||||||
|
|
||||||
|
expect(result.pointCount).toBe(4);
|
||||||
|
expect(result.hasColor).toBe(true);
|
||||||
|
expect(result.hasError).toBe(false);
|
||||||
|
expect(result.buffer.byteLength).toBe(4 * 28);
|
||||||
|
expect(result.fileStats.map((file) => file.sampled)).toEqual([2, 2]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('限制总采样数量,无色 PLY 使用 16 字节步长', async () => {
|
||||||
|
const first = writeBinaryPly(['x', 'y', 'z'], Array.from({ length: 10 }, (_, i) => [i, i, i]));
|
||||||
|
const second = writeBinaryPly(['x', 'y', 'z'], Array.from({ length: 10 }, (_, i) => [i + 10, i, i]));
|
||||||
|
|
||||||
|
const result = await samplePointcloudFiles([first, second], 100, 6);
|
||||||
|
|
||||||
|
expect(result.pointCount).toBe(6);
|
||||||
|
expect(result.hasColor).toBe(false);
|
||||||
|
expect(result.buffer.byteLength).toBe(6 * 16);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -3,6 +3,14 @@ import * as path from 'path';
|
|||||||
|
|
||||||
export type SampledPoint = { x: number; y: number; z: number; error?: number; r?: number; g?: number; b?: number };
|
export type SampledPoint = { x: number; y: number; z: number; error?: number; r?: number; g?: number; b?: number };
|
||||||
|
|
||||||
|
export type SampledPointcloudBatch = {
|
||||||
|
buffer: Buffer;
|
||||||
|
pointCount: number;
|
||||||
|
hasColor: boolean;
|
||||||
|
hasError: boolean;
|
||||||
|
fileStats: Array<{ name: string; sizeGB: number; sampled: number }>;
|
||||||
|
};
|
||||||
|
|
||||||
/** 与 fileRoutes 批量采样一致:随机 seek 文本点云行 */
|
/** 与 fileRoutes 批量采样一致:随机 seek 文本点云行 */
|
||||||
export async function randomSeekSample(
|
export async function randomSeekSample(
|
||||||
filePath: string,
|
filePath: string,
|
||||||
@@ -356,3 +364,70 @@ export async function samplePointcloudFile(
|
|||||||
// .txt / .xyz / .pts / etc. — existing text line sampler
|
// .txt / .xyz / .pts / etc. — existing text line sampler
|
||||||
return randomSeekSample(filePath, fileSize, k);
|
return randomSeekSample(filePath, fileSize, k);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 对一组已校验的点云文件按文件大小分配总采样配额,并编码为浏览器消费的二进制协议。
|
||||||
|
* 每点固定包含 x/y/z/error;任一文件带颜色时,所有点再包含归一化后的 r/g/b。
|
||||||
|
*/
|
||||||
|
export async function samplePointcloudFiles(
|
||||||
|
filePaths: string[],
|
||||||
|
requestedTotalPoints: number,
|
||||||
|
maxTotalPoints = 2_000_000,
|
||||||
|
): Promise<SampledPointcloudBatch> {
|
||||||
|
const totalPoints = Number.isFinite(requestedTotalPoints)
|
||||||
|
? Math.max(0, Math.min(Math.floor(requestedTotalPoints), maxTotalPoints))
|
||||||
|
: 0;
|
||||||
|
const fileInfos = filePaths.map((filePath) => ({
|
||||||
|
filePath,
|
||||||
|
name: path.basename(filePath),
|
||||||
|
size: fs.statSync(filePath).size,
|
||||||
|
}));
|
||||||
|
const totalBytes = fileInfos.reduce((sum, file) => sum + file.size, 0);
|
||||||
|
|
||||||
|
// floor 后把剩余名额依次补给文件,保证所有配额之和严格不超过总预算。
|
||||||
|
const quotas = fileInfos.map((file) => totalBytes > 0
|
||||||
|
? Math.floor((file.size / totalBytes) * totalPoints)
|
||||||
|
: 0);
|
||||||
|
let remainder = totalPoints - quotas.reduce((sum, quota) => sum + quota, 0);
|
||||||
|
for (let index = 0; remainder > 0 && fileInfos.length > 0; index = (index + 1) % fileInfos.length) {
|
||||||
|
quotas[index] += 1;
|
||||||
|
remainder -= 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sampledPerFile = await Promise.all(
|
||||||
|
fileInfos.map((file, index) => samplePointcloudFile(file.filePath, file.size, quotas[index])),
|
||||||
|
);
|
||||||
|
const pointCount = sampledPerFile.reduce((sum, points) => sum + points.length, 0);
|
||||||
|
const hasError = sampledPerFile.some((points) => points.some((point) => Number.isFinite(point.error)));
|
||||||
|
const hasColor = sampledPerFile.some((points) => points.some((point) => point.r !== undefined));
|
||||||
|
const bytesPerPoint = hasColor ? 28 : 16;
|
||||||
|
const buffer = Buffer.allocUnsafe(pointCount * bytesPerPoint);
|
||||||
|
let offset = 0;
|
||||||
|
|
||||||
|
for (const points of sampledPerFile) {
|
||||||
|
for (const point of points) {
|
||||||
|
buffer.writeFloatLE(point.x, offset);
|
||||||
|
buffer.writeFloatLE(point.y, offset + 4);
|
||||||
|
buffer.writeFloatLE(point.z, offset + 8);
|
||||||
|
buffer.writeFloatLE(point.error ?? Number.NaN, offset + 12);
|
||||||
|
if (hasColor) {
|
||||||
|
buffer.writeFloatLE((point.r ?? 128) / 255, offset + 16);
|
||||||
|
buffer.writeFloatLE((point.g ?? 128) / 255, offset + 20);
|
||||||
|
buffer.writeFloatLE((point.b ?? 128) / 255, offset + 24);
|
||||||
|
}
|
||||||
|
offset += bytesPerPoint;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
buffer,
|
||||||
|
pointCount,
|
||||||
|
hasColor,
|
||||||
|
hasError,
|
||||||
|
fileStats: fileInfos.map((file, index) => ({
|
||||||
|
name: file.name,
|
||||||
|
sizeGB: +(file.size / 1e9).toFixed(3),
|
||||||
|
sampled: sampledPerFile[index].length,
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user