#!/usr/bin/env python3 """Create a reproducible source-only ZIP and its SHA-1/manifest files.""" from __future__ import annotations import argparse import hashlib import json import os import stat import subprocess import sys import zipfile from collections import Counter from pathlib import Path, PurePosixPath EXCLUDED_SUFFIXES = { # Point clouds, CAD and meshes ".ply", ".pcd", ".las", ".laz", ".e57", ".xyz", ".pts", ".stp", ".step", ".iges", ".igs", ".obj", ".stl", ".glb", ".gltf", # Databases and generated tabular results ".db", ".sqlite", ".sqlite3", ".mdb", ".accdb", ".xlsx", ".xls", # Logs, caches, backups and generated archives ".log", ".pyc", ".pyo", ".bak", ".tmp", ".zip", ".7z", ".rar", ".tar", ".gz", ".bz2", ".xz", # Model weights / binary artifacts ".pt", ".pth", ".onnx", ".ckpt", ".safetensors", ".bin", # Private-key/container formats ".pem", ".key", ".p12", ".pfx", ".jks", } EXCLUDED_DIRS = { ".git", ".manus-logs", ".pnpm-store", "node_modules", "dist", "build", "coverage", "results", "result", "outputs", "output", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".cache", "data", "datasets", } EXCLUDED_NAMES = {".env", "credentials.json", "secrets.json", "=1.10", "=3.1"} FIXED_ZIP_TIME = (1980, 1, 1, 0, 0, 0) ZIP_ROOT = "massive-cloud-v3-source" def git_output(root: Path, *args: str) -> bytes: result = subprocess.run( ["git", "-C", str(root), *args], check=True, stdout=subprocess.PIPE ) return result.stdout def candidates(root: Path) -> list[str]: raw = git_output(root, "ls-files", "--cached", "--others", "--exclude-standard", "-z") return sorted(os.fsdecode(item) for item in raw.split(b"\0") if item) def exclusion_reason(relative: str) -> str | None: path = PurePosixPath(relative) lowered_parts = tuple(part.lower() for part in path.parts) name = path.name.lower() if any(part in EXCLUDED_DIRS for part in lowered_parts[:-1]): return "excluded-directory" if name in EXCLUDED_NAMES: return "secret-or-artifact-name" if name.startswith(".env.") and name not in {".env.example", ".env.template", ".env.sample"}: return "environment-secret" if path.suffix.lower() in EXCLUDED_SUFFIXES: return "non-source-suffix" return None def sha1_file(path: Path) -> str: digest = hashlib.sha1() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def git_text(root: Path, *args: str) -> str: return git_output(root, *args).decode("utf-8", "replace").strip() def build_manifest(root: Path, included: list[str], excluded: list[dict[str, str]]) -> dict: return { "format": "massive-cloud-source-package-v1", "source_root": str(root), "git_branch": git_text(root, "branch", "--show-current"), "git_commit": git_text(root, "rev-parse", "HEAD"), "git_status": git_text(root, "status", "--short"), "included_count": len(included), "included_files": included, "excluded_count": len(excluded), "excluded_files": excluded, "rules": { "candidate_source": "git ls-files --cached --others --exclude-standard", "excluded_directories": sorted(EXCLUDED_DIRS), "excluded_names": sorted(EXCLUDED_NAMES), "excluded_suffixes": sorted(EXCLUDED_SUFFIXES), }, } def write_zip(root: Path, output: Path, included: list[str]) -> None: output.parent.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: for relative in included: source = root / relative if not source.is_file() or source.is_symlink(): continue info = zipfile.ZipInfo(f"{ZIP_ROOT}/{PurePosixPath(relative)}", FIXED_ZIP_TIME) mode = source.stat().st_mode normalized_mode = 0o755 if mode & stat.S_IXUSR else 0o644 info.create_system = 3 info.external_attr = (stat.S_IFREG | normalized_mode) << 16 info.compress_type = zipfile.ZIP_DEFLATED with source.open("rb") as handle: archive.writestr(info, handle.read(), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source", type=Path, required=True, help="Git working tree") parser.add_argument("--output", type=Path, required=True, help="Output .zip path") parser.add_argument("--dry-run", action="store_true", help="Print selection summary only") args = parser.parse_args() root = args.source.resolve() output = args.output.resolve() if output.suffix.lower() != ".zip": parser.error("--output must end with .zip") if not (root / ".git").exists(): parser.error(f"not a Git working tree: {root}") try: output.relative_to(root) except ValueError: pass else: parser.error("write the package outside the source tree") selected: list[str] = [] excluded: list[dict[str, str]] = [] for relative in candidates(root): reason = exclusion_reason(relative) if reason: excluded.append({"path": relative, "reason": reason}) elif (root / relative).is_file() and not (root / relative).is_symlink(): selected.append(relative) reasons = Counter(item["reason"] for item in excluded) print(f"source: {root}") print(f"included: {len(selected)} files") print(f"excluded: {len(excluded)} files ({dict(sorted(reasons.items()))})") print(f"git commit: {git_text(root, 'rev-parse', 'HEAD')}") status = git_text(root, "status", "--short") print("working tree: dirty" if status else "working tree: clean") if args.dry_run: return 0 manifest = build_manifest(root, selected, excluded) write_zip(root, output, selected) digest = sha1_file(output) manifest.update({"zip_file": output.name, "zip_size": output.stat().st_size, "sha1": digest}) sha_path = output.with_suffix(output.suffix + ".sha1") manifest_path = output.with_suffix(output.suffix + ".manifest.json") sha_path.write_text(f"{digest} {output.name}\n", encoding="ascii") manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(f"zip: {output}") print(f"sha1: {digest}") print(f"manifest: {manifest_path}") return 0 if __name__ == "__main__": sys.exit(main())