get_data.py

"""Download and build every benchmark mesh, then check it against the pinned hashes.

    python scripts/get_data.py                 # fetch what is missing (resumable), then verify
    python scripts/get_data.py --verify-only
    python scripts/get_data.py --write-pins    # maintainers: record the current corpus as the pins

Groups: STAN (Stanford 3D Scanning Repository), AMD (crab, XYZ RGB dragon, Happy Buddha: AMD's published DGF
sizes), T100 (Thingi10K, 100 largest after dedup; Hugging Face revision pinned in bench/meshes.py), TDS (Three D
Scans, 100 largest). Every mesh lands in assets/corpus/<group>/<name>.smesh with a manifest row (source URL,
licence, citation key). A mesh whose cleaned geometry hashes differently from bench/corpus_pins.csv is reported."""
from __future__ import annotations

import argparse
import csv
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

PINS = ROOT / "bench" / "corpus_pins.csv"
PIN_COLS = ("group", "name", "mesh_sha", "n_v", "n_t", "url")
BUILDERS = (("STAN", "scripts.corpus_stanford"), ("AMD", "scripts.corpus_amd"), ("T100", "scripts.corpus_thingi"),
            ("TDS", "scripts.corpus_threedscans"))


def current() -> list:
    from bench import meshes
    rows = []
    for g in meshes.GROUPS:
        for r in meshes._manifest(g):
            if r["status"] == "ok":
                rows.append({c: r[c] for c in PIN_COLS})
    return rows


def verify() -> list:
    """Problems: pinned meshes missing or changed, and ok meshes that are not pinned."""
    with open(PINS, newline="", encoding="utf-8") as f:
        pins = {(r["group"], r["name"]): r for r in csv.DictReader(f)}
    have = {(r["group"], r["name"]): r for r in current()}
    out = [f"missing {g}/{n}" for (g, n) in pins if (g, n) not in have]
    out += [f"changed {g}/{n}: mesh_sha {have[(g, n)]['mesh_sha']} != pinned {p['mesh_sha']}"
            for (g, n), p in pins.items() if (g, n) in have and have[(g, n)]["mesh_sha"] != p["mesh_sha"]]
    out += [f"not pinned {g}/{n}" for (g, n) in have if (g, n) not in pins]
    return out


def main(argv=None) -> int:
    import importlib
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--verify-only", action="store_true")
    ap.add_argument("--write-pins", action="store_true")
    a = ap.parse_args(argv)
    if a.write_pins:
        with open(PINS, "w", newline="", encoding="utf-8") as f:
            w = csv.DictWriter(f, PIN_COLS, lineterminator="\n")
            w.writeheader()
            w.writerows(sorted(current(), key=lambda r: (r["group"], r["name"])))
        print(f"wrote {PINS}")
        return 0
    if not a.verify_only:
        for g, module in BUILDERS:
            print(f"== {g}", flush=True)
            try:
                importlib.import_module(module).main([])
            except SystemExit as e:
                if e.code:
                    raise
    problems = verify()
    for p in problems:
        print(p)
    print("corpus OK" if not problems else f"{len(problems)} problems")
    return 1 if problems else 0


if __name__ == "__main__":
    sys.exit(main())