#!/usr/bin/env python3 """Measure the standalone against the oracle, and record the distance. The milestone is: the standalone loads a save, runs one strategic turn, and writes an autosave that byte-matches what the original produces from the same state. This tool measures how far off that is, in the only currency the campaign trusts -- named leaves of `verify/state-checksum/state_checksum.py`, whose coverage is proved by re-serialisation. For each (before, after) pair of real saves it computes three numbers: baseline leaves that differ between the INPUT save and the oracle's post-turn save. This is the distance a standalone that does nothing has to travel. result leaves that differ between OUR post-turn save and the oracle's. closed baseline - result, and -- separately -- any leaf we made worse. `closed` alone would be a comfortable number, so `regressed` is reported next to it: a leaf that agreed with the oracle before the turn and disagrees after it is a phase doing damage, and it is counted and named rather than netted off. tools/standalone_report.py # every pair, write the JSON + text report tools/standalone_report.py --print # also echo the report tools/standalone_report.py --pair A.sav B.sav # one ad-hoc pair tools/standalone_report.py --binary PATH # a sots_turn built elsewhere Outputs (overwritten): verify/results/standalone/status.json the completion metric, read by tools/dashboard.py verify/results/standalone/report.txt the human divergence report """ import argparse import datetime import json import os import shutil import subprocess import sys import tempfile RE_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) CK_DIR = os.path.join(RE_ROOT, "verify", "state-checksum") SAVE_DIR = os.path.join(RE_ROOT, "verify", "results", "saves") OUT_DIR = os.path.join(RE_ROOT, "verify", "results", "standalone") sys.path.insert(0, CK_DIR) import state_checksum as ck # noqa: E402 # The consecutive-turn pairs the corpus holds. A pair is (input, oracle): the oracle is the # state the game itself produced by ending a turn on the input. Only the first family is a # true End-Turn transition of one game; the others are listed so a regression on them is # still visible, with their nature stated. PAIRS = [ ("turn1-state.sav", "turn2-state.sav", "real End Turn"), ("turn2-state.sav", "turn3-state.sav", "real End Turn"), ] DEFAULT_BINARIES = [ os.path.expanduser("~/sots-engine-wt-yield/build-host/src/app/sots_turn"), os.path.expanduser("~/sots-engine-wt-standalone/build-host/src/app/sots_turn"), os.path.expanduser("~/sots-engine/build-host/src/app/sots_turn"), "/srv/re-lab/build/sots-engine-s2/src/app/sots_turn", ] def find_binary(explicit): if explicit: return explicit if os.path.exists(explicit) else None for p in DEFAULT_BINARIES: if os.path.exists(p): return p return shutil.which("sots_turn") def leaf_paths(a, b, limit=200000): """The set of leaf paths on which two checksummed saves differ.""" entries = ck.diff(a.root, b.root, limit=limit) return {e.path: e for e in entries} def run_pair(binary, src, oracle, note, workdir, keep_saves): out_sav = os.path.join(workdir, "post-" + os.path.basename(src)) metric = os.path.join(workdir, "metric-" + os.path.basename(src) + ".json") cmd = [binary, src, "--out", out_sav, "--metric", metric, "--roundtrip"] proc = subprocess.run(cmd, capture_output=True, text=True) row = { "input": os.path.basename(src), "oracle": os.path.basename(oracle), "note": note, "exit": proc.returncode, "stdout": proc.stdout.strip().splitlines()[-30:], } if proc.returncode != 0: row["error"] = proc.stderr.strip()[:2000] return row, [] ck_in = ck.checksum_save(src) ck_or = ck.checksum_save(oracle) ck_ours = ck.checksum_save(out_sav) base = leaf_paths(ck_in, ck_or) ours = leaf_paths(ck_ours, ck_or) closed = sorted(set(base) - set(ours)) regressed = sorted(set(ours) - set(base)) remaining = sorted(set(ours) & set(base)) row.update({ "coverage": { "input": ck_in.coverage.get("ok"), "oracle": ck_or.coverage.get("ok"), "ours": ck_ours.coverage.get("ok"), }, "roots": {"input": ck_in.digest, "oracle": ck_or.digest, "ours": ck_ours.digest}, "baselineDiverging": len(base), "divergingAfterTurn": len(ours), "closed": len(closed), "regressed": len(regressed), "closedPaths": closed, "regressedPaths": [repr(ours[p]) for p in regressed], "remainingSample": [repr(ours[p]) for p in remaining[:40]], "byteMatch": ck_ours.digest == ck_or.digest, }) if os.path.exists(metric): with open(metric) as f: row["standalone"] = json.load(f) if keep_saves: dst = os.path.join(OUT_DIR, os.path.basename(out_sav)) shutil.copyfile(out_sav, dst) row["savedTo"] = os.path.relpath(dst, RE_ROOT) return row, remaining def subsystem_breakdown(remaining): """Group the remaining divergences by the subsystem they land in.""" buckets = {} for p in remaining: parts = p.strip("/").split("/") key = "/" + "/".join(parts[:2]) if len(parts) > 1 else "/" + parts[0] buckets[key] = buckets.get(key, 0) + 1 return dict(sorted(buckets.items(), key=lambda kv: -kv[1])) def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--binary", help="path to sots_turn") ap.add_argument("--pair", nargs=2, metavar=("INPUT", "ORACLE"), help="one ad-hoc (input, oracle) save pair") ap.add_argument("--print", dest="echo", action="store_true", help="echo the report") ap.add_argument("--no-write", action="store_true", help="render only, touch nothing") ap.add_argument("--keep-saves", action="store_true", help="copy each post-turn save into verify/results/standalone/") args = ap.parse_args(argv) binary = find_binary(args.binary) if not binary: print("standalone_report: sots_turn not found; build sots-engine's host preset first", file=sys.stderr) print(" looked in: " + ", ".join(DEFAULT_BINARIES), file=sys.stderr) return 2 pairs = ([(args.pair[0], args.pair[1], "ad-hoc")] if args.pair else [(os.path.join(SAVE_DIR, a), os.path.join(SAVE_DIR, b), n) for a, b, n in PAIRS]) pairs = [(a, b, n) for a, b, n in pairs if os.path.exists(a) and os.path.exists(b)] if not pairs: print("standalone_report: no save pairs available, nothing to measure", file=sys.stderr) return 0 if not args.no_write: os.makedirs(OUT_DIR, exist_ok=True) rows, all_remaining = [], [] with tempfile.TemporaryDirectory() as tmp: for src, oracle, note in pairs: row, remaining = run_pair(binary, src, oracle, note, tmp, args.keep_saves and not args.no_write) rows.append(row) if row["input"] == os.path.basename(pairs[0][0]): all_remaining = remaining ok = [r for r in rows if r.get("exit") == 0] ref = ok[0] if ok else {} status = { "schema": "sots-standalone-status/1", "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "binary": binary, "reference": { "input": ref.get("input"), "oracle": ref.get("oracle"), "baselineDiverging": ref.get("baselineDiverging"), "divergingAfterTurn": ref.get("divergingAfterTurn"), "closed": ref.get("closed"), "regressed": ref.get("regressed"), "byteMatch": ref.get("byteMatch"), "subsystems": subsystem_breakdown(all_remaining), }, "phases": (ref.get("standalone") or {}).get("spine"), "tailPhases": (ref.get("standalone") or {}).get("tail"), "rng": { "wordsModelled": ((ref.get("standalone") or {}).get("run") or {}).get("rngWords"), "wordsPerTurnUnattributed": "18-20 (lane Z, in flight)", }, "pairs": rows, } lines = [] w = lines.append w("# standalone vs the oracle") w("") w(f"generated {status['generated']} binary {binary}") w("") ph = status["phases"] or {} tp = status["tailPhases"] or {} if ph: w(f"phases: {ph['modelled']}/{ph['total']} of the two turn drivers modelled, " f"{ph['committed']} committed " f"(implemented {ph['implemented']}, partial {ph['partial']}, " f"blocked {ph['blocked']}, stub {ph['stub']})") if tp: w(f" {tp['modelled']}/{tp['total']} of the post-combat tail modelled") w("") for r in rows: w(f"## {r['input']} -> {r['oracle']} ({r['note']})") if r.get("exit"): w(f" FAILED, exit {r['exit']}: {r.get('error', '')[:400]}") w("") continue w(f" baseline (do nothing) {r['baselineDiverging']:4d} leaves diverge") w(f" after one standalone turn {r['divergingAfterTurn']:4d} leaves diverge") w(f" closed {r['closed']}, regressed {r['regressed']}, " f"byte match: {'YES' if r['byteMatch'] else 'no'}") w(f" coverage proved on all three saves: {r['coverage']}") if r["closedPaths"]: w(" closed:") for p in r["closedPaths"]: w(f" + {p}") if r["regressedPaths"]: w(" REGRESSED (agreed before the turn, disagrees after):") for p in r["regressedPaths"]: w(f" - {p}") w("") if all_remaining: w("## what still differs on the reference pair, by subsystem") for k, v in subsystem_breakdown(all_remaining).items(): w(f" {v:4d} {k}") w("") w("## first 40 remaining, named") for p in (ok[0]["remainingSample"] if ok else []): w(f" {p}") report = "\n".join(lines) + "\n" if not args.no_write: with open(os.path.join(OUT_DIR, "status.json"), "w") as f: json.dump(status, f, indent=2) f.write("\n") with open(os.path.join(OUT_DIR, "report.txt"), "w") as f: f.write(report) print(f"wrote {os.path.relpath(OUT_DIR, RE_ROOT)}/status.json and report.txt") if args.echo or args.no_write: print(report) return 0 if __name__ == "__main__": sys.exit(main())