283 lines
12 KiB
Python
283 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Measure the standalone against the oracle, and record the distance.
|
|
|
|
The milestone is: the standalone loads a save, runs one strategic turn, and writes an
|
|
autosave that byte-matches what the original produces from the same state. This tool
|
|
measures how far off that is, in the only currency the campaign trusts -- named leaves of
|
|
`verify/state-checksum/state_checksum.py`, whose coverage is proved by re-serialisation.
|
|
|
|
For each (before, after) pair of real saves it computes three numbers:
|
|
|
|
baseline leaves that differ between the INPUT save and the oracle's post-turn save.
|
|
This is the distance a standalone that does nothing has to travel.
|
|
result leaves that differ between OUR post-turn save and the oracle's.
|
|
closed baseline - result, and -- separately -- any leaf we made worse.
|
|
|
|
`closed` alone would be a comfortable number, so `regressed` is reported next to it: a leaf
|
|
that agreed with the oracle before the turn and disagrees after it is a phase doing damage,
|
|
and it is counted and named rather than netted off.
|
|
|
|
tools/standalone_report.py # every pair, write the JSON + text report
|
|
tools/standalone_report.py --print # also echo the report
|
|
tools/standalone_report.py --pair A.sav B.sav # one ad-hoc pair
|
|
tools/standalone_report.py --binary PATH # a sots_turn built elsewhere
|
|
|
|
Outputs (overwritten):
|
|
verify/results/standalone/status.json the completion metric, read by tools/dashboard.py
|
|
verify/results/standalone/report.txt the human divergence report
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
RE_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
|
|
CK_DIR = os.path.join(RE_ROOT, "verify", "state-checksum")
|
|
SAVE_DIR = os.path.join(RE_ROOT, "verify", "results", "saves")
|
|
OUT_DIR = os.path.join(RE_ROOT, "verify", "results", "standalone")
|
|
|
|
sys.path.insert(0, CK_DIR)
|
|
import state_checksum as ck # noqa: E402
|
|
|
|
# The consecutive-turn pairs the corpus holds. A pair is (input, oracle): the oracle is the
|
|
# state the game itself produced by ending a turn on the input. Only the first family is a
|
|
# true End-Turn transition of one game; the others are listed so a regression on them is
|
|
# still visible, with their nature stated.
|
|
# Order matters: pairs[0] is the REFERENCE the dashboard quotes.
|
|
#
|
|
# 2026-09-08: turn2->turn3 is now the reference. turn1->turn2 is NOT deterministic in the original
|
|
# (lane L5: three runs, three autosaves, four leaves — one AI empire's research pick plus the
|
|
# derived checksum; hooks=off was a third value, so it is the game). turn2->turn3 is the pair the
|
|
# byte-identical oracle hashes were taken on (every AI already holds a target, so the decision
|
|
# layer is idle). turn1->turn2 stays as a secondary pair; its four non-deterministic leaves are a
|
|
# Rung-B residual by construction and should be read as such (masking them is a follow-up).
|
|
PAIRS = [
|
|
("turn2-state.sav", "turn3-state.sav", "real End Turn (canonical reference)"),
|
|
("turn1-state.sav", "turn2-state.sav", "real End Turn (secondary; 4 leaves non-deterministic in the original)"),
|
|
]
|
|
|
|
DEFAULT_BINARIES = [
|
|
os.path.expanduser("~/sots-engine-wt-yield/build-host/src/app/sots_turn"),
|
|
os.path.expanduser("~/sots-engine-wt-standalone/build-host/src/app/sots_turn"),
|
|
os.path.expanduser("~/sots-engine/build-host/src/app/sots_turn"),
|
|
"/srv/re-lab/build/sots-engine-s2/src/app/sots_turn",
|
|
]
|
|
|
|
|
|
def find_binary(explicit):
|
|
if explicit:
|
|
return explicit if os.path.exists(explicit) else None
|
|
for p in DEFAULT_BINARIES:
|
|
if os.path.exists(p):
|
|
return p
|
|
return shutil.which("sots_turn")
|
|
|
|
|
|
def leaf_paths(a, b, limit=200000):
|
|
"""The set of leaf paths on which two checksummed saves differ."""
|
|
entries = ck.diff(a.root, b.root, limit=limit)
|
|
return {e.path: e for e in entries}
|
|
|
|
|
|
def run_pair(binary, src, oracle, note, workdir, keep_saves, engine_args=()):
|
|
out_sav = os.path.join(workdir, "post-" + os.path.basename(src))
|
|
metric = os.path.join(workdir, "metric-" + os.path.basename(src) + ".json")
|
|
cmd = [binary, src, "--out", out_sav, "--metric", metric, "--roundtrip"] + list(engine_args)
|
|
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
row = {
|
|
"input": os.path.basename(src),
|
|
"oracle": os.path.basename(oracle),
|
|
"note": note,
|
|
"engineArgs": list(engine_args),
|
|
"exit": proc.returncode,
|
|
"stdout": proc.stdout.strip().splitlines()[-30:],
|
|
}
|
|
if proc.returncode != 0:
|
|
row["error"] = proc.stderr.strip()[:2000]
|
|
return row, []
|
|
|
|
ck_in = ck.checksum_save(src)
|
|
ck_or = ck.checksum_save(oracle)
|
|
ck_ours = ck.checksum_save(out_sav)
|
|
|
|
base = leaf_paths(ck_in, ck_or)
|
|
ours = leaf_paths(ck_ours, ck_or)
|
|
|
|
closed = sorted(set(base) - set(ours))
|
|
regressed = sorted(set(ours) - set(base))
|
|
remaining = sorted(set(ours) & set(base))
|
|
|
|
row.update({
|
|
"coverage": {
|
|
"input": ck_in.coverage.get("ok"),
|
|
"oracle": ck_or.coverage.get("ok"),
|
|
"ours": ck_ours.coverage.get("ok"),
|
|
},
|
|
"roots": {"input": ck_in.digest, "oracle": ck_or.digest, "ours": ck_ours.digest},
|
|
"baselineDiverging": len(base),
|
|
"divergingAfterTurn": len(ours),
|
|
"closed": len(closed),
|
|
"regressed": len(regressed),
|
|
"closedPaths": closed,
|
|
"regressedPaths": [repr(ours[p]) for p in regressed],
|
|
"remainingSample": [repr(ours[p]) for p in remaining[:40]],
|
|
"byteMatch": ck_ours.digest == ck_or.digest,
|
|
})
|
|
if os.path.exists(metric):
|
|
with open(metric) as f:
|
|
row["standalone"] = json.load(f)
|
|
if keep_saves:
|
|
dst = os.path.join(OUT_DIR, os.path.basename(out_sav))
|
|
shutil.copyfile(out_sav, dst)
|
|
row["savedTo"] = os.path.relpath(dst, RE_ROOT)
|
|
return row, remaining
|
|
|
|
|
|
def subsystem_breakdown(remaining):
|
|
"""Group the remaining divergences by the subsystem they land in."""
|
|
buckets = {}
|
|
for p in remaining:
|
|
parts = p.strip("/").split("/")
|
|
key = "/" + "/".join(parts[:2]) if len(parts) > 1 else "/" + parts[0]
|
|
buckets[key] = buckets.get(key, 0) + 1
|
|
return dict(sorted(buckets.items(), key=lambda kv: -kv[1]))
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--binary", help="path to sots_turn")
|
|
ap.add_argument("--pair", nargs=2, metavar=("INPUT", "ORACLE"),
|
|
help="one ad-hoc (input, oracle) save pair")
|
|
ap.add_argument("--print", dest="echo", action="store_true", help="echo the report")
|
|
ap.add_argument("--no-write", action="store_true", help="render only, touch nothing")
|
|
ap.add_argument("--keep-saves", action="store_true",
|
|
help="copy each post-turn save into verify/results/standalone/")
|
|
# Some phases are fed by operator inputs the save does not carry -- the game's data root
|
|
# (--data), the AI roster (--ai-player N), and which blocked phases may commit
|
|
# (--commit-blocked=IDS). Those are properties of the recorded game, not of the tool, so
|
|
# they are passed through rather than hard-coded, and every run records what it was given.
|
|
ap.add_argument("--engine-arg", dest="engine_args", action="append", default=[],
|
|
metavar="ARG", help="extra argument for sots_turn; repeatable")
|
|
args = ap.parse_args(argv)
|
|
|
|
binary = find_binary(args.binary)
|
|
if not binary:
|
|
print("standalone_report: sots_turn not found; build sots-engine's host preset first",
|
|
file=sys.stderr)
|
|
print(" looked in: " + ", ".join(DEFAULT_BINARIES), file=sys.stderr)
|
|
return 2
|
|
|
|
pairs = ([(args.pair[0], args.pair[1], "ad-hoc")] if args.pair
|
|
else [(os.path.join(SAVE_DIR, a), os.path.join(SAVE_DIR, b), n)
|
|
for a, b, n in PAIRS])
|
|
pairs = [(a, b, n) for a, b, n in pairs if os.path.exists(a) and os.path.exists(b)]
|
|
if not pairs:
|
|
print("standalone_report: no save pairs available, nothing to measure", file=sys.stderr)
|
|
return 0
|
|
|
|
if not args.no_write:
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
|
|
rows, all_remaining = [], []
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
for src, oracle, note in pairs:
|
|
row, remaining = run_pair(binary, src, oracle, note, tmp,
|
|
args.keep_saves and not args.no_write,
|
|
args.engine_args)
|
|
rows.append(row)
|
|
if row["input"] == os.path.basename(pairs[0][0]):
|
|
all_remaining = remaining
|
|
|
|
ok = [r for r in rows if r.get("exit") == 0]
|
|
ref = ok[0] if ok else {}
|
|
status = {
|
|
"schema": "sots-standalone-status/1",
|
|
"generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"binary": binary,
|
|
"engineArgs": list(args.engine_args),
|
|
"reference": {
|
|
"input": ref.get("input"),
|
|
"oracle": ref.get("oracle"),
|
|
"baselineDiverging": ref.get("baselineDiverging"),
|
|
"divergingAfterTurn": ref.get("divergingAfterTurn"),
|
|
"closed": ref.get("closed"),
|
|
"regressed": ref.get("regressed"),
|
|
"byteMatch": ref.get("byteMatch"),
|
|
"subsystems": subsystem_breakdown(all_remaining),
|
|
},
|
|
"phases": (ref.get("standalone") or {}).get("spine"),
|
|
"tailPhases": (ref.get("standalone") or {}).get("tail"),
|
|
"rng": {
|
|
"wordsModelled": ((ref.get("standalone") or {}).get("run") or {}).get("rngWords"),
|
|
"wordsPerTurnUnattributed": "18-20 (lane Z, in flight)",
|
|
},
|
|
"pairs": rows,
|
|
}
|
|
|
|
lines = []
|
|
w = lines.append
|
|
w("# standalone vs the oracle")
|
|
w("")
|
|
w(f"generated {status['generated']} binary {binary}")
|
|
if args.engine_args:
|
|
w("engine args: " + " ".join(args.engine_args))
|
|
w("")
|
|
ph = status["phases"] or {}
|
|
tp = status["tailPhases"] or {}
|
|
if ph:
|
|
w(f"phases: {ph['modelled']}/{ph['total']} of the two turn drivers modelled, "
|
|
f"{ph['committed']} committed "
|
|
f"(implemented {ph['implemented']}, partial {ph['partial']}, "
|
|
f"blocked {ph['blocked']}, stub {ph['stub']})")
|
|
if tp:
|
|
w(f" {tp['modelled']}/{tp['total']} of the post-combat tail modelled")
|
|
w("")
|
|
for r in rows:
|
|
w(f"## {r['input']} -> {r['oracle']} ({r['note']})")
|
|
if r.get("exit"):
|
|
w(f" FAILED, exit {r['exit']}: {r.get('error', '')[:400]}")
|
|
w("")
|
|
continue
|
|
w(f" baseline (do nothing) {r['baselineDiverging']:4d} leaves diverge")
|
|
w(f" after one standalone turn {r['divergingAfterTurn']:4d} leaves diverge")
|
|
w(f" closed {r['closed']}, regressed {r['regressed']}, "
|
|
f"byte match: {'YES' if r['byteMatch'] else 'no'}")
|
|
w(f" coverage proved on all three saves: {r['coverage']}")
|
|
if r["closedPaths"]:
|
|
w(" closed:")
|
|
for p in r["closedPaths"]:
|
|
w(f" + {p}")
|
|
if r["regressedPaths"]:
|
|
w(" REGRESSED (agreed before the turn, disagrees after):")
|
|
for p in r["regressedPaths"]:
|
|
w(f" - {p}")
|
|
w("")
|
|
if all_remaining:
|
|
w("## what still differs on the reference pair, by subsystem")
|
|
for k, v in subsystem_breakdown(all_remaining).items():
|
|
w(f" {v:4d} {k}")
|
|
w("")
|
|
w("## first 40 remaining, named")
|
|
for p in (ok[0]["remainingSample"] if ok else []):
|
|
w(f" {p}")
|
|
report = "\n".join(lines) + "\n"
|
|
|
|
if not args.no_write:
|
|
with open(os.path.join(OUT_DIR, "status.json"), "w") as f:
|
|
json.dump(status, f, indent=2)
|
|
f.write("\n")
|
|
with open(os.path.join(OUT_DIR, "report.txt"), "w") as f:
|
|
f.write(report)
|
|
print(f"wrote {os.path.relpath(OUT_DIR, RE_ROOT)}/status.json and report.txt")
|
|
if args.echo or args.no_write:
|
|
print(report)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|