sots-re/tools/standalone_report.py

#!/usr/bin/env python3
"""Measure the standalone against the oracle, and record the distance.

The milestone is: the standalone loads a save, runs one strategic turn, and writes an
autosave that byte-matches what the original produces from the same state.  This tool
measures how far off that is, in the only currency the campaign trusts -- named leaves of
`verify/state-checksum/state_checksum.py`, whose coverage is proved by re-serialisation.

For each (before, after) pair of real saves it computes three numbers:

  baseline   leaves that differ between the INPUT save and the oracle's post-turn save.
             This is the distance a standalone that does nothing has to travel.
  result     leaves that differ between OUR post-turn save and the oracle's.
  closed     baseline - result, and -- separately -- any leaf we made worse.

`closed` alone would be a comfortable number, so `regressed` is reported next to it: a leaf
that agreed with the oracle before the turn and disagrees after it is a phase doing damage,
and it is counted and named rather than netted off.

  tools/standalone_report.py                     # every pair, write the JSON + text report
  tools/standalone_report.py --print             # also echo the report
  tools/standalone_report.py --pair A.sav B.sav  # one ad-hoc pair
  tools/standalone_report.py --binary PATH       # a sots_turn built elsewhere

Outputs (overwritten):
  verify/results/standalone/status.json    the completion metric, read by tools/dashboard.py
  verify/results/standalone/report.txt     the human divergence report
"""
import argparse
import datetime
import json
import os
import shutil
import subprocess
import sys
import tempfile

RE_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
CK_DIR = os.path.join(RE_ROOT, "verify", "state-checksum")
SAVE_DIR = os.path.join(RE_ROOT, "verify", "results", "saves")
OUT_DIR = os.path.join(RE_ROOT, "verify", "results", "standalone")

sys.path.insert(0, CK_DIR)
import state_checksum as ck  # noqa: E402

# The consecutive-turn pairs the corpus holds.  A pair is (input, oracle): the oracle is the
# state the game itself produced by ending a turn on the input.  Only the first family is a
# true End-Turn transition of one game; the others are listed so a regression on them is
# still visible, with their nature stated.
PAIRS = [
    ("turn1-state.sav", "turn2-state.sav", "real End Turn"),
    ("turn2-state.sav", "turn3-state.sav", "real End Turn"),
]

DEFAULT_BINARIES = [
    os.path.expanduser("~/sots-engine-wt-yield/build-host/src/app/sots_turn"),
    os.path.expanduser("~/sots-engine-wt-standalone/build-host/src/app/sots_turn"),
    os.path.expanduser("~/sots-engine/build-host/src/app/sots_turn"),
    "/srv/re-lab/build/sots-engine-s2/src/app/sots_turn",
]


def find_binary(explicit):
    if explicit:
        return explicit if os.path.exists(explicit) else None
    for p in DEFAULT_BINARIES:
        if os.path.exists(p):
            return p
    return shutil.which("sots_turn")


def leaf_paths(a, b, limit=200000):
    """The set of leaf paths on which two checksummed saves differ."""
    entries = ck.diff(a.root, b.root, limit=limit)
    return {e.path: e for e in entries}


def run_pair(binary, src, oracle, note, workdir, keep_saves, engine_args=()):
    out_sav = os.path.join(workdir, "post-" + os.path.basename(src))
    metric = os.path.join(workdir, "metric-" + os.path.basename(src) + ".json")
    cmd = [binary, src, "--out", out_sav, "--metric", metric, "--roundtrip"] + list(engine_args)
    proc = subprocess.run(cmd, capture_output=True, text=True)
    row = {
        "input": os.path.basename(src),
        "oracle": os.path.basename(oracle),
        "note": note,
        "engineArgs": list(engine_args),
        "exit": proc.returncode,
        "stdout": proc.stdout.strip().splitlines()[-30:],
    }
    if proc.returncode != 0:
        row["error"] = proc.stderr.strip()[:2000]
        return row, []

    ck_in = ck.checksum_save(src)
    ck_or = ck.checksum_save(oracle)
    ck_ours = ck.checksum_save(out_sav)

    base = leaf_paths(ck_in, ck_or)
    ours = leaf_paths(ck_ours, ck_or)

    closed = sorted(set(base) - set(ours))
    regressed = sorted(set(ours) - set(base))
    remaining = sorted(set(ours) & set(base))

    row.update({
        "coverage": {
            "input": ck_in.coverage.get("ok"),
            "oracle": ck_or.coverage.get("ok"),
            "ours": ck_ours.coverage.get("ok"),
        },
        "roots": {"input": ck_in.digest, "oracle": ck_or.digest, "ours": ck_ours.digest},
        "baselineDiverging": len(base),
        "divergingAfterTurn": len(ours),
        "closed": len(closed),
        "regressed": len(regressed),
        "closedPaths": closed,
        "regressedPaths": [repr(ours[p]) for p in regressed],
        "remainingSample": [repr(ours[p]) for p in remaining[:40]],
        "byteMatch": ck_ours.digest == ck_or.digest,
    })
    if os.path.exists(metric):
        with open(metric) as f:
            row["standalone"] = json.load(f)
    if keep_saves:
        dst = os.path.join(OUT_DIR, os.path.basename(out_sav))
        shutil.copyfile(out_sav, dst)
        row["savedTo"] = os.path.relpath(dst, RE_ROOT)
    return row, remaining


def subsystem_breakdown(remaining):
    """Group the remaining divergences by the subsystem they land in."""
    buckets = {}
    for p in remaining:
        parts = p.strip("/").split("/")
        key = "/" + "/".join(parts[:2]) if len(parts) > 1 else "/" + parts[0]
        buckets[key] = buckets.get(key, 0) + 1
    return dict(sorted(buckets.items(), key=lambda kv: -kv[1]))


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--binary", help="path to sots_turn")
    ap.add_argument("--pair", nargs=2, metavar=("INPUT", "ORACLE"),
                    help="one ad-hoc (input, oracle) save pair")
    ap.add_argument("--print", dest="echo", action="store_true", help="echo the report")
    ap.add_argument("--no-write", action="store_true", help="render only, touch nothing")
    ap.add_argument("--keep-saves", action="store_true",
                    help="copy each post-turn save into verify/results/standalone/")
    # Some phases are fed by operator inputs the save does not carry -- the game's data root
    # (--data), the AI roster (--ai-player N), and which blocked phases may commit
    # (--commit-blocked=IDS).  Those are properties of the recorded game, not of the tool, so
    # they are passed through rather than hard-coded, and every run records what it was given.
    ap.add_argument("--engine-arg", dest="engine_args", action="append", default=[],
                    metavar="ARG", help="extra argument for sots_turn; repeatable")
    args = ap.parse_args(argv)

    binary = find_binary(args.binary)
    if not binary:
        print("standalone_report: sots_turn not found; build sots-engine's host preset first",
              file=sys.stderr)
        print("  looked in: " + ", ".join(DEFAULT_BINARIES), file=sys.stderr)
        return 2

    pairs = ([(args.pair[0], args.pair[1], "ad-hoc")] if args.pair
             else [(os.path.join(SAVE_DIR, a), os.path.join(SAVE_DIR, b), n)
                   for a, b, n in PAIRS])
    pairs = [(a, b, n) for a, b, n in pairs if os.path.exists(a) and os.path.exists(b)]
    if not pairs:
        print("standalone_report: no save pairs available, nothing to measure", file=sys.stderr)
        return 0

    if not args.no_write:
        os.makedirs(OUT_DIR, exist_ok=True)

    rows, all_remaining = [], []
    with tempfile.TemporaryDirectory() as tmp:
        for src, oracle, note in pairs:
            row, remaining = run_pair(binary, src, oracle, note, tmp,
                                      args.keep_saves and not args.no_write,
                                      args.engine_args)
            rows.append(row)
            if row["input"] == os.path.basename(pairs[0][0]):
                all_remaining = remaining

    ok = [r for r in rows if r.get("exit") == 0]
    ref = ok[0] if ok else {}
    status = {
        "schema": "sots-standalone-status/1",
        "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "binary": binary,
        "engineArgs": list(args.engine_args),
        "reference": {
            "input": ref.get("input"),
            "oracle": ref.get("oracle"),
            "baselineDiverging": ref.get("baselineDiverging"),
            "divergingAfterTurn": ref.get("divergingAfterTurn"),
            "closed": ref.get("closed"),
            "regressed": ref.get("regressed"),
            "byteMatch": ref.get("byteMatch"),
            "subsystems": subsystem_breakdown(all_remaining),
        },
        "phases": (ref.get("standalone") or {}).get("spine"),
        "tailPhases": (ref.get("standalone") or {}).get("tail"),
        "rng": {
            "wordsModelled": ((ref.get("standalone") or {}).get("run") or {}).get("rngWords"),
            "wordsPerTurnUnattributed": "18-20 (lane Z, in flight)",
        },
        "pairs": rows,
    }

    lines = []
    w = lines.append
    w("# standalone vs the oracle")
    w("")
    w(f"generated {status['generated']}   binary {binary}")
    if args.engine_args:
        w("engine args: " + " ".join(args.engine_args))
    w("")
    ph = status["phases"] or {}
    tp = status["tailPhases"] or {}
    if ph:
        w(f"phases: {ph['modelled']}/{ph['total']} of the two turn drivers modelled, "
          f"{ph['committed']} committed "
          f"(implemented {ph['implemented']}, partial {ph['partial']}, "
          f"blocked {ph['blocked']}, stub {ph['stub']})")
    if tp:
        w(f"        {tp['modelled']}/{tp['total']} of the post-combat tail modelled")
    w("")
    for r in rows:
        w(f"## {r['input']} -> {r['oracle']}   ({r['note']})")
        if r.get("exit"):
            w(f"   FAILED, exit {r['exit']}: {r.get('error', '')[:400]}")
            w("")
            continue
        w(f"   baseline (do nothing)     {r['baselineDiverging']:4d} leaves diverge")
        w(f"   after one standalone turn {r['divergingAfterTurn']:4d} leaves diverge")
        w(f"   closed {r['closed']}, regressed {r['regressed']}, "
          f"byte match: {'YES' if r['byteMatch'] else 'no'}")
        w(f"   coverage proved on all three saves: {r['coverage']}")
        if r["closedPaths"]:
            w("   closed:")
            for p in r["closedPaths"]:
                w(f"     + {p}")
        if r["regressedPaths"]:
            w("   REGRESSED (agreed before the turn, disagrees after):")
            for p in r["regressedPaths"]:
                w(f"     - {p}")
        w("")
    if all_remaining:
        w("## what still differs on the reference pair, by subsystem")
        for k, v in subsystem_breakdown(all_remaining).items():
            w(f"   {v:4d}  {k}")
        w("")
        w("## first 40 remaining, named")
        for p in (ok[0]["remainingSample"] if ok else []):
            w(f"   {p}")
    report = "\n".join(lines) + "\n"

    if not args.no_write:
        with open(os.path.join(OUT_DIR, "status.json"), "w") as f:
            json.dump(status, f, indent=2)
            f.write("\n")
        with open(os.path.join(OUT_DIR, "report.txt"), "w") as f:
            f.write(report)
        print(f"wrote {os.path.relpath(OUT_DIR, RE_ROOT)}/status.json and report.txt")
    if args.echo or args.no_write:
        print(report)
    return 0


if __name__ == "__main__":
    sys.exit(main())