tools/displacement.py: the honest progress metric - what fraction of the game runs on our code, by rung, with coverage caveats attached
This commit is contained in:
alex
parent
c4f1c2407a
commit
3a44f9178b
1 changed files with 133 additions and 0 deletions
133
tools/displacement.py
Normal file
133
tools/displacement.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Report engine displacement — how much of the game runs on OUR code, verified.
|
||||
|
||||
The campaign's honest progress metric. Coverage of the *save format* is closed and the
|
||||
board is full of green rows, but neither answers the question the north star actually asks:
|
||||
**how much of the original's behaviour has our code replaced, proven against the live game?**
|
||||
|
||||
Four rungs, deliberately steep:
|
||||
|
||||
replaced ours RAN INSTEAD OF the original on the live game, and the End-Turn autosave
|
||||
oracle still matched. This is the only rung that means the original's code did
|
||||
not execute. It is the number that matters.
|
||||
compared ours ran ALONGSIDE the original on the live game and agreed on every declared
|
||||
region. Strong, but the original still did the work — and a clean compare on a
|
||||
wrong region set checks nothing (three of B4's hooks printed 0 diverged while
|
||||
comparing nothing; see guides/method-rules.md rule 1).
|
||||
modelled implemented in sots-engine with host tests, never run against the live game.
|
||||
mapped understood and written up; no code.
|
||||
|
||||
Coverage caveats travel WITH the rung, never netted off it. A hook is only listed at
|
||||
`replaced` if a report names the oracle it satisfied.
|
||||
|
||||
Source of truth is this file's table, curated at integration from the compare reports in
|
||||
verify/results/compare/ and the board rows that cite them. It is deliberately hand-curated:
|
||||
deriving it by grepping status words would let an over-claimed board row silently inflate
|
||||
the project's headline number.
|
||||
|
||||
tools/displacement.py # human-readable table
|
||||
tools/displacement.py --json # machine-readable, for the dashboard
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
# (subsystem, rung, live evidence, coverage caveat that travels with it)
|
||||
ROWS = [
|
||||
("mars/parse + text (data readers)", "modelled",
|
||||
"oracles 1,531/1,531 and 64/64 vs the Python reference",
|
||||
"never hooked; the original's parser still runs the game"),
|
||||
("mars/vfs (.gob + native override)", "modelled",
|
||||
"10,268 files CRC-clean, byte-equal spot checks",
|
||||
"never hooked"),
|
||||
("mars/stream (save read/write)", "modelled",
|
||||
"100% named coverage, byte-identical round-trip on 11 saves, coverage proved by re-serialisation",
|
||||
"the game's own load/save path is untouched; ours reads the same bytes offline"),
|
||||
("mars/rng (MT19937 + 7 entry points)", "compared",
|
||||
"every draw of a full turn attributed live; 3 instruments agreed on turn 5's 20 words",
|
||||
"our generator is not driving the game; it reproduces the ledger"),
|
||||
("GlobalConsts::LoadFile (flat-KV config)", "replaced",
|
||||
"trace 19 calls / 1,088 regions -> compare 19/19 0 div -> REPLACE, End-Turn hash oracle held",
|
||||
"config load only; one call site"),
|
||||
("manifest / id registry (weapons)", "replaced",
|
||||
"trace 22 calls -> compare 1/1 0 div -> REPLACE + oracle byte-identical",
|
||||
"weapons path only; the section path still crashes in compare (board: section-loader compare crash)"),
|
||||
("ServerPlayer::ComputeBudget", "replaced",
|
||||
"4,437 compares 0 div; replace-mode oracle byte-identical",
|
||||
"QUALIFIED: replace runs the ORIGINAL a second time to harvest slots -> a real per-turn double "
|
||||
"effect (ComputeOutput repairs ships in orbit) that no region reaches. 13 of 22 slots are 0 on "
|
||||
"every call; only 20 distinct states across 4,437 calls"),
|
||||
("TechTree::ProcessResearch + unlock cascade", "compared",
|
||||
"35 calls across 3 workloads, 0 divergences, tracecmp exit 0; advance prediction held on a "
|
||||
"changed workload (unlock costs no earlier report contained)",
|
||||
"compare only, never replaced"),
|
||||
("ServerPlayer::OnTechResearched", "compared",
|
||||
"3 completions, 0 div; float32 confirmed bit-for-bit on the game",
|
||||
"compare only; RollResearchEvent's branch has fired once in four sessions"),
|
||||
("ServerSystem::ComputeTotalOutput + GroupOutput", "compared",
|
||||
"24,357 guarded calls, 0 undeclared writes, 1 residual ulp the caller rounds away",
|
||||
"compare only; 24,357 calls is THIRTEEN distinct system states"),
|
||||
("MoveFleet (position update)", "compared",
|
||||
"45 calls, live 8 divergences -> 0 after the five-narrowing fix; mechanism match, not a numeric fit",
|
||||
"compare only; all 15 moving calls are the same straight-run waypoint type"),
|
||||
("ServerSystem::ProcessTurn (colony)", "compared",
|
||||
"36 calls 0 div; RNG left-delta 0 on all 28 systems",
|
||||
"compare only; 3 owned systems, gate traffic all zero, most branches untested"),
|
||||
("events (EventStorage + PostEvent)", "compared",
|
||||
"next_id 3->4 both sides live; observed_techs delta measured at exactly 44 bytes",
|
||||
"COUNT-ONLY by design: ours never calls the game's PostEvent and replace mode writes nothing"),
|
||||
("game/nav (route classifier)", "modelled",
|
||||
"offline vs 58 waypoints / 46 flight plans across 11 saves, 0 failures",
|
||||
"never instrumented; not one leg has executed under a hook"),
|
||||
("game/combat (retreat planner)", "modelled",
|
||||
"53 hand-computed checks",
|
||||
"never instrumented; the resolver has never executed under a hook at all "
|
||||
"(every observed encounter had the apply flag set)"),
|
||||
("game/design (hull class, census)", "modelled",
|
||||
"11 saves, 503 designs, 480 census leaves, 0 mismatched, computed twice independently",
|
||||
"3 of 6 census leaves are unexercised: no cruiser and no DN platform in the corpus"),
|
||||
("app/sots_turn (whole strategic turn)", "modelled",
|
||||
"runs 11/11 saves; reference pair 209 -> 204 leaves, closed 5 regressed 0",
|
||||
"14 of 44 driver phases and 2 of 37 tail phases; verified 0 BY DEFINITION (no VM this cycle)"),
|
||||
("AI turn logic", "mapped", "-", "essentially unread; gates the full byte-match (Rung B)"),
|
||||
("combat simulation", "mapped", "-", "parked by decision"),
|
||||
("renderer / UI", "mapped", "-", "not on any path; DXVK carries rendering"),
|
||||
]
|
||||
|
||||
RUNGS = ["replaced", "compared", "modelled", "mapped"]
|
||||
BLURB = {
|
||||
"replaced": "ours ran INSTEAD of the original, live, oracle held",
|
||||
"compared": "ours ran alongside and agreed; the original still did the work",
|
||||
"modelled": "implemented + host-tested; never run against the live game",
|
||||
"mapped": "understood; no code",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
counts = {r: sum(1 for x in ROWS if x[1] == r) for r in RUNGS}
|
||||
if "--json" in sys.argv:
|
||||
print(json.dumps({
|
||||
"schema": "sots-displacement/1",
|
||||
"counts": counts,
|
||||
"total": len(ROWS),
|
||||
"rows": [dict(zip(("subsystem", "rung", "evidence", "caveat"), r)) for r in ROWS],
|
||||
}, indent=2))
|
||||
return
|
||||
print("Engine displacement — how much of the game runs on our code\n")
|
||||
for r in RUNGS:
|
||||
print(f" {r:9s} {counts[r]:2d} {BLURB[r]}")
|
||||
print(f"\n {len(ROWS)} subsystems tracked. Only `replaced` means the original's code did not run.\n")
|
||||
for r in RUNGS:
|
||||
rows = [x for x in ROWS if x[1] == r]
|
||||
if not rows:
|
||||
continue
|
||||
print(f"== {r} ==")
|
||||
for name, _, ev, caveat in rows:
|
||||
print(f" {name}")
|
||||
if ev != "-":
|
||||
print(f" live: {ev}")
|
||||
print(f" caveat: {caveat}")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue