From 496a5124c9202504c196eb69a1ee32a26bddf70f Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 8 Sep 2026 09:16:24 -0400 Subject: [PATCH] Z: measure the strategic RNG, do not assume it The turn's RNG cost has never been measured end to end. combat-done-tail.md found two draw sites in OnAllCombatDone_Tail that nothing models and that both run before the autosave, so a reimplementation that reproduces both ProcessTurn functions exactly still diverges the first turn a node line expires. RngLedger recovers an ABSOLUTE WORD POSITION from (mt[624], left) alone, by indexing the forward-only chain of blocks the twist generates. Word deltas between any two observations are then exact -- across twists, across NextInt rejection loops, and across draws nobody hooked. That last point is not theoretical: the image has four draw entry points, one of which (NextUInt 0x004f7670) appears in no previous lane's primitive set, plus inlined draws in twelve functions. A primitive-counting hook would have undercounted silently. Six nested trace hooks bracket one End Turn between the two autosaves and attribute the words: the two turn drivers, the two tail phases that can draw, and ProcessNodeSpaceTravel because it runs twice a turn. NodeLineDecay carries a real model -- one word per expired node line under NodePath::RemainingLife -- so compare mode checks the count rather than reporting it. Corrections from the instruction stream, both load-bearing: * StrategyHost::Autosave is ret 8, not ret 4, and returns the std::string* in EAX. A void-returning hook would have dropped it at both call sites. * node-line decay's 0x20000-fleet skip runs AFTER the Chance(0.5f) call, not before, so it cannot change the draw count -- combat-done-tail.md reads as if it gated the roll. fpu.sample_turn releases StrategyServer::ProcessTurn, which the fpu sampler and this ledger both want and MinHook grants to one of them. Default on: no existing run changes behaviour. Host ctest 37/37; shim cross-built on CT111; clean-room check OK. --- CMakeLists.txt | 14 +- docs/Z-tail-rng.md | 4 +- include/generated/sots_addresses.h | 34 +- src/shim/fpu_force.cpp | 20 +- src/shim/hooks/rng_ledger.cpp | 146 ++++++ src/shim/hooks/rng_ledger.h | 95 ++++ src/shim/hooks/tail_rng.cpp | 664 +++++++++++++++++++++++++++ src/shim/hooks/tail_rng.h | 164 +++++++ src/shim/main.cpp | 27 ++ src/shim/shim.cfg.zledger | 40 ++ tests/shim_rng_ledger/CMakeLists.txt | 6 + tests/shim_rng_ledger/unit_tests.cpp | 182 ++++++++ 12 files changed, 1389 insertions(+), 7 deletions(-) create mode 100644 src/shim/hooks/rng_ledger.cpp create mode 100644 src/shim/hooks/rng_ledger.h create mode 100644 src/shim/hooks/tail_rng.cpp create mode 100644 src/shim/hooks/tail_rng.h create mode 100644 src/shim/shim.cfg.zledger create mode 100644 tests/shim_rng_ledger/CMakeLists.txt create mode 100644 tests/shim_rng_ledger/unit_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bbe6168..b9fc920 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,13 @@ add_library(shim_events STATIC src/shim/hooks/event_inputs.cpp) target_link_libraries(shim_events PUBLIC shim_trace sots_addresses sots_game_events) target_compile_options(shim_events PRIVATE -Wall -Wextra -Werror) +# ---- shim: the strategic RNG word ledger (lane Z) -- absolute generator position recovered +# from (mt[624], left) alone, so a turn's RNG cost can be attributed to a phase without +# hooking a single RNG primitive. Host-testable; linked into the hook descriptors below. +add_library(shim_rng_ledger STATIC src/shim/hooks/rng_ledger.cpp) +target_link_libraries(shim_rng_ledger PUBLIC shim_trace sots_addresses mars_rng) +target_compile_options(shim_rng_ledger PRIVATE -Wall -Wextra -Werror) + if(WIN32) # ---- shim: proxy binkw32.dll that the original game loads (Phase 2 frontend) ---- add_library(minhook STATIC @@ -88,11 +95,12 @@ if(WIN32) src/shim/hooks/compute_budget.cpp src/shim/hooks/colony_turn.cpp src/shim/hooks/fleet_movement.cpp - src/shim/hooks/player_turn.cpp) + src/shim/hooks/player_turn.cpp + src/shim/hooks/tail_rng.cpp) target_link_libraries(shim_hooks PUBLIC shim_trace sots_addresses sots_game_config sots_game_sim sots_game_effects mars_rng shim_budget shim_techfx shim_colony shim_movement shim_events - shim_player_turn) + shim_player_turn shim_rng_ledger) target_compile_options(shim_hooks PRIVATE -Wall -Wextra -Werror) add_library(binkw32 SHARED src/shim/main.cpp src/shim/fpu_force.cpp src/shim/binkw32.def) @@ -108,7 +116,7 @@ else() add_executable(addr_smoke tests/addr_smoke.cpp) target_link_libraries(addr_smoke PRIVATE sots_addresses) add_test(NAME addr_smoke COMMAND addr_smoke) - foreach(_t mars_parse game_config game_data game_design game_sim mars_stream mars_text mars_vfs shim_trace game_effects game_events shim_budget shim_techfx shim_colony shim_movement shim_events shim_player_turn) + foreach(_t mars_parse game_config game_data game_design game_sim mars_stream mars_text mars_vfs shim_trace game_effects game_events shim_budget shim_techfx shim_colony shim_movement shim_events shim_player_turn shim_rng_ledger) if(EXISTS ${CMAKE_SOURCE_DIR}/tests/${_t}/CMakeLists.txt) add_subdirectory(tests/${_t}) endif() diff --git a/docs/Z-tail-rng.md b/docs/Z-tail-rng.md index dfe2acb..963433f 100644 --- a/docs/Z-tail-rng.md +++ b/docs/Z-tail-rng.md @@ -80,7 +80,7 @@ then the interesting question moves entirely to what the AI does), or by hundred per-system or per-fleet draw nobody has found). **P5 — the attributed subtotals do not sum to the bracket.** Predicted residual > 0, because combat -itself (`RunCombatRound` 0x007cbe80 / the combat server `FUN_007cfd00`) runs *between* +itself (`RunCombatRound` 0x007cbe80 / the combat server `0x007cfd00`) runs *between* `ProcessTurn` and the tail and is hooked by nobody. On a **quiet** turn, though, predicted residual = **0** exactly: the two drivers should account for every word between the two autosaves. *Falsified by:* a nonzero residual on a quiet turn. That is a draw site outside both drivers and @@ -113,7 +113,7 @@ unmodelled subtrees, not a prediction dressed as one. ## 4. What this cannot settle -* The combat resolver `FUN_007d5af0` (7499 B) stays unread. Its draw count is measured here as part +* The combat resolver `0x007d5af0` (7499 B) stays unread. Its draw count is measured here as part of the phase-6 subtotal, never modelled. * A turn with no combat cannot exercise phase 6 at all, so its subtotal on a quiet turn is 0 by construction and proves nothing about combat. diff --git a/include/generated/sots_addresses.h b/include/generated/sots_addresses.h index 13d4e5a..7e79072 100644 --- a/include/generated/sots_addresses.h +++ b/include/generated/sots_addresses.h @@ -1,5 +1,5 @@ // GENERATED — do not edit. Facts about Sword of the Stars.exe (GOG 1.8.1). -// Source: sots-re ghidra/addresses.json @ 612bfe6, generated 2026-09-08 by tools/gen_addresses.py +// Source: sots-re ghidra/addresses.json @ 729cfc0, generated 2026-09-08 by tools/gen_addresses.py // Runtime address = (uintptr_t)GetModuleHandle(NULL) + RVA (the exe is ASLR-relocated). #pragma once #include @@ -1351,6 +1351,18 @@ constexpr uint32_t ServerSystem_ComputeMaxIncome = 0x003521c0; constexpr uint32_t SNMAllCombatDone_layout = 0x00624758; // data const float 0.5f -- the progress-ratio threshold in ServerPlayer::ProcessTurn's `ResT != NULL && ResErrRoll != 0 && CONST < progressRatio` gate (events.md §3, window 0x008915ec-0x00891624). Read out of dumps/sots.exe: .rdata bytes at 0x00a2c788 are 00 00 00 3f (float 0.5); the following word 0x00a2c78c is float 100.0, so this is a float32, NOT the double an 8-byte read would suggest (that reads as 5.28e13). CONSEQUENCE, measured on the live game: ResErrRoll survives into ProcessResearch only while progress/cost <= 0.5 at the START of the turn, so the OnTechResearched draw (0x0088df20) can fire only when a single turn supplies more than half the target tech's remaining cost. See the board's `research_roll_pending save` row [verified] constexpr uint32_t ResearchRollProgressThreshold = 0x0062c788; +// thiscall void __thiscall Game::TurnCommands::Write(Mars::IStream* s) -- the writer for the `Player..TurnCommands_v5` custom-data block (CDT id -> one CD frame). 764 bytes, no loops of its own. Two halves. (1) A PROLOGUE of six flag-gated groups, each a WriteBool on a member followed, only when that bool is set, by the group's payload; write order is NOT offset order, which is why the offset-sorted layout view cannot be aligned to the wire: bool@0x0c gates f32@0x08 (research rate); bool@0x14 gates i32@0x10 (research target tech); bool@0x20 gates i32@0x18 + f32@0x1c (research boost spend + fraction); bool@0x2c gates bool@0x24 + i32@0x28; bool@0x3c gates f32@0x30,0x34,0x38; bool@0x6c gates a StreamableHelper frame on the member at 0x40. i32@0x04 (player id) is written first and unconditionally. (2) TWENTY-SEVEN std::list members at 0x70..0x1a8, stride 0x0c ({_Myhead, _Mysize, _Alval}), each passed to its own free-function writer as helper(stream, &list). Every one is written unconditionally, so an empty list still costs one zero int: 8 prologue items + 27 zero counts = the 35-item block every no-orders save carries [verified] +constexpr uint32_t TurnCommands_Write = 0x00442540; +// thiscall void __thiscall Game::TurnCommands::Read(Mars::IStream* s) -- the reader paired with TurnCommands_Write (0x00842540). 1543 bytes; not decompiled by lane Q, listed so the pair is on the record [mapped] +constexpr uint32_t TurnCommands_Read = 0x00492ae0; +// cdecl void __cdecl (Mars::IStream* s, std::list* moves) -- writer for TurnCommands member +0xc4, the FLEET MOVE list. Per element: WriteInt(fleetId) then a NESTED counted int vector -- n = (v._Mylast - v._Myfirst) >> 2 written with WriteInt, then n ints. So a fleet move on the wire is {fleetId, hopCount, hopCount x systemId}, NOT a fixed {fleetId, 1, destSystemId, 0} quadruple: a multi-hop route is longer. Observed single-hop in human-turn2-orders.sav {1456, 1, [128]} and zuul-turn15-orders.sav {688, 1, [432]} [verified] +constexpr uint32_t TurnCommands_WriteFleetMoveList = 0x0043e550; +// cdecl void __cdecl (Mars::IStream* s, std::list* orders) -- writer for TurnCommands member +0x88, the BUILD ORDER list. Per element four WriteInts taken in DESCENDING member order (node+0x14, +0x10, +0xc, +0x8), so the wire order is {ordinal, designId, systemId, w}. Observed in zuul-turn15-orders.sav (5 orders, designs 608/576, system 384) and zuul-turn17-orders2.sav (20 orders, ordinals 6..25) [verified] +constexpr uint32_t TurnCommands_WriteBuildOrderList = 0x00422870; +// cdecl void __cdecl (Mars::IStream* s, std::list* cmds) -- writer for TurnCommands member +0xa0, the PLANETARY-BUDGET list. Per element WriteInt(systemId) then a StreamableHelper frame (helper vftable 0x00a1f884), which is the only NAMED sub-frame anywhere in the block (SRs SRt SRsc SRtf SRi SRoh SRnr). Observed once, in zuul-turn17-orders2.sav: system 384 with SRsc = 1.0 [verified] +constexpr uint32_t TurnCommands_WriteSystemRatesList = 0x0042e3d0; +// cdecl void __cdecl (Mars::IStream* s, std::list* orders) -- writer for TurnCommands member +0xb8, the COLONIZE list. Per element two WriteInts (node+0x8 then node+0xc): {shipId, w}. Observed once, in zuul-turn17-orders2.sav: three ships 2512/2544/2624, each with w = 1 [verified] +constexpr uint32_t TurnCommands_WriteColonizeList = 0x00422960; // thiscall void (ServerPlayer* this, float dt) // RET 4. The per-player turn driver, called once per player from StrategyServer::ProcessTurn's player loop. THE dt ARGUMENT IS NEVER READ: the whole 1086-byte body contains zero [ebp+N] references (mechanical check over the full instruction decode), so a reimplementation may ignore it. Order: ComputeBudget -> Sav = SatAdd(Sav, net) -> record aid given -> ProcessSpecialProjects -> (ResT ? RollResearchAccident/ProcessResearch) -> research refund -> zero TRM/TRA/TRP -> RebAI decay -> timed-bonus sweep -> ResearchRollPending site -> EVENT_NO_RESEARCH -> PruneRaidTargets [verified] constexpr uint32_t ServerPlayer_ProcessTurn = 0x00491340; // site site in ServerPlayer::ProcessTurn: `if (this->ResT(+0x294) != 0) { if (!RollResearchAccident(&budget)) TechTree::ProcessResearch(this->TechTree(+0xf4), rng, &budget.researchAlloc, &overBudget); }`. Argument order read off the push order at 0x00891496-0x008914a5: pushes are (edx=&overBudget), (ecx=&allocVector), (eax=rng), so left-to-right the args are (RNG*, vector*, int*). The RNG is `*(ServerPlayer+8 - 4 + 0x16c)`. `overBudget` is a FRESH STACK LOCAL at [ebp-0x14], NOT Budget+0x64 [verified] @@ -1507,5 +1519,25 @@ constexpr uint32_t SVSOCrowDefenders_Write = 0x000f8c90; constexpr uint32_t SVSOMonitor_Write = 0x000fd810; // thiscall void (Game::SVSODerelict* this, Mars::IStream* s) // NDsn count then a loop of (DsnID, Dwght); NAsg count then a loop of (Eflt, Esys). Two fields per iteration in each, confirmed by the 8-byte element strides [verified] constexpr uint32_t SVSODerelict_Write = 0x000fc2b0; +// thiscall void (StrategyServer* this /*base S*/) // phase 11 of OnAllCombatDone_Tail, called at 0x007d9714 as `mov ecx,esi; call`. 1117 B, three loops. LOOP 1 (0x007ae07a..0x007ae1e2) walks the 0x30-stride Game::NodePath records in the vector at (*(S+0x154))+0x8/+0xc, re-reading _Myfirst/_Mylast every iteration, and per record: (1) NodePath::RemainingLife(r, S->Frame) 0x006e2130, `test eax,eax; jg` -> not expired, NEXT RECORD, NO DRAW; (2) THE DRAW, `mov ecx,[esi+0x16c]; fld [0x009e2ea0] /*0.5f*/; call 0x008e6dd0` = Mars::RNG::Chance(0.5f), exactly one MT word; (3) `test al,al; je` -> roll failed, next record; (4) the 0x20000-fleet scan over S->Fleets (S+0x64/+0x68) calling 0x00703500(fleet,0x20000,0) then 0x0078c360(fleet,npid) and dropping the record when that returns 3; (5) push_back npid into a scratch vector. LOOP 2 collapses each collected line via 0x007a92e0(690 B) then 0x007a4700(2244 B); LOOP 3 posts the decay-stage events through NodePath::DecayStage 0x006e21b0. THE ONLY RNG SITE IN THE WHOLE 1117 BYTES: direct-call sweep to depth 5 over 140 functions from 0x007ae010 against {NextFloat 0x0047d830, NextInt 0x004271c0, Chance 0x008e6dd0, Twist 0x00426e00, Seed 0x0049fdf0} yields exactly one hit, 0x007ae010 -> 0x008e6dd0. Neither downstream function draws (138 and 49 functions reached, zero hits) -- caveat: direct calls only, their subtrees contain unresolved indirect sites [verified] +constexpr uint32_t StrategyServer_NodeLineDecay = 0x003ae010; +// site site, and a CORRECTION to findings/control-flow/combat-done-tail.md §3, which says "the roll is skipped for a line if any fleet with flag 0x20000 is targeting it". IT IS NOT: the fleet scan begins HERE, at 0x007ae0b2, which is 0x1d bytes AFTER the Chance(0.5f) call at 0x007ae0a5 and is reached only when the roll SUCCEEDED (`test al,al; je 0x007ae1e2` at 0x007ae0aa). The scan therefore cannot change the draw count -- it suppresses only the collapse (the 0x007a92e0 / 0x007a4700 pair), never the draw. The straight-line order in loop 1 is: expiry test -> DRAW -> roll gate -> fleet gate -> collect. Lane K's headline claim, one NextFloat per expired node line per turn, survives intact and is now pinned to a concrete expiry formula (NodePath_RemainingLife) [verified] +constexpr uint32_t StrategyServer_NodeLineDecay_FleetSkipIsPostDraw = 0x003ae0b2; +// thiscall int (NodePath* this, int turn) // RET 4, whole 122-byte body read as instructions. THE EXPIRY PREDICATE for node-line decay. `if (npt(+0x4) == 0) return INT_MAX; if (npdtn(+0x1c) == INT_MAX) return INT_MAX; aged = (npctm(+0x14) >= 0 && turn >= npctm) ? turn - npctm : 0; wear = (npdtf(+0x20) != INT_MAX && npdtf > 0) ? nptf(+0x24) / npdtf : 0; /* SIGNED idiv, nptf is never sign-checked */ rem = npdtn - wear - aged; return rem > 0 ? rem : 0;`. Writes nothing -- the lifetime is DERIVED from a creation stamp and a traffic accumulator, never ticked, so there is no decrement-ordering question. Two never-expire escape hatches (npt == 0, npdtn == INT_MAX). A line is expired exactly when this returns 0 [verified] +constexpr uint32_t NodePath_RemainingLife = 0x002e2130; +// thiscall int (NodePath* this, int turn) // 52 B. Wraps NodePath::RemainingLife and buckets it: <=2 -> 0, <=5 -> 1, <=10 -> 2, else 3. Called TWICE per record by loop 3 of node-line decay (once with turn-1, once with turn) to detect a stage transition and post the two decay-stage events. Draw-free -- but note that instrumenting node-line decay by counting RemainingLife CALLS rather than Chance calls over-counts badly because of this wrapper [verified] +constexpr uint32_t NodePath_DecayStage = 0x002e21b0; +// thiscall void (StrategyServer* this /*base S*/) // 2945 B. Called TWICE per turn: StrategyServer::ProcessTurn phase 7 (0x007dc93a) and OnAllCombatDone_Tail phase 10 (0x007d970b). Body unread; hooked by lane Z only to measure whether it advances the strategic generator, because a draw inside it would be double-counted by anyone who modelled it as running once [mapped] +constexpr uint32_t StrategyServer_ProcessNodeSpaceTravel = 0x003a0e20; +// offset Game::ServerNodeGraph* -- the node-line graph. Stored frame (S+4), so it is S+0x154 in the frame OnAllCombatDone_Tail and ProcessTurn receive; node-line decay reads it as `mov eax,[esi+0x154]` at 0x007ae03a with esi = S. Dereferenced without a null check by the original [verified] +constexpr uint32_t StrategyServer_off_NodeGraph = 0x00000150; +// offset std::vector (_Myfirst @+0x8, _Mylast @+0xc). Element stride 0x30, confirmed four ways: the reciprocal 0x2aaaaaab / sar 3 at four sites in node-line decay, `add [ebp-0x14],0x30` in its loop 1, `add [ebp-0x18],0x30` in its loop 3, and `add eax,0x30` in ServerNodeGraph::FindPathById 0x006e23d0. Loop 1 of node-line decay re-reads both words every iteration but writes neither, so an entry snapshot is a valid prediction basis [verified] +constexpr uint32_t ServerNodeGraph_off_Paths = 0x00000008; +// thiscall NodePath* (ServerNodeGraph* this, int npid) // 59 B, linear scan of the 0x30-stride paths vector comparing npid(+0x8). Node-line decay collects npid HANDLES rather than record pointers in loop 1 and re-resolves them here in loop 2, which is how the original hedges against the collapse functions mutating the vector under it [verified] +constexpr uint32_t ServerNodeGraph_FindPathById = 0x002e23d0; +// thiscall bool (StarFleet* this, uint maskA, uint maskB) // RET 8, a 29-byte thunk onto 0x00702d70(this, maskA, maskB, out = 0). Returns count > 0 where count is the number of ships in the fleet's NShips vector (+0xa4/+0xa8) passing the two-mask ship-flag predicate 0x00814da0, itself gated on (fleet->+0xb8 & maskA) == maskA. Called from node-line decay's post-draw fleet scan with maskA = 0x20000 [verified] +constexpr uint32_t StarFleet_HasFlagShips = 0x00303500; +// cdecl int (StarFleet* fleet, int npid) // 234 B. Returns 3 exactly when the fleet's FRONT waypoint (the deque at fleet+0xc4, element +0x10) names this npid -- i.e. the fleet is currently riding this line; 0/1/2/4 otherwise. Node-line decay drops a rolled line when any 0x20000-flagged fleet returns 3 for it [verified] +constexpr uint32_t StarFleet_PathRelation = 0x0038c360; } // namespace sots::addr diff --git a/src/shim/fpu_force.cpp b/src/shim/fpu_force.cpp index 762c91f..1846250 100644 --- a/src/shim/fpu_force.cpp +++ b/src/shim/fpu_force.cpp @@ -35,6 +35,10 @@ void logf(const char* fmt, ...) { bool g_forceEnabled = false; unsigned g_forceValue = 0; bool g_sampleTicks = true; +// The StrategyServer::ProcessTurn sampler owns that address, and MinHook allows one hook per +// target. Lane Z's RNG-ledger hook needs the same function, so this releases it on request. +// Default `on` -- no existing run changes behaviour. +bool g_sampleTurn = true; // ---- state ----------------------------------------------------------------------------------- @@ -245,6 +249,18 @@ bool apply_config(const char* key, const char* value, std::string* err) { g_forceEnabled = true; return true; } + if (std::strcmp(key, "fpu.sample_turn") == 0) { + if (std::strcmp(value, "on") == 0) { + g_sampleTurn = true; + return true; + } + if (std::strcmp(value, "off") == 0) { + g_sampleTurn = false; + return true; + } + if (err) *err = "expected on or off"; + return true; + } if (std::strcmp(key, "fpu.sample_ticks") == 0) { if (std::strcmp(value, "on") == 0) { g_sampleTicks = true; @@ -276,6 +292,8 @@ void install(std::uintptr_t exeBase, void (*log)(const char*)) { logf("fpu: module init, entry cw=0x%04x %s/%s; force=%s value=0x%04x sample_ticks=%s", entry, pc_name(entry), rc_name(entry), g_forceEnabled ? "on" : "off", g_forceValue, g_sampleTicks ? "on" : "off"); + logf("fpu: sample_turn=%s (off releases StrategyServer::ProcessTurn for another hook)", + g_sampleTurn ? "on" : "off"); struct Site { const char* name; @@ -290,7 +308,7 @@ void install(std::uintptr_t exeBase, void (*log)(const char*)) { {"StrategyServer::BeginProcessTurn", sots::addr::StrategyServer_BeginProcessTurn, reinterpret_cast(&FpuBeginProcessTurnDetour), &g_origFpuBeginProcessTurn, true}, {"StrategyServer::ProcessTurn", sots::addr::StrategyServer_ProcessTurn, - reinterpret_cast(&FpuProcessTurnDetour), &g_origFpuProcessTurn, true}, + reinterpret_cast(&FpuProcessTurnDetour), &g_origFpuProcessTurn, g_sampleTurn}, {"DemoApp::OnTick", sots::addr::DemoApp_OnTick, reinterpret_cast(&FpuOnTickDetour), &g_origFpuOnTick, g_sampleTicks}, }; diff --git a/src/shim/hooks/rng_ledger.cpp b/src/shim/hooks/rng_ledger.cpp new file mode 100644 index 0000000..4c8dc88 --- /dev/null +++ b/src/shim/hooks/rng_ledger.cpp @@ -0,0 +1,146 @@ +#include "shim/hooks/rng_ledger.h" + +#include + +#include "generated/sots_addresses.h" +#include "mars/rng/mt19937.h" + +namespace shim::hooks { + +namespace { + +namespace A = sots::addr; + +constexpr std::size_t kBlockBytes = std::size_t(RngLedger::kWordsPerBlock) * 4; + +long long words_of(int block, int left) { + return static_cast(block) * RngLedger::kWordsPerBlock + + (RngLedger::kWordsPerBlock - left); +} + +// One block transform. MT19937::twist() is private, but `next_u32()` on an exhausted state +// twists and then hands out a word, so loading `left = 0` and drawing once leaves `state()` +// holding exactly the next block. +void next_block(const std::uint32_t* in, std::uint32_t* out) { + mars::rng::MT19937 g; + g.load_state(in, 0); + (void)g.next_u32(); + std::memcpy(out, g.state(), kBlockBytes); +} + +} // namespace + +std::uint64_t hash_bytes(const void* p, std::size_t n) { + const std::uint8_t* b = static_cast(p); + std::uint64_t h = 1469598103934665603ull; + for (std::size_t i = 0; i < n; ++i) { + h ^= b[i]; + h *= 1099511628211ull; + } + return h; +} + +RngLedger& RngLedger::instance() { + static RngLedger g; + return g; +} + +RngPos RngLedger::observe(const std::uint32_t* mt, int left) { + RngPos out; + out.left = left; + if (!mt || left < 0 || left > kWordsPerBlock) return out; + out.hash = hash_bytes(mt, kBlockBytes); + + LockGuard g(mu_); + + if (!anchored_) { + anchored_ = true; + std::memcpy(frontier_, mt, kBlockBytes); + frontier_block_ = 0; + memo_[out.hash] = 0; + out.known = true; + out.block = 0; + out.words = words_of(0, left); + return out; + } + + const std::unordered_map::const_iterator it = memo_.find(out.hash); + if (it != memo_.end()) { + out.known = true; + out.block = it->second; + out.words = words_of(out.block, left); + // Keep the frontier at the furthest block actually seen, so the next forward walk is + // as short as the generator's own progress and no shorter. + if (out.block > frontier_block_) { + std::memcpy(frontier_, mt, kBlockBytes); + frontier_block_ = out.block; + } + return out; + } + if (exhausted_ || unknown_.count(out.hash) != 0) { + ++misses_; + return out; + } + + std::uint32_t cur[kWordsPerBlock]; + std::uint32_t nxt[kWordsPerBlock]; + std::memcpy(cur, frontier_, kBlockBytes); + int index = frontier_block_; + for (int i = 0; i < kMaxTwistsPerLookup; ++i) { + next_block(cur, nxt); + ++index; + const std::uint64_t h = hash_bytes(nxt, kBlockBytes); + memo_[h] = index; + if (h == out.hash) { + std::memcpy(frontier_, nxt, kBlockBytes); + frontier_block_ = index; + out.known = true; + out.block = index; + out.words = words_of(index, left); + return out; + } + std::memcpy(cur, nxt, kBlockBytes); + } + unknown_.insert(out.hash); + ++misses_; + if (misses_ >= kMaxMisses) exhausted_ = true; + return out; +} + +RngPos RngLedger::observe_object(const void* obj, std::size_t size) { + RngPos out; + if (!obj || size < A::RNG_size) return out; + const char* p = static_cast(obj); + std::uint32_t mt[kWordsPerBlock]; + std::memcpy(mt, p + A::RNG_off_State, kBlockBytes); + std::int32_t left = 0; + std::memcpy(&left, p + A::RNG_off_Left, 4); + return observe(mt, static_cast(left)); +} + +long long RngLedger::blocks_indexed() const { + LockGuard g(mu_); + return static_cast(memo_.size()); +} + +long long RngLedger::misses() const { + LockGuard g(mu_); + return misses_; +} + +bool RngLedger::anchored() const { + LockGuard g(mu_); + return anchored_; +} + +void RngLedger::reset() { + LockGuard g(mu_); + anchored_ = false; + exhausted_ = false; + frontier_block_ = 0; + memo_.clear(); + unknown_.clear(); + misses_ = 0; +} + +} // namespace shim::hooks diff --git a/src/shim/hooks/rng_ledger.h b/src/shim/hooks/rng_ledger.h new file mode 100644 index 0000000..1f109b4 --- /dev/null +++ b/src/shim/hooks/rng_ledger.h @@ -0,0 +1,95 @@ +// RngLedger — absolute word positions for the strategic Mersenne Twister, recovered from +// state alone. +// +// WHY THIS EXISTS. The milestone is a standalone that runs one strategic turn and writes an +// autosave that byte-matches the original's. The generator's state is part of that file, so +// every word the turn consumes has to be accounted for. Counting *draws* by hooking +// Mars::RNG::NextFloat / NextInt would undercount: NextInt rejection-samples, so one call can +// spend several words, and Chance() sometimes spends none at all. The unit that matters for +// save reproduction is the word, and the state says how many were spent whether or not anyone +// hooked the function that spent them. +// +// HOW. MT19937's block transform is a pure function, so the blocks a generator visits form a +// forward-only chain from wherever it starts. The ledger indexes that chain: the first block it +// ever sees is index 0, and any later block is found by twisting forward from the frontier. A +// state is then positioned exactly: +// +// words(block, left) = block * 624 + (624 - left) +// +// and the number of words consumed between any two observations is the difference — correct +// across twists, across rejection loops, and across code nobody hooked. +// +// ORDERING. Observations may arrive out of chronological order: Hook<> takes its `before` +// snapshot at entry but renders it (calling the region's `describe`) only after the original +// returns, so a nested call's states are rendered first. Blocks are therefore memoised by hash +// and resolved from the memo regardless of arrival order; a caller that observes at entry (from +// `describe_args`) guarantees the entry block is already indexed by the time its `before` +// snapshot is rendered. Walking *backwards* is impossible, which is exactly why entry +// observation is not optional. +// +// LIMITS, stated because they are the failure modes: +// * A block more than kMaxTwistsPerLookup twists ahead of the frontier is reported UNKNOWN +// rather than guessed. So is every state seen before the anchor. +// * Two generator instances share one ledger only by accident of the anchor: the second one's +// blocks are not on the first one's chain, so they read UNKNOWN. That is the intended +// signal, not a defect — one strategic generator is an assumption this measures. +// * Position is relative to the anchor. Only differences are meaningful. +#pragma once + +#include +#include +#include +#include + +#include "shim/trace/platform.h" + +namespace shim::hooks { + +struct RngPos { + bool known = false; // false: the block is not on the indexed chain + long long words = 0; // absolute word position from the anchor (valid when `known`) + int block = 0; // block index from the anchor (valid when `known`) + int left = 0; // words still unread in the block, as read from the state + std::uint64_t hash = 0; // FNV-1a 64 of the 2496-byte block +}; + +class RngLedger { +public: + static constexpr int kWordsPerBlock = 624; + // 4096 twists is 2.5M words: far more than a turn spends, small enough that a miss costs + // milliseconds. A gap larger than this is a finding, not something to search harder for. + static constexpr int kMaxTwistsPerLookup = 4096; + // After this many unresolvable blocks the ledger stops walking. A second generator would + // otherwise make every observation pay the full search. + static constexpr int kMaxMisses = 8; + + // The shim uses `instance()`; tests construct their own so one case cannot poison another. + RngLedger() = default; + static RngLedger& instance(); + + // `mt` is the 624-word block, `left` the counter. Out-of-range `left` returns unknown. + RngPos observe(const std::uint32_t* mt, int left); + // The same over a live Mars::RNG object (vptr, mt[624] at +4, next at +0x9c4, left at + // +0x9c8). `size` must cover the object; a short region returns unknown. + RngPos observe_object(const void* obj, std::size_t size); + + long long blocks_indexed() const; + long long misses() const; + bool anchored() const; + void reset(); + +private: + mutable Mutex mu_; + bool anchored_ = false; + bool exhausted_ = false; + std::uint32_t frontier_[kWordsPerBlock] = {}; + int frontier_block_ = 0; + std::unordered_map memo_; + std::unordered_set unknown_; + long long misses_ = 0; +}; + +// FNV-1a 64 over `n` bytes. Exposed so tests can pin the block identity. +std::uint64_t hash_bytes(const void* p, std::size_t n); + +} // namespace shim::hooks diff --git a/src/shim/hooks/tail_rng.cpp b/src/shim/hooks/tail_rng.cpp new file mode 100644 index 0000000..4184e46 --- /dev/null +++ b/src/shim/hooks/tail_rng.cpp @@ -0,0 +1,664 @@ +#include "shim/hooks/tail_rng.h" + +#include +#include +#include +#include + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include "generated/sots_addresses.h" +#include "mars/rng/mt19937.h" +#include "shim/hooks/rng_ledger.h" + +namespace shim::hooks { + +using trace::Tv; +namespace tv = trace::tv; + +namespace { + +namespace A = sots::addr; + +constexpr std::size_t kRngSize = A::RNG_size; +constexpr int kMtWords = mars::rng::MT19937::N; + +// The `S` frame. Every StrategyServer_off_* in the generated header is an S+4 offset except +// StrategyServer_off_RNG, which is already S-relative. Adding 4 is therefore correct for every +// use below EXCEPT the generator, and each site says which it is using. +constexpr std::size_t kSFrame = 4; + +// Game::NodePath, stride 0x30, in the vector at ServerNodeGraph+0x8/+0xc. Field offsets and the +// expiry formula are an instruction read of NodePath::RemainingLife 0x006e2130 (whole 122-byte +// body) plus the loop at 0x007ae07a..0x007ae0af. +constexpr std::size_t kServerOffNodeGraph = kSFrame + A::StrategyServer_off_NodeGraph; // S+0x154 +constexpr std::size_t kGraphOffPaths = A::ServerNodeGraph_off_Paths; +constexpr std::size_t kNodePathStride = 0x30; +constexpr std::size_t kNpType = 0x04; // npt -- 0 means the line never expires +constexpr std::size_t kNpCreated = 0x14; // npctm -- creation turn +constexpr std::size_t kNpLife = 0x1c; // npdtn -- lifetime budget; INT_MAX means immortal +constexpr std::size_t kNpTrafficDiv = 0x20; // npdtf +constexpr std::size_t kNpTraffic = 0x24; // nptf +// A sane bound on the node-line count; a wilder number means we are reading the wrong object. +constexpr std::size_t kMaxNodePaths = 65536; + +struct Env { + std::uintptr_t exe_base = 0; + void (*log_line)(const char*) = nullptr; +}; +Env g_env; + +void logf(const char* fmt, ...) { + if (!g_env.log_line) return; + char line[512]; + va_list ap; + va_start(ap, fmt); + std::vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + g_env.log_line(line); +} + +bool readable(const void* p, std::size_t n) { + if (!p) return false; + if (n == 0) return true; +#if defined(_WIN32) + const char* c = static_cast(p); + const char* const end = c + n; + while (c < end) { + MEMORY_BASIC_INFORMATION mbi; + if (!VirtualQuery(c, &mbi, sizeof mbi)) return false; + if (mbi.State != MEM_COMMIT) return false; + if (mbi.Protect & (PAGE_NOACCESS | PAGE_GUARD)) return false; + const DWORD ok = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ | + PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; + if (!(mbi.Protect & ok)) return false; + c = static_cast(mbi.BaseAddress) + mbi.RegionSize; + } + return true; +#else + return true; +#endif +} + +template +T peek(const void* base, std::size_t off) { + T v{}; + std::memcpy(&v, static_cast(base) + off, sizeof v); + return v; +} +void* ptr_at(const void* base, std::size_t off) { return peek(base, off); } + +// ---- the generator --------------------------------------------------------------------------- + +// The last StrategyServer any of these hooks saw, so the Autosave markers -- whose `this` is a +// StrategyHost, not the server -- can reach the generator. Deliberately a cache rather than a +// derivation: `StrategyHost::Autosave`'s `this` is the global at 0x00b29f98 and it has never +// been proved to be the same object whose +0x54 holds the StrategyServer. The Autosave hook +// records BOTH the cached pointer and the +0x54 candidate so a run says whether they agree. +void* g_server = nullptr; + +void* rng_of_server(void* self) { + if (!readable(self, A::StrategyServer_off_RNG + 4)) return nullptr; + void* r = ptr_at(self, A::StrategyServer_off_RNG); // S frame already; no +4 + return readable(r, kRngSize) ? r : nullptr; +} + +std::int32_t turn_of(void* self) { + if (!readable(self, kSFrame + A::StrategyServer_off_ModCount + 4)) return -1; + return peek(self, kSFrame + A::StrategyServer_off_ModCount); +} +std::int32_t phase_counter_of(void* self) { + if (!readable(self, kSFrame + A::StrategyServer_off_PhaseCounter + 4)) return -1; + return peek(self, kSFrame + A::StrategyServer_off_PhaseCounter); +} +std::int32_t encounter_count_of(void* self) { + const std::size_t off = kSFrame + A::StrategyServer_off_TeamRecords; // S+0x1e8 + if (!readable(self, off + 8)) return -1; + const char* first = static_cast(ptr_at(self, off)); + const char* last = static_cast(ptr_at(self, off + 4)); + if (!first || !last || last < first) return -1; + return static_cast((last - first) / 0x74); +} +std::int32_t player_count_of(void* self) { + const std::size_t off = kSFrame + A::StrategyServer_off_Players; // S+0x54 + if (!readable(self, off + 8)) return -1; + const char* first = static_cast(ptr_at(self, off)); + const char* last = static_cast(ptr_at(self, off + 4)); + if (!first || !last || last < first) return -1; + return static_cast((last - first) / 4); +} + +// The record every hook here emits as arguments: where the generator is when the call starts. +// Observing HERE, at entry, is what makes the nested `before` snapshots resolvable -- Hook<> +// renders them after the original returns, by which time the ledger's frontier has moved on and +// walking backwards is impossible. See rng_ledger.h. +struct RngEntry { + void* rng = nullptr; + RngPos pos; + bool have = false; +}; + +RngEntry observe_entry(void* self) { + RngEntry e; + e.rng = rng_of_server(self); + if (!e.rng) return e; + e.pos = RngLedger::instance().observe_object(e.rng, kRngSize); + e.have = true; + return e; +} + +void push_rng_args(std::vector& out, const RngEntry& e) { + out.push_back(tv::ptr(e.rng).named("rng")); + out.push_back((e.have && e.pos.known ? tv::i64(e.pos.words) : tv::null()).named("rng_words_in")); + out.push_back((e.have ? tv::i32(e.pos.left) : tv::null()).named("rng_left_in")); +} + +void push_server_args(std::vector& out, void* self) { + out.push_back(tv::i32(turn_of(self)).named("turn")); // S+0xc ModCount + out.push_back(tv::i32(phase_counter_of(self)).named("phase_counter")); // S+0x8, lane K §7.1 + out.push_back(tv::i32(player_count_of(self)).named("players")); + out.push_back(tv::i32(encounter_count_of(self)).named("encounters")); +} + +// The region describe. Called once on the `before` snapshot and once on the `after` snapshot, +// both after the original returns; `words` is the ledger position, so the difference between +// the two is exactly the number of words this call consumed. +Tv describe_rng(const void* p, std::size_t size, unsigned) { + const RngPos pos = RngLedger::instance().observe_object(p, size); + Tv s = tv::struct_(); + s.add("left", tv::i32(pos.left)); + s.add("index", tv::i32(RngLedger::kWordsPerBlock - pos.left)); + if (pos.known) { + s.add("block", tv::i32(pos.block)); + s.add("words", tv::i64(pos.words)); + } else { + // Not on the indexed chain. Said plainly rather than guessed: this is the signal that a + // second generator exists, or that the gap exceeded the ledger's search bound. + s.add("block", tv::null()); + s.add("words", tv::null()); + } + s.add("block_hash", tv::u64(pos.hash)); + return s; +} + +void push_rng_region(std::vector& out, void* rng) { + if (!rng || !readable(rng, kRngSize)) return; + trace::Region r; + r.name = "rng"; + r.ptr = rng; + r.size = kRngSize; + r.describe = &describe_rng; + out.push_back(r); +} + +// ---- the node-line expiry model --------------------------------------------------------------- + +// NodePath::RemainingLife 0x006e2130, whole body read as instructions. Two escape hatches return +// INT_MAX (never expires); otherwise `npdtn - nptf/npdtf - (turn - npctm)`, clamped at 0 by the +// callee. The division is a signed `idiv` and `nptf` is never sign-checked, so this reproduces +// signed truncation deliberately. +std::int32_t node_path_remaining_life(const void* rec, std::int32_t turn) { + if (peek(rec, kNpType) == 0) return 0x7fffffff; + const std::int32_t life = peek(rec, kNpLife); + if (life == 0x7fffffff) return 0x7fffffff; + + std::int32_t aged = 0; + const std::int32_t created = peek(rec, kNpCreated); + if (created >= 0 && turn >= created) aged = turn - created; + + std::int32_t wear = 0; + const std::int32_t div = peek(rec, kNpTrafficDiv); + if (div != 0x7fffffff && div > 0) wear = peek(rec, kNpTraffic) / div; + + const std::int32_t rem = life - wear - aged; + return rem > 0 ? rem : 0; +} + +// Words node-line decay 0x007ae010 will consume, predicted from the state at entry. The gate on the draw is +// the expiry test and nothing else: the `Chance` result and the "a fleet with flag 0x20000 is +// riding this line" scan both run AFTER the draw at 0x007ae095 and gate only the collapse. +// (That corrects combat-done-tail.md §3, which reads as if the fleet check suppressed the roll.) +// Returns -1 when the state could not be read, which is reported as unknown rather than as 0. +std::int32_t predicted_node_line_words(void* self, std::int32_t* out_paths) { + if (out_paths) *out_paths = -1; + if (!readable(self, kServerOffNodeGraph + 4)) return -1; + const char* graph = static_cast(ptr_at(self, kServerOffNodeGraph)); + if (!readable(graph, kGraphOffPaths + 8)) return -1; + const char* first = static_cast(ptr_at(graph, kGraphOffPaths)); + const char* last = static_cast(ptr_at(graph, kGraphOffPaths + 4)); + if (!first || !last || last < first) return -1; + const std::size_t span = static_cast(last - first); + if (span % kNodePathStride != 0) return -1; + const std::size_t n = span / kNodePathStride; + if (n > kMaxNodePaths || !readable(first, span)) return -1; + if (out_paths) *out_paths = static_cast(n); + + const std::int32_t turn = turn_of(self); + std::int32_t expired = 0; + for (std::size_t i = 0; i < n; ++i) + if (node_path_remaining_life(first + i * kNodePathStride, turn) <= 0) ++expired; + return expired; +} + +// Advance a scratch copy of the RNG object by `words` words, the way the original would. +void advance_scratch_rng(void* scratch, std::uintptr_t live_base, int words) { + if (!scratch || words < 0) return; + char* dst = static_cast(scratch); + std::uint32_t mt[kMtWords]; + std::memcpy(mt, dst + A::RNG_off_State, sizeof mt); + std::int32_t left = 0; + std::memcpy(&left, dst + A::RNG_off_Left, 4); + mars::rng::MT19937 g; + g.load_state(mt, static_cast(left)); + for (int i = 0; i < words; ++i) (void)g.next_u32(); + std::memcpy(dst + A::RNG_off_State, g.state(), sizeof mt); + const std::int32_t new_left = static_cast(g.left()); + std::memcpy(dst + A::RNG_off_Left, &new_left, 4); + // `next` is an absolute cursor into the live object's own state block, so it is rebuilt + // against the live base rather than the scratch's. + void* next = reinterpret_cast(live_base + A::RNG_off_State + + static_cast(kMtWords - new_left) * 4); + std::memcpy(dst + A::RNG_off_Next, &next, sizeof next); +} + +// ---- per-hook state --------------------------------------------------------------------------- +// +// Turn processing is single-threaded and every hook reads its own slot between describe_args and +// regions/rebind/ours in the same call, so a plain global per hook is safe and matches the +// pattern the other hook TUs use. + +struct CallState { + RngEntry entry; +}; +CallState g_autosave, g_process_turn, g_tail, g_apply, g_nodespace; + +struct NodeLineState { + RngEntry entry; + std::int32_t predicted = -1; + std::int32_t paths = -1; + void* s_rng = nullptr; + bool compare = false; +}; +NodeLineState g_nld; + +void refuse_replace(const char* who) { + static bool warned = false; + if (warned) return; + warned = true; + logf("tail-rng: replace mode is not supported for %s; falling back to the original", who); +} + +} // namespace + +void init_tail_rng(std::uintptr_t exe_base, void (*log_line)(const char* line)) { + g_env.exe_base = exe_base; + g_env.log_line = log_line; +} + +void tail_rng_common_coverage(trace::Coverage& c) { + c.unmodelled("everything the original writes except the strategic generator", + trace::Risk::High, + "this hook family measures ONE thing -- how many words the generator advances " + "and where. It declares no region over game state and makes no claim about it. " + "A clean run here says the RNG accounting is right and says nothing whatever " + "about whether the turn was computed correctly", + "region:rng is the only check; the turn's own correctness is B1/B3/B4's job"); + c.unmodelled("the ledger reports WORDS, not draws", + trace::Risk::Low, + "a NextInt that rejects three times is four words and one call. Words are the " + "unit that decides whether a save reproduces; they are the wrong unit for " + "counting decisions, and nothing here should be read as a draw count", + "region:rng carries left/block/words, never a call count"); + c.unmodelled("the ledger is deliberately blind to WHICH primitive spent a word", + trace::Risk::Low, + "that is the design, and it is why this instrument was preferred to hooking " + "the primitives: the image has FOUR draw entry points (NextFloat 0x0047d830, " + "NextInt 0x004271c0, Chance 0x008e6dd0 and NextUInt 0x004f7670, the last of " + "which appears in no previous lane's primitive set) plus inlined draws in at " + "least twelve functions, two of them reachable from the turn roots. A " + "primitive-counting hook would have silently undercounted every one of those", + "region:rng reads the state, so an inlined draw is as visible as a called one"); + c.unmodelled("a generator position the ledger cannot place reads `words: null`", + trace::Risk::Medium, + "a block more than 4096 twists ahead of the frontier, or any state behind the " + "anchor, is reported unknown rather than guessed. A null in a ledger field is a " + "measurement failure and must not be read as zero", + "region:rng emits null explicitly; tracecmp shows it as a value, not a gap"); +} + +// ---- Game::StrategyHost::Autosave -------------------------------------------------------------- + +void StrategyHostAutosaveHook::describe_args(std::vector& out, void* self, void* name_out, + bool end_turn) { + // `this` is the global at 0x00b29f98, hardcoded by both call sites -- not an object handed + // in, and not proved to be the same StrategyHost whose +0x54 holds the StrategyServer. So + // the generator is reached through the pointer the turn drivers cached, and the +0x54 + // candidate is recorded beside it so one run settles whether the two are the same object. + void* from_this = readable(self, 0x58) ? ptr_at(self, 0x54) : nullptr; + g_autosave.entry = observe_entry(g_server); + + out.push_back(tv::ptr(self).named("host")); + out.push_back(tv::ptr(name_out).named("out_name")); + out.push_back(tv::boolean(end_turn).named("end_turn")); + out.push_back(tv::ptr(g_server).named("server_cached")); + out.push_back(tv::ptr(from_this).named("host_plus_0x54")); + out.push_back(tv::boolean(from_this != nullptr && from_this == g_server).named("server_agrees")); + push_rng_args(out, g_autosave.entry); +} + +Tv StrategyHostAutosaveHook::describe_ret(void* r) { return tv::ptr(r); } + +void StrategyHostAutosaveHook::regions(std::vector& out, void*, void*, bool) { + push_rng_region(out, g_autosave.entry.rng); +} + +StrategyHostAutosaveHook::Args StrategyHostAutosaveHook::rebind(trace::Scratch&, void* self, + void* name_out, bool end_turn) { + return Args(self, name_out, end_turn); +} + +void* StrategyHostAutosaveHook::ours(void* self, void* name_out, bool end_turn) { + using H = trace::Hook; + if (H::mode == trace::Mode::Replace) { + refuse_replace("StrategyHost::Autosave"); + if (H::original) return H::original(self, name_out, end_turn); + } + // Compare mode: ours writes nothing. The autosave is a marker, not a model -- the useful + // output is the pair of ledger positions in `side.rng`, and a model that "predicted" the + // generator does not move here would be a check of nothing. The NRV slot is echoed back so + // the return value is the one the caller expects. + return name_out; +} + +void StrategyHostAutosaveHook::coverage(trace::Coverage& c) { + tail_rng_common_coverage(c); + c.unmodelled("the whole save write: four path buffers, the ENDTURN pair removal, the " + "backup rotation, the per-player connection detach/reattach, and " + "SaveGame_WriteFile 0x00877070 itself", + trace::Risk::Low, + "this hook exists to timestamp the generator at the two moments the two save " + "files are written. It is a marker and models nothing", + "region:rng only"); + c.unmodelled("the generator is reached through a CACHED StrategyServer pointer, not from " + "this call's own arguments", + trace::Risk::Medium, + "`this` is the global at 0x00b29f98, hardcoded by both call sites, and no " + "argument here names the server. On the first pre-turn autosave after a load " + "no turn driver has run yet, so the cache is empty and that record carries no " + "ledger position -- the FIRST BRACKET OF A SESSION IS INCOMPLETE BY " + "CONSTRUCTION and must not be read as a zero-cost turn", + "arg:server_cached / host_plus_0x54 / server_agrees say which pointer was used " + "and whether the +0x54 candidate is the same object"); +} + +// ---- Game::StrategyServer::ProcessTurn --------------------------------------------------------- + +void StrategyServerProcessTurnHook::describe_args(std::vector& out, void* self, float dt) { + g_server = self; + g_process_turn.entry = observe_entry(self); + out.push_back(tv::ptr(self).named("server")); + out.push_back(tv::f32(dt).named("dt")); + push_server_args(out, self); + push_rng_args(out, g_process_turn.entry); +} + +void StrategyServerProcessTurnHook::regions(std::vector& out, void*, float) { + push_rng_region(out, g_process_turn.entry.rng); +} + +StrategyServerProcessTurnHook::Args StrategyServerProcessTurnHook::rebind(trace::Scratch&, + void* self, float dt) { + return Args(self, dt); +} + +void StrategyServerProcessTurnHook::ours(void* self, float dt) { + using H = trace::Hook; + if (H::mode == trace::Mode::Replace) { + refuse_replace("StrategyServer::ProcessTurn"); + if (H::original) H::original(self, dt); + } +} + +void StrategyServerProcessTurnHook::coverage(trace::Coverage& c) { + tail_rng_common_coverage(c); + c.unmodelled("no model of the turn's RNG cost: 32 phases, each of which may draw", + trace::Risk::High, + "ProcessResearch's completion roll, RollResearchAccident's NextInt(100), the " + "ResearchRollPending roll (one word, or two on the plague path), and whatever " + "ProcessStations / ProcessSurrenders / ProcessMissions / ProcessSpecialProjects " + "spend -- none of which has ever been measured. `ours` predicts nothing and the " + "record reports the measurement", + "region:rng measures the total; the per-phase split is not resolved here"); + c.unmodelled("this hook takes the address the fpu module also wants to sample", + trace::Risk::Low, + "MinHook allows one hook per target. `fpu.sample_turn=off` releases " + "StrategyServer::ProcessTurn so this hook can install; with it on, this hook " + "fails to install and the trace is missing half the ledger", + "shim.log records the MH_CreateHook status for both"); +} + +// ---- Game::StrategyServer::OnAllCombatDone_Tail ------------------------------------------------- + +void OnAllCombatDoneTailHook::describe_args(std::vector& out, void* self, void* results) { + g_server = self; + g_tail.entry = observe_entry(self); + + // The message payload: `results` is `msg+4`, a vector of stride 0x178. + std::int32_t result_count = -1; + if (readable(results, 8)) { + const char* first = static_cast(ptr_at(results, 0)); + const char* last = static_cast(ptr_at(results, 4)); + if (first && last && last >= first) result_count = static_cast((last - first) / 0x178); + } + std::int32_t paths = -1; + const std::int32_t predicted = predicted_node_line_words(self, &paths); + + out.push_back(tv::ptr(self).named("server")); + out.push_back(tv::ptr(results).named("results")); + out.push_back(tv::i32(result_count).named("result_count")); + push_server_args(out, self); + // The tail's only predictable draw source, evaluated before the tail runs: if phase 11 is + // the whole story on a quiet turn, the tail's measured word delta equals this number. + out.push_back(tv::i32(paths).named("node_paths")); + out.push_back(tv::i32(predicted).named("predict_nodeline_words")); + push_rng_args(out, g_tail.entry); +} + +void OnAllCombatDoneTailHook::regions(std::vector& out, void*, void*) { + push_rng_region(out, g_tail.entry.rng); +} + +OnAllCombatDoneTailHook::Args OnAllCombatDoneTailHook::rebind(trace::Scratch&, void* self, + void* results) { + return Args(self, results); +} + +void OnAllCombatDoneTailHook::ours(void* self, void* results) { + using H = trace::Hook; + if (H::mode == trace::Mode::Replace) { + refuse_replace("StrategyServer::OnAllCombatDone_Tail"); + if (H::original) H::original(self, results); + } +} + +void OnAllCombatDoneTailHook::coverage(trace::Coverage& c) { + tail_rng_common_coverage(c); + c.unmodelled("36 phases, of which two can draw and neither is modelled here", + trace::Risk::High, + "phase 6 reaches the unread 7499-byte combat resolver 0x007d5af0 (NextInt on " + "the node-cannon path, Twist plus NextInt on the salvage path) and phase 11 " + "draws one word per expired node line. `predict_nodeline_words` covers only the " + "second, and the nested ApplyEncounterResult / NodeLineDecay hooks are what " + "attribute the split", + "region:rng plus the two nested hooks"); + c.unmodelled("whether this handler runs on a turn with NO combat is what this hook is here " + "to settle, and until it has run it is a hypothesis", + trace::Risk::Medium, + "combat-done-tail.md §6 infers it from the determinism note -- the post-turn " + "autosave appears on every End Turn and this handler is its only reachable " + "caller -- not from the instruction stream", + "arg:encounters says how many encounters this call saw; a call with 0 settles " + "it"); +} + +// ---- Game::StrategyServer::ApplyEncounterResult ------------------------------------------------- + +void ApplyEncounterResultHook::describe_args(std::vector& out, void* self, void* enc, + void* res) { + g_apply.entry = observe_entry(self); + out.push_back(tv::ptr(self).named("server")); + out.push_back(tv::ptr(enc).named("encounter")); + out.push_back(tv::ptr(res).named("result")); + // The three dispatch bytes. `+0x4 != 0` makes the whole function a no-op, so a call with it + // set that still moves the generator would be a real surprise. + if (readable(res, 8)) { + out.push_back(tv::u8(peek(res, 4)).named("res_no_battle")); + out.push_back(tv::u8(peek(res, 6)).named("res_peaceful")); + out.push_back(tv::u8(peek(res, 7)).named("res_surrendered")); + } + push_server_args(out, self); + push_rng_args(out, g_apply.entry); +} + +void ApplyEncounterResultHook::regions(std::vector& out, void*, void*, void*) { + push_rng_region(out, g_apply.entry.rng); +} + +ApplyEncounterResultHook::Args ApplyEncounterResultHook::rebind(trace::Scratch&, void* self, + void* enc, void* res) { + return Args(self, enc, res); +} + +void ApplyEncounterResultHook::ours(void* self, void* enc, void* res) { + using H = trace::Hook; + if (H::mode == trace::Mode::Replace) { + refuse_replace("StrategyServer::ApplyEncounterResult"); + if (H::original) H::original(self, enc, res); + } +} + +void ApplyEncounterResultHook::coverage(trace::Coverage& c) { + tail_rng_common_coverage(c); + c.unmodelled("the combat resolver 0x007d5af0 (7499 B) is completely unread", + trace::Risk::High, + "this hook measures what its subtree spends and models none of it. Nothing " + "about combat determinism can be settled until that function is read; this " + "only puts a number on the hole", + "region:rng measures the subtotal"); + c.unmodelled("the ~0xea0-byte combat report, the CombatReport list append at S+0x1fc, the " + "ClientEncounterResults push into S+0x2f4, the per-ship turn stamps and the " + "pairwise engagement bits", + trace::Risk::Medium, + "all of it is game state this hook does not declare and does not check", + ""); +} + +// ---- Game::StrategyServer::NodeLineDecay -------------------------------------------------------- + +void NodeLineDecayHook::describe_args(std::vector& out, void* self) { + g_nld.entry = observe_entry(self); + g_nld.paths = -1; + g_nld.predicted = predicted_node_line_words(self, &g_nld.paths); + out.push_back(tv::ptr(self).named("server")); + push_server_args(out, self); + out.push_back(tv::i32(g_nld.paths).named("node_paths")); + // Written before the original runs: this is the falsifiable claim, not a report of what + // happened. If the measured delta on `rng` is not this number, the model is wrong. + out.push_back(tv::i32(g_nld.predicted).named("predict_words")); + push_rng_args(out, g_nld.entry); +} + +void NodeLineDecayHook::regions(std::vector& out, void*) { + push_rng_region(out, g_nld.entry.rng); +} + +NodeLineDecayHook::Args NodeLineDecayHook::rebind(trace::Scratch& s, void* self) { + g_nld.compare = true; + g_nld.s_rng = (s.count() > 0 && s.size(0) >= kRngSize) ? s.ptr(0) : nullptr; + return Args(self); +} + +void NodeLineDecayHook::ours(void* self) { + using H = trace::Hook; + const bool compare = g_nld.compare; + g_nld.compare = false; + if (H::mode == trace::Mode::Replace) { + refuse_replace("StrategyServer::NodeLineDecay"); + if (H::original) H::original(self); + return; + } + if (!compare) return; + // The model: one word per expired node line, and nothing else in the 1117-byte body reaches + // a generator (direct-call sweep to depth 5 over 140 functions, one hit). If the prediction + // failed to read the graph it advances nothing, which diverges loudly rather than quietly. + if (g_nld.s_rng && g_nld.entry.have && g_nld.predicted >= 0) + advance_scratch_rng(g_nld.s_rng, reinterpret_cast(g_nld.entry.rng), + g_nld.predicted); +} + +void NodeLineDecayHook::coverage(trace::Coverage& c) { + tail_rng_common_coverage(c); + c.unmodelled("the collapse itself: 0x007a92e0 (690 B) and 0x007a4700 (2244 B) destroy " + "or halt fleets and post EVENT_NODEDECAY_FLEET_DESTROYED_VIANODE / " + "_HALTED / _HALTED_VIANODE, and loop 3 posts two more decay-stage events", + trace::Risk::High, + "ours advances the generator and writes nothing else. In compare mode that is " + "the intent -- the check is the word count -- but it means a clean verdict here " + "says nothing about which lines actually collapsed", + "region:rng only"); + c.unmodelled("the draw-count model is verified by a DIRECT-call sweep of the downstream " + "pair; their subtrees contain unresolved indirect call sites", + trace::Risk::Medium, + "if one of those vtable slots reaches a generator, the measured delta will " + "exceed `predict_words` and this hook will diverge -- which is the correct " + "outcome, and the reason the prediction is recorded as an argument", + "arg:predict_words vs region:rng is exactly that check"); + c.unmodelled("the expiry formula reproduces a signed idiv on nptf/npdtf without knowing " + "whether nptf can be negative", + trace::Risk::Low, + "the original never sign-checks the traffic accumulator. The model truncates " + "toward zero the same way; if the field is always non-negative the question " + "never arises, and no save has been observed with a negative one", + ""); +} + +// ---- Game::StrategyServer::ProcessNodeSpaceTravel ------------------------------------------------ + +void ProcessNodeSpaceTravelHook::describe_args(std::vector& out, void* self) { + g_nodespace.entry = observe_entry(self); + out.push_back(tv::ptr(self).named("server")); + push_server_args(out, self); + push_rng_args(out, g_nodespace.entry); +} + +void ProcessNodeSpaceTravelHook::regions(std::vector& out, void*) { + push_rng_region(out, g_nodespace.entry.rng); +} + +ProcessNodeSpaceTravelHook::Args ProcessNodeSpaceTravelHook::rebind(trace::Scratch&, void* self) { + return Args(self); +} + +void ProcessNodeSpaceTravelHook::ours(void* self) { + using H = trace::Hook; + if (H::mode == trace::Mode::Replace) { + refuse_replace("StrategyServer::ProcessNodeSpaceTravel"); + if (H::original) H::original(self); + } +} + +void ProcessNodeSpaceTravelHook::coverage(trace::Coverage& c) { + tail_rng_common_coverage(c); + c.unmodelled("2945 bytes of node-space movement, entirely unmodelled and never swept for " + "RNG by any lane", + trace::Risk::Medium, + "it is hooked here only because it runs TWICE a turn -- ProcessTurn phase 7 and " + "tail phase 10 -- so a draw inside it would be double-counted by anyone " + "modelling it once. The record says whether it draws at all", + "region:rng"); +} + +} // namespace shim::hooks diff --git a/src/shim/hooks/tail_rng.h b/src/shim/hooks/tail_rng.h new file mode 100644 index 0000000..58b7bee --- /dev/null +++ b/src/shim/hooks/tail_rng.h @@ -0,0 +1,164 @@ +// Lane Z — the per-turn RNG ledger. Six nested hooks that measure how many words the strategic +// generator consumes in one End Turn and attribute them to a phase. +// +// WHY. `sots-re/findings/control-flow/combat-done-tail.md` §3 found two draw sites in +// `StrategyServer::OnAllCombatDone_Tail` that nothing in the repo models -- one `NextFloat` per +// expired node line (phase 11, instruction-verified at 0x007ae095) and whatever the combat +// resolver spends under phase 6 -- and **both run before the autosave**. A reimplementation that +// reproduces `StrategyServer::ProcessTurn` and `ServerPlayer::ProcessTurn` perfectly still +// diverges the first turn a node line expires, because the generator's state is part of the +// saved state. The defect is invisible on our current saves, which is why it survived. +// +// WHAT IS MEASURED, not asserted: every hook declares the live `Mars::RNG` object at +// `StrategyServer+0x16c` as a Result region whose `describe` reports an ABSOLUTE WORD POSITION +// (see rng_ledger.h). The delta between a record's `side.rng.before.words` and +// `side.rng.after.words` is the exact number of words that call consumed, whatever spent them +// and whether or not anyone hooked it. Because the hooks nest, the subtotals attribute: +// +// Autosave(endTurn=1) ....................... the pre-turn state marker +// StrategyServer::ProcessTurn ............. the half the repo already models +// (combat: RunCombatRound / the combat server -- HOOKED BY NOBODY) +// StrategyServer::OnAllCombatDone_Tail .... the half nothing models +// ApplyEncounterResult (x encounters) ... phase 6, the combat-resolver subtree +// NodeLineDecay ......................... phase 11, one word per expired node line +// ProcessNodeSpaceTravel .................. runs TWICE a turn; never swept for draws +// Autosave(endTurn=0) ....................... the post-turn state marker +// +// and the residual -- bracket total minus the attributed subtotals -- is the part of a turn a +// reimplementation would silently miss. That number is the deliverable. +// +// THE BASE. `this` for the four StrategyServer hooks is **S**, the frame `ProcessTurn` uses, +// not `S+4`. Every `StrategyServer_off_*` in the generated header is an `S+4` offset EXCEPT +// `StrategyServer_off_RNG`, which is already the S frame. This file adds 4 where it must and +// says so at each site; reading the wrong base is a mistake this campaign has already paid for. +// +// REPLACE MODE IS REFUSED EVERYWHERE. These hooks model RNG consumption, nothing else; running +// `ours` instead of a turn driver would produce a state no code path produces. +#pragma once + +#include +#include +#include + +#include "shim/trace/hook.h" + +namespace shim::hooks { + +// Shared coverage note: what none of these hooks check. Each descriptor adds its own on top. +void tail_rng_common_coverage(trace::Coverage& c); + +// ---- the two absolute markers --------------------------------------------------------------- +// +// `StrategyHost::Autosave(outName, endTurn)` 0x00895210. Two call sites in the image: +// `SendEndTurn` 0x007839d7 pushes `1` and writes the PRE-turn state, the `SNMAllCombatDone` +// handler 0x00784e59 pushes `0` and writes the POST-turn state. The words between them are the +// turn's whole RNG cost as the two save files see it. +// +// Two corrections to lane K §6.1, both read from the bytes: the epilogue is **`ret 8`**, not +// `ret 4`, and the function **returns the `std::string*` in EAX** (the MSVC named-return slot), +// which is why `Ret` is `void*` here -- declaring it `void` would drop EAX on the floor at +// both call sites. `this` is not passed by the caller at all: both sites hardcode +// `mov ecx,0xb29f98`. +struct StrategyHostAutosaveHook { + static constexpr const char* name = "Game::StrategyHost::Autosave"; + static constexpr trace::CallConv conv = trace::CallConv::Thiscall; + using Ret = void*; // the NRV std::string*, returned in EAX + using Args = std::tuple; // this (the global), std::string* out, endTurn + + static void describe_args(std::vector& out, void* self, void* name_out, bool end_turn); + static trace::Tv describe_ret(void* r); + static void regions(std::vector& out, void* self, void* name_out, bool end_turn); + static Args rebind(trace::Scratch& s, void* self, void* name_out, bool end_turn); + static void* ours(void* self, void* name_out, bool end_turn); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } + static void coverage(trace::Coverage& c); +}; + +// ---- the two turn drivers ------------------------------------------------------------------- + +// `StrategyServer::ProcessTurn(float dt)` 0x007dc6c0, `ret 4` (lane T §1). +struct StrategyServerProcessTurnHook { + static constexpr const char* name = "Game::StrategyServer::ProcessTurn"; + static constexpr trace::CallConv conv = trace::CallConv::Thiscall; + using Ret = void; + using Args = std::tuple; // this (StrategyServer* S), dt + + static void describe_args(std::vector& out, void* self, float dt); + static void regions(std::vector& out, void* self, float dt); + static Args rebind(trace::Scratch& s, void* self, float dt); + static void ours(void* self, float dt); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } + static void coverage(trace::Coverage& c); +}; + +// `StrategyServer::OnAllCombatDone_Tail(std::vector*)` 0x007d92a0, `ret 4` +// (lane K). Exactly one caller: the `SNMAllCombatDone` case of `StrategyHost::OnMessage`. +struct OnAllCombatDoneTailHook { + static constexpr const char* name = "Game::StrategyServer::OnAllCombatDone_Tail"; + static constexpr trace::CallConv conv = trace::CallConv::Thiscall; + using Ret = void; + using Args = std::tuple; // this (StrategyServer* S), results vector + + static void describe_args(std::vector& out, void* self, void* results); + static void regions(std::vector& out, void* self, void* results); + static Args rebind(trace::Scratch& s, void* self, void* results); + static void ours(void* self, void* results); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } + static void coverage(trace::Coverage& c); +}; + +// ---- the two phases inside the tail that can draw ------------------------------------------- + +// Tail phase 6: `StrategyServer::ApplyEncounterResult(Encounter*, EncounterResults*)` +// 0x007d8920, `ret 8`. The gateway to the unread combat resolver `0x007d5af0`. +struct ApplyEncounterResultHook { + static constexpr const char* name = "Game::StrategyServer::ApplyEncounterResult"; + static constexpr trace::CallConv conv = trace::CallConv::Thiscall; + using Ret = void; + using Args = std::tuple; // this (S), Encounter*, EncounterResults* + + static void describe_args(std::vector& out, void* self, void* enc, void* res); + static void regions(std::vector& out, void* self, void* enc, void* res); + static Args rebind(trace::Scratch& s, void* self, void* enc, void* res); + static void ours(void* self, void* enc, void* res); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } + static void coverage(trace::Coverage& c); +}; + +// Tail phase 11: node-line decay, 0x007ae010. One `Mars::RNG::Chance(0.5f)` -- i.e. exactly one +// word -- per EXPIRED node line, instruction-verified at 0x007ae095. This is the only hook here +// that carries a model of its own draw count. +struct NodeLineDecayHook { + static constexpr const char* name = "Game::StrategyServer::NodeLineDecay"; + static constexpr trace::CallConv conv = trace::CallConv::Thiscall; + using Ret = void; + using Args = std::tuple; // this (StrategyServer* S) + + static void describe_args(std::vector& out, void* self); + static void regions(std::vector& out, void* self); + static Args rebind(trace::Scratch& s, void* self); + static void ours(void* self); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } + static void coverage(trace::Coverage& c); +}; + +// `ProcessNodeSpaceTravel` 0x007a0e20, run TWICE a turn (ProcessTurn phase 7 and tail phase 10) +// and never swept for draws by any lane. +struct ProcessNodeSpaceTravelHook { + static constexpr const char* name = "Game::StrategyServer::ProcessNodeSpaceTravel"; + static constexpr trace::CallConv conv = trace::CallConv::Thiscall; + using Ret = void; + using Args = std::tuple; // this (StrategyServer* S) + + static void describe_args(std::vector& out, void* self); + static void regions(std::vector& out, void* self); + static Args rebind(trace::Scratch& s, void* self); + static void ours(void* self); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } + static void coverage(trace::Coverage& c); +}; + +// Process facts the hooks need (exe base for RVAs, a line logger). Call once before installing. +void init_tail_rng(std::uintptr_t exe_base, void (*log_line)(const char* line)); + +} // namespace shim::hooks diff --git a/src/shim/main.cpp b/src/shim/main.cpp index 2bb4605..cbbdcbe 100644 --- a/src/shim/main.cpp +++ b/src/shim/main.cpp @@ -23,6 +23,7 @@ #include "shim/hooks/fleet_movement.h" #include "shim/hooks/global_consts.h" #include "shim/hooks/research.h" +#include "shim/hooks/tail_rng.h" #include "shim/hooks/tech_effects.h" #include "shim/trace/hook.h" #include "shim/trace/selftest.h" @@ -166,6 +167,13 @@ using ColonyTurnHook = shim::trace::Hook; using MoveFleetHook = shim::trace::Hook; using FleetMovementHook = shim::trace::Hook; +// Lane Z: the per-turn RNG ledger (docs/Z-tail-rng.md). +using AutosaveHook = shim::trace::Hook; +using ServerTurnHook = shim::trace::Hook; +using CombatDoneTailHook = shim::trace::Hook; +using ApplyEncounterHook = shim::trace::Hook; +using NodeDecayHook = shim::trace::Hook; +using NodeSpaceHook = shim::trace::Hook; void InstallHooks(shim::trace::Tracer& tracer) { const uintptr_t exeBase = reinterpret_cast(GetModuleHandleA(nullptr)); @@ -218,6 +226,19 @@ void InstallHooks(shim::trace::Tracer& tracer) { InstallTemplateHook(tracer, exeBase, sots::addr::StrategyServer_MoveFleet); InstallTemplateHook(tracer, exeBase, sots::addr::StrategyServer_ProcessFleetMovement); + // Lane Z: the per-turn RNG ledger. Six nested hooks bracketing one End Turn between the two + // autosaves; every one declares the strategic generator and nothing else. Installed BEFORE + // the fpu module because both want StrategyServer::ProcessTurn and MinHook allows one hook + // per target -- `fpu.sample_turn=off` is the config that hands it over cleanly, and if it is + // left on the fpu sampler's MH_CreateHook is what fails and says so. + shim::hooks::init_tail_rng(exeBase, &ShimLogLine); + InstallTemplateHook(tracer, exeBase, sots::addr::StrategyHost_Autosave); + InstallTemplateHook(tracer, exeBase, sots::addr::StrategyServer_ProcessTurn); + InstallTemplateHook(tracer, exeBase, sots::addr::StrategyServer_OnAllCombatDone_Tail); + InstallTemplateHook(tracer, exeBase, sots::addr::StrategyServer_ApplyEncounterResult); + InstallTemplateHook(tracer, exeBase, sots::addr::StrategyServer_NodeLineDecay); + InstallTemplateHook(tracer, exeBase, sots::addr::StrategyServer_ProcessNodeSpaceTravel); + // Lane F: x87 control-word forcing at the turn gate + the per-tick change sampler. // Installed last so it is nowhere near the template hooks it is meant to measure. shim::fpu::install(exeBase, &ShimLogLine); @@ -268,6 +289,12 @@ void Shim_Init(HMODULE self) { PlayerTurnHook::register_policy(tracer); MoveFleetHook::register_policy(tracer); FleetMovementHook::register_policy(tracer); + AutosaveHook::register_policy(tracer); + ServerTurnHook::register_policy(tracer); + CombatDoneTailHook::register_policy(tracer); + ApplyEncounterHook::register_policy(tracer); + NodeDecayHook::register_policy(tracer); + NodeSpaceHook::register_policy(tracer); // A hook that never stated what it does not check is a defect, not a detail: say so in // shim.log as well as in the trace's meta line (docs/harness-audit.md). for (const std::string& h : tracer.unstated_hooks()) diff --git a/src/shim/shim.cfg.zledger b/src/shim/shim.cfg.zledger new file mode 100644 index 0000000..581877b --- /dev/null +++ b/src/shim/shim.cfg.zledger @@ -0,0 +1,40 @@ +# Lane Z -- the per-turn RNG ledger, TRACE. Copy over C:\SOTS\shim.cfg. +# +# Every other hook is off so the log holds nothing but the ledger: six records per End Turn +# instead of ~40 per player. Read `side.rng.before.words` and `side.rng.after.words` on each +# record -- the difference is the number of 32-bit words the strategic generator consumed inside +# that call. Because the hooks nest, subtracting the inner subtotals from the outer one +# attributes the turn. +# +# fpu.sample_turn=off is REQUIRED: the fpu module samples the control word at +# StrategyServer::ProcessTurn, MinHook allows one hook per target, and this config needs that +# address for the ledger. With it left on, shim.log records which of the two failed to install. +hooks=trace +hook.Shim::SelfTest::Fill=off +hook.Mars::GlobalConsts::LoadFile=off +hook.Game::WeaponDictionary::Init=off +hook.Game::SectionDictionary::SectionDictionary=off +hook.Game::ServerPlayer::ComputeBudget=off +hook.Game::TechTree::ProcessResearch=off +hook.Game::ServerPlayer::OnTechResearched=off +hook.Game::ServerSystem::ProcessTurn=off +hook.Game::ServerPlayer::ProcessTurn=off +hook.Game::StrategyServer::MoveFleet=off +hook.Game::StrategyServer::ProcessFleetMovement=off + +hook.Game::StrategyHost::Autosave=trace +hook.Game::StrategyServer::ProcessTurn=trace +hook.Game::StrategyServer::OnAllCombatDone_Tail=trace +hook.Game::StrategyServer::ApplyEncounterResult=trace +hook.Game::StrategyServer::NodeLineDecay=trace +hook.Game::StrategyServer::ProcessNodeSpaceTravel=trace + +fpu.sample_turn=off +fpu.sample_ticks=off + +# The generator is 0x9cc bytes and every record carries it twice. Inlining it as hex would make +# each record ~5 kB of state nobody reads -- the `describe` already reduces it to +# {left, index, block, words, block_hash}, and the raw bytes are only ever a sha256 line. +trace.inline_max=64 +trace.path=C:\SOTS\shim.trace.jsonl +trace.flush=always diff --git a/tests/shim_rng_ledger/CMakeLists.txt b/tests/shim_rng_ledger/CMakeLists.txt new file mode 100644 index 0000000..fc279ff --- /dev/null +++ b/tests/shim_rng_ledger/CMakeLists.txt @@ -0,0 +1,6 @@ +# Lane Z: the RNG word ledger (absolute generator position recovered from state alone). +add_executable(shim_rng_ledger_unit_tests unit_tests.cpp) +target_link_libraries(shim_rng_ledger_unit_tests PRIVATE shim_rng_ledger) +target_include_directories(shim_rng_ledger_unit_tests PRIVATE ${CMAKE_SOURCE_DIR}/tests/game_sim) +target_compile_options(shim_rng_ledger_unit_tests PRIVATE -Wall -Wextra -Werror) +add_test(NAME shim_rng_ledger_unit COMMAND shim_rng_ledger_unit_tests) diff --git a/tests/shim_rng_ledger/unit_tests.cpp b/tests/shim_rng_ledger/unit_tests.cpp new file mode 100644 index 0000000..cbd12d0 --- /dev/null +++ b/tests/shim_rng_ledger/unit_tests.cpp @@ -0,0 +1,182 @@ +// Lane Z: the RNG word ledger. +// +// The whole point of the instrument is that a word delta is exact whatever spent the words, so +// the tests drive a real MT19937 by known amounts and check the ledger's arithmetic against the +// count -- including across block boundaries, across a NextInt rejection loop, and when the +// observations arrive out of chronological order (which is how Hook<> renders nested calls). +#include "shim/hooks/rng_ledger.h" + +#include +#include + +#include "check.h" +#include "mars/rng/mt19937.h" + +using shim::hooks::RngLedger; +using shim::hooks::RngPos; +using mars::rng::MT19937; + +namespace { + +RngPos observe(RngLedger& L, const MT19937& g) { return L.observe(g.state(), g.left()); } + +void test_delta_within_one_block() { + RngLedger L; + L.reset(); + MT19937 g(12345); + const RngPos a = observe(L, g); + CHECK(a.known); + for (int i = 0; i < 100; ++i) (void)g.next_u32(); + const RngPos b = observe(L, g); + CHECK(b.known); + CHECK_EQ(b.words - a.words, 100LL); + CHECK_EQ(b.block, a.block); +} + +void test_delta_across_blocks() { + RngLedger L; + L.reset(); + MT19937 g(999); + const RngPos a = observe(L, g); + // Well past a block boundary, and not a multiple of 624 -- an off-by-one in the position + // formula would survive a multiple. + const int n = 624 * 3 + 17; + for (int i = 0; i < n; ++i) (void)g.next_u32(); + const RngPos b = observe(L, g); + CHECK(b.known); + CHECK_EQ(b.words - a.words, static_cast(n)); + CHECK_EQ(b.block - a.block, 3); +} + +void test_exhausted_block_boundary() { + // left == 0 is a real state: the block is spent and the next draw twists. Position must be + // continuous across it, or every ledger entry straddling a boundary is off by 624. + RngLedger L; + L.reset(); + MT19937 g(7); + const RngPos a = observe(L, g); + for (int i = 0; i < 624; ++i) (void)g.next_u32(); + const RngPos b = observe(L, g); + CHECK_EQ(b.left, 0); + CHECK_EQ(b.block, a.block); // still the same block; the twist has not happened yet + CHECK_EQ(b.words - a.words, 624LL); + (void)g.next_u32(); + const RngPos c = observe(L, g); + CHECK_EQ(c.block, a.block + 1); + CHECK_EQ(c.words - a.words, 625LL); +} + +void test_rejection_loop_counts_words_not_draws() { + // next_int_inclusive can spend several words on one call. The ledger must report the words. + RngLedger L; + L.reset(); + MT19937 g(4242); + MT19937 shadow(4242); + const RngPos a = observe(L, g); + (void)g.next_int_inclusive(100); + const RngPos b = observe(L, g); + long long words = 0; + for (;;) { + const std::uint32_t r = shadow.next_u32() & MT19937::cover_mask(100); + ++words; + if (r <= 100) break; + } + CHECK(words >= 1); + CHECK_EQ(b.words - a.words, words); +} + +void test_out_of_order_observation() { + // Hook<> renders a nested call's snapshots before the outer call's. The outer `before` + // state is therefore observed twice: once at entry (in order) and once at render time, + // by which point the chain has moved on. The second lookup must still resolve. + RngLedger L; + L.reset(); + MT19937 g(31337); + std::uint32_t outer_block[624]; + std::memcpy(outer_block, g.state(), sizeof outer_block); + const int outer_left = g.left(); + const RngPos entry = L.observe(outer_block, outer_left); // observed at entry, in order + CHECK(entry.known); + + for (int i = 0; i < 624 * 2 + 5; ++i) (void)g.next_u32(); + const RngPos inner_after = observe(L, g); // rendered first + CHECK(inner_after.known); + for (int i = 0; i < 30; ++i) (void)g.next_u32(); + const RngPos outer_after = observe(L, g); + CHECK(outer_after.known); + + // ... and now the stale `before` snapshot is rendered. + const RngPos rendered = L.observe(outer_block, outer_left); + CHECK(rendered.known); + CHECK_EQ(rendered.words, entry.words); + CHECK_EQ(outer_after.words - rendered.words, static_cast(624 * 2 + 35)); +} + +void test_backwards_without_entry_observation_is_unknown() { + // The honest failure: a state behind the anchor cannot be positioned, and the ledger says + // so rather than inventing a number. This is why hooks observe at entry. + RngLedger L; + L.reset(); + MT19937 g(555); + std::uint32_t early[624]; + std::memcpy(early, g.state(), sizeof early); + const int early_left = g.left(); + for (int i = 0; i < 624 * 4; ++i) (void)g.next_u32(); + const RngPos anchor = observe(L, g); // the chain starts HERE + CHECK(anchor.known); + const RngPos behind = L.observe(early, early_left); + CHECK(!behind.known); + CHECK(L.misses() >= 1); +} + +void test_second_generator_reads_unknown() { + // One ledger, two independent generators: the second one's blocks are not on the first + // one's chain. That must read unknown, because it is the signal that the "one strategic + // generator" assumption failed. + RngLedger L; + L.reset(); + MT19937 a(1); + MT19937 b(2); + CHECK(observe(L, a).known); + CHECK(!observe(L, b).known); +} + +void test_bad_left_is_rejected() { + RngLedger L; + L.reset(); + MT19937 g(8); + CHECK(!L.observe(g.state(), -1).known); + CHECK(!L.observe(g.state(), 625).known); + CHECK(!L.observe(nullptr, 100).known); +} + +void test_observe_object_layout() { + // The live-memory path: vptr, mt[624] at +4, next at +0x9c4, left at +0x9c8 (RNG_size + // 0x9cc). A short region must be refused rather than read past its end. + RngLedger L; + L.reset(); + MT19937 g(2024); + std::vector obj(0x9cc, 0); + std::memcpy(obj.data() + 4, g.state(), 624 * 4); + const std::int32_t left = g.left(); + std::memcpy(obj.data() + 0x9c8, &left, 4); + const RngPos a = L.observe_object(obj.data(), obj.size()); + CHECK(a.known); + CHECK_EQ(a.left, left); + CHECK(!L.observe_object(obj.data(), 0x100).known); +} + +} // namespace + +int main() { + test_delta_within_one_block(); + test_delta_across_blocks(); + test_exhausted_block_boundary(); + test_rejection_loop_counts_words_not_draws(); + test_out_of_order_observation(); + test_backwards_without_entry_observation_is_unknown(); + test_second_generator_reads_unknown(); + test_bad_left_is_rejected(); + test_observe_object_layout(); + return simtest::finish("shim_rng_ledger_unit"); +}