diff --git a/CMakeLists.txt b/CMakeLists.txt index b0d09c5..ad562c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,11 @@ add_library(shim_movement STATIC src/shim/hooks/movement_inputs.cpp) target_link_libraries(shim_movement PUBLIC shim_trace sots_game_sim) target_compile_options(shim_movement PRIVATE -Wall -Wextra -Werror) +# ---- P events adapter: live Game::EventStorage <-> sots::events model (pure, host-tested) ---- +add_library(shim_events STATIC src/shim/hooks/event_inputs.cpp) +target_link_libraries(shim_events PUBLIC shim_trace sots_addresses sots_game_events) +target_compile_options(shim_events PRIVATE -Wall -Wextra -Werror) + if(WIN32) # ---- shim: proxy binkw32.dll that the original game loads (Phase 2 frontend) ---- add_library(minhook STATIC @@ -80,7 +85,7 @@ if(WIN32) src/shim/hooks/fleet_movement.cpp) target_link_libraries(shim_hooks PUBLIC shim_trace sots_addresses sots_game_config sots_game_sim sots_game_effects mars_rng shim_budget shim_techfx - shim_colony shim_movement) + shim_colony shim_movement shim_events) target_compile_options(shim_hooks PRIVATE -Wall -Wextra -Werror) add_library(binkw32 SHARED src/shim/main.cpp src/shim/binkw32.def) @@ -96,7 +101,7 @@ else() add_executable(addr_smoke tests/addr_smoke.cpp) target_link_libraries(addr_smoke PRIVATE sots_addresses) add_test(NAME addr_smoke COMMAND addr_smoke) - foreach(_t mars_parse game_config game_data game_design game_sim mars_stream mars_text mars_vfs shim_trace game_effects game_events shim_budget shim_techfx shim_colony shim_movement) + foreach(_t mars_parse game_config game_data game_design game_sim mars_stream mars_text mars_vfs shim_trace game_effects game_events shim_budget shim_techfx shim_colony shim_movement shim_events) if(EXISTS ${CMAKE_SOURCE_DIR}/tests/${_t}/CMakeLists.txt) add_subdirectory(tests/${_t}) endif() diff --git a/docs/P-events-wiring.md b/docs/P-events-wiring.md new file mode 100644 index 0000000..443b8ba --- /dev/null +++ b/docs/P-events-wiring.md @@ -0,0 +1,241 @@ +# P — wiring the event posts into `ours` + +Lane E recovered the event API and landed `src/game/events/` as a pure module. Lane R's guarded +recapture then made the missing post precise: `TechTree::ProcessResearch` exits 1 with exactly one +divergent field, + +``` +side.events.after.v.next_id [exact] orig={"t":"i32","v":4} ours={"t":"i32","v":3} +``` + +on a call where the arithmetic reproduces perfectly (`node[144].progress` 2879→5768, +`flag` 1→2, the single RNG draw identical). This lane closes that field. + +It takes lane E's **option (a), count-only**. The game's `EventStorage::PostEvent` is never +called and no live byte is written; `ours` posts into its own `sots::events::EventStorage` and +writes the resulting counts into the **scratch copy** of the owner's storage header. That is what +makes it safe to put near the VM, and it is also the honest limit of what it proves. + +--- + +## 1. What changed + +| file | change | +|---|---| +| `src/game/events/research_events.{h,cpp}` | `PostResearchPassEvents` — the events one `ProcessResearch` call raises, as a pure function of the per-node outcomes; `KeylessEventText()` for a caller with no string table | +| `src/shim/hooks/event_inputs.{h,cpp}` (new lib `shim_events`) | the live↔model adapter: scan a `Game::EventStorage` before the original runs, seed the model from it, write the counts back into a scratch header | +| `src/shim/hooks/research.{h,cpp}` | the wiring, the `observed_techs` region, the new args, the rewritten `Coverage` | +| `tests/game_events/test_events.cpp` | 6 new cases on the pass driver (the three call shapes lane R captured, plus dedup and the keyless lookup) | +| `tests/shim_events/unit_tests.cpp` (new) | 10 cases on the adapter, driven on a byte buffer laid out with the real offsets | +| `include/generated/sots_addresses.h` | regenerated from `sots-re ghidra/addresses.json` (386 → 396 entries) | + +`tools/clean_room_check.sh` — **OK**. Host `ctest` — **33/33 passed** (was 32/32; `shim_events_unit` +is new). Both run as separate commands. The shim TU itself is syntax-checked only +(`g++ -std=c++17 -Wall -Wextra -fsyntax-only`, clean): there is no MinGW i686 cross toolchain on +this box, so `tools/build-shim.sh` still has to run on CT111 before any deploy. + +### New binary facts (via `addresses.json` → `tools/gen_addresses.py`, never hand-edited) + +Read out of `TechTree::SetResearched` (0x00581e10) by this lane, plus one from lane R's guard: + +`ServerPlayer_off_ObservedTechs` (0x274) · `TechNode_off_TurnAvailable` (0x20) · +`TechNode_off_TurnResearched` (0x24) · `TechNode_off_Order` (0x28) · +`TechNode_off_Children` (0x04) · `TechEdge_off_CostRP` (0x1c) · `TechEdge_off_ChildDef` (0x40) · +`TechTree_off_OrderCounter` (0x20) · `TechTree_SetResearched_flag_Force` (0x2) · +`TechTree_SetResearched_flag_Silent` (0x4) + +The last one settles a question the count depends on. `SetResearched` invokes the owner callback as +`vft+0x10(def, (flags >> 2) & 1)`, so bit 2 of its `flags` argument **is** `OnTechResearched`'s +`silent`. `ProcessResearch` calls `SetResearched(def, 2)`, so `silent` is false and the completion +event *is* posted. That was previously an inference from an event count; it is now read off the +instruction stream. + +--- + +## 2. How the count model works, and why each piece is there + +### The scan is taken *before* the original + +In compare mode `ours` runs **after** the original (`hook.h`, `run`). A scan of the event +list taken inside `ours` would see the original's own freshly posted records — and the model would +deduplicate against them, post nothing, and agree for exactly the wrong reason. So +`ScanEventStorage` runs in `describe_args`, which `hook.h` calls immediately before `regions()` +and before the original. + +### Ours posts into its own storage, seeded from the scan + +`SeedFromScan` rebuilds the **turn buckets** (their `EvTurn` values, in order) and `EvNxID`, and +leaves the buckets **empty**. Empty because the records' `EvDsc`/`EvMsg` are localized game text, +which the engine must not carry. + +That has one consequence, and it is measured rather than assumed. A post can only be deduplicated +against a record already in the same turn bucket, and only a record with the same `EvImg` can ever +match. So the scan counts the events in the current turn's bucket that carry one of the six +research `EvImg` identifiers and reports it as the argument **`events_dedup_risk`**: + +* `events_dedup_risk = 0` → nothing in the bucket can collide with anything we post, and the + count-only model is **exact** for that call. +* anything else → our id count is a lower bound, and the report says so instead of the reader + having to guess. + +On the reference workload it should be 0 on every call (see §4). + +### The text stand-in preserves exactly the property dedup needs + +`KeylessEventText()` resolves every string-table key to the bare format `"%s"`, and the hook passes +the node's own index as the substitution. So `EvMsg` is the node index and nothing else. That is +not the game's text and must never be shown or serialized — but it preserves the one property the +comparator cares about: + +* two different nodes going over budget in the same turn get different `EvMsg` → both are kept, + which is what the original does (their real messages differ by the tech name); +* the same node twice collapses to one id, which is also what the original does; +* an over-budget and a completion for the same node share `EvMsg` but differ in `EvImg`, which the + comparator also tests → both are kept. + +`tests/game_events` pins all three through this lookup. + +### Only compare mode writes + +`WriteBackCounts` is called **only when `compare` is set**. In replace mode the region pointer is +live game memory, and bumping `EvNxID` there without a serialized record behind it would corrupt +the very save the oracle hashes — strictly worse than the missing event. Replace mode therefore +still posts nothing, and the `Coverage` note says so. + +--- + +## 3. What is verified, what is modelled by analogy, and what is not modelled + +**Verified from the instruction stream** (lane E's read, plus this lane's read of 0x00581e10): + +* `EVENT_RESEARCH_OVERBUDGET` fires iff the completion roll failed **and** `!wasDone && nowDone`. + `sim::ResearchStepResult::overbudgetEvent` already computes exactly that, in the same branch + that sets `node.flag = 2` — the flag lane R confirmed on the game. +* One completion event fires on every completion reached from `ProcessResearch`, because + `silent = (flags >> 2) & 1` and `ProcessResearch` passes `flags = 2`. +* The posting rules themselves: dedup keys, `act == 0 && !obj && !pos → 2`, the `FLT_MAX` default, + `EvNxID` starting at 0, the last-match bucket lookup, the prune off-by-one. All of lane E's, all + unit-tested. + +**Modelled by analogy, and count-neutral:** which of `EVENT_RESEARCH_COMPLETE` / +`_UNDERBUDGET` is posted. The original splits on `TechTree::GetProgressRatio(tree, def)` against +`(double)0.8f`; `ours` splits on `ResearchStepResult::completedEarly`, which is `progress/cost` +against the same constant, computed by `ProcessResearch` itself two instructions earlier for the +`flag = 0` decision. They are very probably the same number, but this lane did not read +`GetProgressRatio` (0x0057e950). **It does not affect `next_id`**: both branches post exactly one +event. It would affect a text or `EvImg` comparison, and no region has one. + +**Not modelled — flagged, not guessed:** + +* **`EVENT_TECHS_UNLOCKED`.** Its trigger *is* pinned: `SetResearched`'s second sweep sets + `state = 2` and stamps `turnAvailable` (only when it reads −1), and the tail loop at 0x00587ff4 + collects `state == 2 && turnAvailable == currentTurn`. But evaluating it needs the child-unlock + cascade, which `ours` deliberately does not run — B3 declared it out of scope and its writes + land on live objects compare mode must not touch. `PostResearchPassEvents` therefore takes the + unlock list as an **input** and is handed `nullptr` (*"no list"*), which is deliberately + distinct from an empty list (*"computed, and empty"*). **Predicted residual: `next_id` short by + exactly 1 on every call that completes a tech.** Guessing the trigger — "post it whenever + something completed" — would score on this save and be wrong the first time a completion + unlocks nothing. +* **`EVENT_TEMPERANCE`.** Posted from the same `OnTechResearched` call, but only when the + completed tech is a temperance tech *and* the per-species sweep actually cured at least one + addicted system. Neither input is available here, and no save in `verify/` has addiction, so + this has never been observed firing. The helper exists and is unit-tested; it is not driven. +* **`EVENT_NO_RESEARCH`.** Posted from `ServerPlayer::ProcessTurn` (0x0089168c), not from this + hook. Out of scope by call site, not by omission. +* **The text.** `EvDsc`/`EvMsg` come from the game's string table. No region can see them, and + none pretends to. + +--- + +## 4. What the next VM run should show + +This is a prediction, not a fishing trip. Run the existing `shim.cfg.recapb3` recipe +(`ref-turn2.sav` → Launch → End Turn) unchanged, then the five-End-Turn continuation. + +### 4.1 The first End Turn — the headline + +**`Game::TechTree::ProcessResearch`: 3 calls, 3 compared, 0 divergent, `tracecmp` exit 0.** + +| call | alloc | `events.next_id` orig | ours before this lane | ours now | +|---|---|---|---|---| +| **0** | `{tech 144, 2889}`, species 2 | 3 → **4** | 3 ✗ | 3 → **4** ✓ | +| 1 | `{90, 0}` | 0 → 0 | 0 ✓ | 0 ✓ | +| 2 | `{9, 0}` | 0 → 0 | 0 ✓ | 0 ✓ | + +Also on call 0, and these are the fields that say the model is right for the right reason: + +* `turn` = **3**. If it is not 3, nothing below is trustworthy: the whole post lands in the wrong + bucket. (`ModCount` at `*(int*)(*(char**)(player+8) + 8)`.) +* `events_turn_bucket_exists` = **true** — the turn-3 bucket already holds `EVENT_SHIPS_BUILT`. + This is what keeps `events.turns` at 2 and `turns_bytes` at 0x30 on both sides. If it comes back + false, our post creates a bucket the original did not and `turns_bytes` diverges by 0x18 — which + would mean `turn` is wrong, not that the model is. +* `events_next_id_in` = **3**, `events_in_turn_bucket` = **1**, `events_dedup_risk` = **0**. +* `events_scan_truncated` must be **absent**. Its presence means a vector header did not parse and + every event number in the record is unreliable. +* `node[144].progress` 2879 → 5768 and `flag` 1 → 2 on both sides, `rng` identical — unchanged + from lane R's capture. If any of those regressed, the events wiring broke something it should + not have touched. +* `observed_techs.bytes` **unchanged on all three calls**: turn 1 of `ref-turn2` contains no tech + completion, so nothing appends. + +Calls 1 and 2 are other players with empty event lists (`EvNxID = 0`). `WriteBackCounts` must leave +their scratch header byte-identical — the "no-op" case is unit-tested. + +### 4.2 The five-turn continuation — expected to still diverge, by a known amount + +Lane R warns that anything past the first End Turn is *a* run, not *the* run (from turn 4 the AI +picks a different target). Treat the call ids as indicative and the **shape** as the prediction: + +| call | what happens | orig `next_id` | ours now | residual | +|---|---|---|---|---| +| 0 | over budget | 4 | **4** | none | +| 3 | completion | 7 | **6** | 1 = `EVENT_TECHS_UNLOCKED` | +| 6 | rolled, failed, no cost crossing | 8 | 8 | none | +| 9 | completion | 12 | **11** | 1 = `EVENT_TECHS_UNLOCKED` | +| 12 | rolled, failed | 14 | 14 | none | +| all others | zero-spend | 0 | 0 | none | + +So `tracecmp` should still exit 1 on the five-turn run, with **exactly two divergent calls instead +of three**, and both remaining divergences off by exactly 1 in `next_id`. Any other number is new +information: off by 2 means the completion event is not firing either; off by −1 means we posted +something the original did not. + +### 4.3 What the guard should stop saying + +`observed_techs` is now a **Result** region, so its bytes are excluded from the `player` guard's +undeclared-write scan. Expect the guard's span count on the two completion calls to drop by the +`player+0x274/0x278/0x27c` entries (lane R saw 3 spans on call 3 and 1 on call 9) and those +offsets to appear in the diff instead. That is the conversion, not a regression. + +### 4.4 The measurement to take while you are there + +On a completion call, `observed_techs.bytes` grows by exactly one element. +**That delta is `sizeof(ObservedTech)`,** which nothing in the project has pinned. On disk the +element is `{int otnF; int otnL; int odet; string otch; int owith}` (confirmed in +`turn3-state.sav` — the `otch` string holds the tech's name, e.g. `WEP_RedLas`), so a by-value +element should measure 0x2c or 0x30 and a `vector` would measure 4. Write the answer +into `addresses.json`; the entry there currently says explicitly that it is not pinned. + +--- + +## 5. Open + +* `sizeof(ObservedTech)` and its **append call site**. It is somewhere under the + `OnTechResearched` callback (both guards see it, which only bounds it to that subtree); it is not + in `OnTechResearched`'s own decompilation, and a constant search for `0x274` finds only unrelated + objects, so it is reached through a `lea` displacement in a callee. §4.4 measures the stride + without finding the site. +* `TechTree::GetProgressRatio` (0x0057e950) — the COMPLETE/UNDERBUDGET split is analogy until it + is read. +* The unlock cascade. It is now fully decompiled (see the sots-re note), so + `EVENT_TECHS_UNLOCKED` is implementable — but it belongs to whichever lane takes `SetResearched`, + because modelling it means `ours` starts writing child `state` / `turn_available` / `cost_rp`, + which B3 declared out of scope. +* `EventStorage::PruneOldTurns` still has never run: the reference saves are at turn ≤ 7 and the + window is 50. The model reproduces the off-by-one and the unit tests pin it; nothing has measured + it. +* Option (b) — delegating to the game's own `PostEvent` so the text and the save hash match — is + untouched and remains the only path that can prove `EvDsc`/`EvMsg`. It changes VM behaviour and + is deliberately not wired here. diff --git a/docs/harness-audit.md b/docs/harness-audit.md index e4cd9a0..86797ec 100644 --- a/docs/harness-audit.md +++ b/docs/harness-audit.md @@ -23,7 +23,7 @@ often it fires. | # | hook | undeclared side effect of the original | why it can hide a divergence | now | |---|---|---|---|---| -| 1 | `TechTree::ProcessResearch` (B3) | posts **`EVENT_RESEARCH_OVERBUDGET`** on the owner's `EventStorage` (`ServerPlayer+0x29c`), bumping `EvNxID` and the turn's list | **the known defect.** Same branch that sets the modelled `node.flag = 2`; serialized into the save, so it changes the oracle | **declared** as region `events`; `ours` still does not post it, so it now diverges loudly instead of passing | +| 1 | `TechTree::ProcessResearch` (B3) | posts **`EVENT_RESEARCH_OVERBUDGET`** on the owner's `EventStorage` (`ServerPlayer+0x29c`), bumping `EvNxID` and the turn's list | **the known defect.** Same branch that sets the modelled `node.flag = 2`; serialized into the save, so it changes the oracle | **declared** as region `events` and **modelled**: `ours` posts the pass's events into its own EventStorage and writes the counts into the region's scratch copy, so `next_id` is now a check. Count-only -- the composed text is still invisible. Lane P, `docs/P-events-wiring.md` | | 2 | `ServerPlayer::OnTechResearched` (B2) | posts **`EVENT_RESEARCH_COMPLETE` / `_UNDERBUDGET` / `_TEMPERANCE`** on the same `EventStorage` | identical shape to #1, and *worse*: B2 has no replace-mode oracle behind it — docs/B2.md gotcha 4 says a changed save hash on a completion turn is expected, so nothing would have flagged it | guard region `player` reports it | | 3 | `ServerSystem::ProcessTurn` (B4) | the addiction sweep constructs **MoraleEvents** and appends them to the system's capped history | same shape again. Worse still: `sim::ProcessColonyTurn` *computes* the morale events, and `DescribeMoraleEvents` exists — but the hook never calls it, so they are dropped on the floor: not compared, not even logged | guard region `system` reports it; the dead describer is called out below | | 4 | `StrategyServer::ProcessFleetMovement` (B4) | `OnFleetArrived` posts **`EVENT_FLEET_ARRIVED`**; the pass also writes `FPdpos` into every fleet and clears flags `0x2` and `0x100` on every fleet | fourth instance of the same class. No replace mode for this hook either, so neither layer could see it | coverage notes only (a guard would have to span every fleet; see §6) | @@ -51,9 +51,9 @@ Things I could **not** settle offline, and did not guess: - Whether the double `ComputeOutput` in B1 replace mode (#6) actually double-repairs ships. It needs a hook on `ComputeOutputFromRates`, or a VM run with a damaged ship in orbit. -- The exact stride of `EventStorage::TurnEvents`, so the `events` region reports the outer - vector's **byte** span (`turns_bytes`) rather than an element count. `EvNxID` is the field - that actually carries the signal. +- ~~The exact stride of `EventStorage::TurnEvents`~~ — settled by lane E (0x18) and read off the + instruction stream; the `events` region now reports `turns` as an element count alongside + `turns_bytes`. `EvNxID` is still the field that carries the signal. - Whether the section dictionary's `LoadSection` appends to the dictionary's own vector (docs/M2.md raises it as a hypothesis). Still a hypothesis; recorded as `risk: high`. - `ServerPlayer` / `ServerSystem` / `StarFleet` sizes come from the recovered object table in diff --git a/include/generated/sots_addresses.h b/include/generated/sots_addresses.h index fd9bb10..c528e84 100644 --- a/include/generated/sots_addresses.h +++ b/include/generated/sots_addresses.h @@ -1,5 +1,5 @@ // GENERATED — do not edit. Facts about Sword of the Stars.exe (GOG 1.8.1). -// Source: sots-re ghidra/addresses.json @ e0c3e64, generated 2026-09-08 by tools/gen_addresses.py +// Source: sots-re ghidra/addresses.json @ b0ef139, generated 2026-09-08 by tools/gen_addresses.py // Runtime address = (uintptr_t)GetModuleHandle(NULL) + RVA (the exe is ASLR-relocated). #pragma once #include @@ -245,10 +245,16 @@ constexpr uint32_t TechTree_Cost = 0x0017da00; constexpr uint32_t ServerPlayer_TechCostMult = 0x0040db50; // thiscall void (TechTree* this, TechDef* def, int flags) // state 4 + turn/order stamps + owner callback + child unlock cascade. Makes no direct RNG draw; the owner callback is not audited, so a compare that runs it is out of scope [verified] constexpr uint32_t TechTree_SetResearched = 0x00181e10; +// constant flags bit 1 of SetResearched(def, flags): skip the PrereqsMet test and complete unconditionally. TechTree::ProcessResearch passes flags = 2 [verified] +constexpr uint32_t TechTree_SetResearched_flag_Force = 0x00000002; +// constant flags bit 2 of SetResearched(def, flags): the owner callback is invoked as vft+0x10(def, (flags>>2)&1), so this bit IS OnTechResearched's `silent` argument. ProcessResearch passes flags = 2, so silent = FALSE and the completion event IS posted [verified] +constexpr uint32_t TechTree_SetResearched_flag_Silent = 0x00000004; // offset ServerPlayer* owner (0 for a tree with no player) [verified] constexpr uint32_t TechTree_off_Owner = 0x0000000c; // offset std::vector indexed by tech id (MSVC2010: 3 words {first@+0x10, last@+0x14, end@+0x18}); entries may be NULL [verified] constexpr uint32_t TechTree_off_Nodes = 0x00000010; +// offset int completion-order counter; SetResearched stamps node+0x28 from it and post-increments it. Harness-audit row 9 [verified] +constexpr uint32_t TechTree_off_OrderCounter = 0x00000020; // offset sizeof(TechNode) -- the ctor's operator new argument [verified] constexpr uint32_t TechNode_size = 0x00000034; // offset TechDef* def; *(int*)def is the tech id used to index TechTree_off_Nodes [verified] @@ -259,6 +265,18 @@ constexpr uint32_t TechNode_off_State = 0x00000014; constexpr uint32_t TechNode_off_CostRP = 0x00000018; // offset int progress in RP; the only node word ProcessResearch itself writes besides the flag [verified] constexpr uint32_t TechNode_off_Progress = 0x0000001c; +// offset int turnAvailable; SetResearched's second sweep stamps it with the server ModCount ONLY when it currently reads -1 (a sticky first-availability stamp). EVENT_TECHS_UNLOCKED collects nodes with state==2 && turnAvailable==currentTurn [verified] +constexpr uint32_t TechNode_off_TurnAvailable = 0x00000020; +// offset int turnResearched; SetResearched writes the server ModCount [verified] +constexpr uint32_t TechNode_off_TurnResearched = 0x00000024; +// offset int order; SetResearched writes the tree's completion-order counter, then bumps it [verified] +constexpr uint32_t TechNode_off_Order = 0x00000028; +// offset std::vector children (3 words at +0x04/+0x08/+0x0c) [verified] +constexpr uint32_t TechNode_off_Children = 0x00000004; +// offset int RP cost carried by the edge; SetResearched sets child.costRP = min(child.costRP, edge.costRP) [verified] +constexpr uint32_t TechEdge_off_CostRP = 0x0000001c; +// offset TechDef* the edge's child tech; SetResearched indexes tree->nodes by childDef->[0] [verified] +constexpr uint32_t TechEdge_off_ChildDef = 0x00000040; // offset int flag (1 default from the ctor, 0 completed below 80% of cost, 2 over-budget event raised) [verified] constexpr uint32_t TechNode_off_Flag = 0x0000002c; // thiscall void (MasterTechTree* this) /* fills TechDef*[196] at this+0 from g_TechIdNames */ [verified] @@ -351,6 +369,8 @@ constexpr uint32_t ServerPlayer_off_SetupResearchMult = 0x0000022c; constexpr uint32_t ServerPlayer_off_Sav = 0x00000284; // offset Tech* current research target (ResT); NULL = none [verified-by-save] constexpr uint32_t ServerPlayer_off_ResearchTarget = 0x00000294; +// offset std::vector otch (3 words {first,last,end}); save tag otch, element {int otnF, otnL, odet; string otch; int owith}. All three words move on every tech completion (a realloc) -- observed live by both the ProcessResearch and the OnTechResearched player guards. NOT PINNED: the append call site and sizeof(ObservedTech); the byte span the observed_techs region reports is what will measure the stride [verified-by-save] +constexpr uint32_t ServerPlayer_off_ObservedTechs = 0x00000274; // offset float IncMod [verified-by-save] constexpr uint32_t ServerPlayer_off_IncMod = 0x0000030c; // offset std::vector (3 words; entry 0x18 B, {+0x8 int researchPercent, +0xc int researchActive, +0x10 int savings, +0x14 int savingsActive}) [verified] @@ -767,5 +787,17 @@ constexpr uint32_t g_ptr_EVENTMSG_ADDICTION_TEMPERENCE = 0x006f0a94; constexpr uint32_t g_dbl_ResearchUnderbudgetThreshold = 0x005e20c8; // data double 0.0 -- the research completion draw is NextFloat()*(1.0-this)+this, so it is a plain NextFloat() [verified] constexpr uint32_t g_dbl_ResearchRollBias = 0x005e1e68; +// cdecl float (Vec3* out, const Vec3* in) /* normalises in-place-capable (MoveFleet passes the same pointer twice) and RETURNS THE LENGTH. FOUR float32 narrowings in this order: sumsq = float32(x*x+y*y+z*z) (products/adds stay in the x87 53-bit registers, only the sum is stored to a dword and reloaded); len = float32(sqrt(sumsq)); inv = float32(1.0/len) -- a RECIPROCAL, stored to a dword and reloaded, then MULTIPLIED through, NOT three divides; out.c = float32(in.c * inv). Zero branch: !(len > 2^-23, the float at 0x009e1ef8) => out = {0,0,0} and it returns with an EMPTY x87 stack, i.e. no return value at all (original bug, only reachable on a zero-length vector). 123 callers. */ [verified] +constexpr uint32_t Mars_Vec3_Normalize = 0x00022520; +// cdecl float (const Vec3* v) /* float32(sqrt(float32(x*x+y*y+z*z))) -- the same two narrowings as the first half of Mars_Vec3_Normalize */ [verified] +constexpr uint32_t Mars_Vec3_Length = 0x000224b0; +// cdecl float (const Vec3* v) /* float32(x*x+y*y+z*z), one narrowing on the sum */ [verified] +constexpr uint32_t Mars_Vec3_LengthSquared = 0x000224f0; +// data const float = 0x34000000 = 2^-23 = 1.1920928955078125e-07 /* Mars_Vec3_Normalize zeroes the direction when !(len > this) */ [verified] +constexpr uint32_t Mars_Vec3_NormaliseEpsilon = 0x005e1ef8; +// site site inside StrategyServer::MoveFleet /* the straight-run leg. Each delta dest.c - fleet.pos.c is computed on the x87 stack and STORED BACK TO A FLOAT32 SLOT before Mars_Vec3_Normalize(&v, &v) is called on it in place; that single call returns the leg DISTANCE and leaves the float32 unit direction in the same slots. The distance is never recomputed. */ [verified] +constexpr uint32_t StrategyServer_MoveFleet_straight_leg = 0x003da0f2; +// site site inside StrategyServer::MoveFleet /* the move != distance branch: exactly two roundings per component, tmp.c = float32(dir.c * move) then pos.c = float32(pos.c + tmp.c). `move` is reloaded from a float32 slot. The sibling branch (move == distance, an EXACT float compare) copies the destination's three words verbatim with mov, so an arrival never steps onto its destination. */ [verified] +constexpr uint32_t StrategyServer_MoveFleet_position_update = 0x003da2ac; } // namespace sots::addr diff --git a/src/game/events/research_events.cpp b/src/game/events/research_events.cpp index 968d175..61db46a 100644 --- a/src/game/events/research_events.cpp +++ b/src/game/events/research_events.cpp @@ -85,4 +85,44 @@ int PostNoResearch(EventStorage& log, const EventText& text, int turn) { return log.PostGlobal(std::move(summary), std::move(message), turn, kImgNoResearch, 1); } +ResearchPassResult PostResearchPassEvents(EventStorage& log, const EventText& text, + const std::vector& outcomes, + const std::vector* unlockedTechNames, + int turn) { + ResearchPassResult r; + for (const ResearchPassOutcome& o : outcomes) { + // The two are the failed and the succeeded side of one roll, so at most one fires. + if (o.overbudgetEvent) { + r.ids.push_back(PostResearchOverbudget(log, text, o.techName, turn)); + ++r.overbudgetPosts; + } else if (o.completed) { + // PostResearchCompleted picks the image from the ratio; feed it a ratio on the + // correct side of the threshold rather than re-deriving one, so the two callers + // cannot drift apart. Both branches post exactly one event, so the count is the + // same either way. + const float ratio = o.completedEarly ? 0.f : 1.f; + r.ids.push_back(PostResearchCompleted(log, text, o.techName, ratio, turn)); + ++r.completionPosts; + } + } + if (unlockedTechNames && !unlockedTechNames->empty()) { + r.ids.push_back( + PostTechsUnlocked(log, text, *unlockedTechNames, kTechsUnlockedSeparator, turn)); + ++r.techsUnlockedPosts; + } + r.nextId = log.nextId(); + return r; +} + +namespace { +std::string_view KeylessLookup(void*, std::string_view) { return std::string_view("%s"); } +} // namespace + +EventText KeylessEventText() { + EventText t; + t.lookup = &KeylessLookup; + t.ctx = nullptr; + return t; +} + } // namespace sots::events diff --git a/src/game/events/research_events.h b/src/game/events/research_events.h index 7bb84f1..8fc61f5 100644 --- a/src/game/events/research_events.h +++ b/src/game/events/research_events.h @@ -130,4 +130,71 @@ int PostTechsUnlocked(EventStorage& log, const EventText& text, // `EvAct = 1`; both strings come from the key pair unformatted. int PostNoResearch(EventStorage& log, const EventText& text, int turn); +// --------------------------------------------------------------------------------------- +// The events one `TechTree::ProcessResearch` pass raises +// --------------------------------------------------------------------------------------- +// +// This is the *decision* half of the research event path, split out so it can be driven from +// a hook and pinned by host tests. It deliberately covers only what ProcessResearch's own +// arithmetic determines. See docs/P-events-wiring.md for what it does not. + +// One allocation entry's outcome, as far as the event path can see. The three flags are exactly +// `sim::ResearchStepResult::overbudgetEvent / completed / completedEarly`. +struct ResearchPassOutcome { + // The substitution for the message's single `%s`. Its only load-bearing property is that + // two techs the original would keep apart carry different tokens here: the dedup comparator + // compares `message`, so a caller that collapses two techs to the same token would post one + // event where the original posts two. + std::string techName; + bool overbudgetEvent = false; // the node crossed its cost this turn and the roll FAILED + bool completed = false; // the roll succeeded -> SetResearched -> OnTechResearched + bool completedEarly = false; // progress/cost < (double)0.8f -> UNDERBUDGET, else COMPLETE +}; + +struct ResearchPassResult { + std::vector ids; // the ids Post returned, in call order (a dedup repeats an id) + int overbudgetPosts = 0; + int completionPosts = 0; + int techsUnlockedPosts = 0; + int nextId = 0; // log.nextId() after the pass +}; + +// Post, in the original's order, every event one ProcessResearch call raises: +// +// * per allocation entry, inside the loop: EVENT_RESEARCH_OVERBUDGET when the node crossed +// its cost this turn and the completion roll failed, OR -- through SetResearched -> +// OnTechResearched, which ProcessResearch invokes with flags = 2 so the callback's `silent` +// argument ((flags >> 2) & 1) is FALSE -- EVENT_RESEARCH_COMPLETE / _UNDERBUDGET on a +// completion. The two are mutually exclusive: they are the failed and the succeeded side of +// the same roll. +// * once after the loop: EVENT_TECHS_UNLOCKED, for the nodes that became available this turn. +// +// `unlockedTechNames` is that list. Passing **nullptr** means "this caller cannot compute the +// unlock set" -- the tail event is then not posted and `techsUnlockedPosts` stays 0. That is +// the shim's case: the set comes from SetResearched's child-unlock cascade, which `ours` does +// not run. An empty (non-null) list means the caller computed the set and it was empty, which +// is also "no post". The distinction is deliberate: a missing input must not look like a +// modelled negative. +// +// EVENT_TEMPERANCE is NOT raised here. It is posted from the same OnTechResearched call, but +// only when the completed tech is a temperance tech AND the sweep actually cured at least one +// addicted system -- a condition outside this function's inputs. CONFIDENCE: high for the two +// modelled triggers (both read out of the instruction stream); the split between COMPLETE and +// UNDERBUDGET is modelled by analogy (see docs/P-events-wiring.md) and does not affect the +// count, since both post exactly one event. +ResearchPassResult PostResearchPassEvents(EventStorage& log, const EventText& text, + const std::vector& outcomes, + const std::vector* unlockedTechNames, + int turn); + +// A `TextLookup` for a caller that has no string table -- the shim, which must never carry the +// game's localized prose. Every key resolves to the bare format `"%s"`, so the composed +// `message` is exactly the caller's `techName` token. +// +// This is enough, and only enough, for a **count-only** model: the dedup comparator tests +// `action`, `location`, `pos`, `message` and `image`, and this lookup preserves the one property +// that matters -- two events the original keeps apart stay apart, and two it collapses collapse. +// The resulting text is NOT the game's text and must never be shown or serialized. +EventText KeylessEventText(); + } // namespace sots::events diff --git a/src/shim/hooks/event_inputs.cpp b/src/shim/hooks/event_inputs.cpp new file mode 100644 index 0000000..b39e661 --- /dev/null +++ b/src/shim/hooks/event_inputs.cpp @@ -0,0 +1,185 @@ +#include "shim/hooks/event_inputs.h" + +#include +#include + +#include "game/events/research_events.h" +#include "generated/sots_addresses.h" + +namespace shim::hooks { + +namespace A = sots::addr; + +namespace { + +std::int32_t word_at(const void* obj, std::size_t off) { + std::int32_t v = 0; + std::memcpy(&v, static_cast(obj) + off, sizeof v); + return v; +} +void set_word(void* obj, std::size_t off, std::int32_t v) { + std::memcpy(static_cast(obj) + off, &v, sizeof v); +} +// A pointer stored *inside a game object*: four bytes, whatever the host's word size is. +std::uint32_t addr_at(const void* obj, std::size_t off) { + std::uint32_t v = 0; + std::memcpy(&v, static_cast(obj) + off, sizeof v); + return v; +} +void set_addr(void* obj, std::size_t off, std::uint32_t v) { + std::memcpy(static_cast(obj) + off, &v, sizeof v); +} + +// A record's EvImg is one of the six the research path uses. +bool is_research_image(const std::string& img) { + using namespace sots::events; + return img == kImgResearchOverbudget || img == kImgResearchComplete || + img == kImgResearchUnderbudget || img == kImgTemperance || img == kImgTechsUnlocked || + img == kImgNoResearch; +} + +// A 3-word MSVC vector header {first, last, end} living in game memory. Returns false when it +// does not look like one; an all-null header is a valid empty vector. +struct GameVector { + std::uint32_t first = 0; + std::size_t count = 0; + std::size_t bytes = 0; + const void* host = nullptr; // `first` mapped into something readable, null when empty +}; + +bool read_vector(const void* vec, std::size_t stride, std::size_t cap, ReadProbe readable, + ToHost toHost, GameVector& out, bool& truncated) { + out = GameVector{}; + const std::uint32_t first = addr_at(vec, 0); + const std::uint32_t last = addr_at(vec, kGamePtrSize); + if (!first && !last) return true; // empty + if (!first || last < first) return false; + const std::size_t span = last - first; + if (span % stride != 0) return false; + if (span / stride > cap) { + truncated = true; + return false; + } + out.first = first; + out.bytes = span; + out.count = span / stride; + if (out.count == 0) return true; + out.host = toHost(first); + if (!readable(out.host, span)) return false; + return true; +} + +} // namespace + +const void* IdentityToHost(std::uint32_t gameAddr) { + if constexpr (sizeof(void*) == kGamePtrSize) { + return reinterpret_cast(static_cast(gameAddr)); + } else { + // A 64-bit build is not in the game's address space, so there is nothing to point at. + // Returning null makes the read probe reject it rather than fabricating an address that + // happens to be mappable. + (void)gameAddr; + return nullptr; + } +} + +bool ReadStdStringMapped(const void* p, ReadProbe readable, ToHost toHost, std::string& out) { + if (!readable(p, kStdStringSize)) return false; + const std::uint32_t size = static_cast(word_at(p, kStdStringOffSize)); + const std::uint32_t res = static_cast(word_at(p, kStdStringOffRes)); + // The event-type identifiers are short interface names; anything long is a bad read, not a + // long string worth chasing. `res < size` is impossible in a well-formed string. + if (size > 0x1000 || res < size) return false; + if (res < kStdStringInlineCap) { + if (size >= kStdStringInlineCap) return false; + out.assign(static_cast(p), size); + return true; + } + const std::uint32_t buf = addr_at(p, 0); + if (!buf) return false; + const void* host = toHost(buf); + if (size && !readable(host, size)) return false; + out.assign(static_cast(host), size); + return true; +} + +EventStorageScan ScanEventStorage(const void* storage, int turn, ReadProbe readable, + ToHost toHost) { + EventStorageScan s; + if (!readable(storage, A::EventStorage_sizeof)) return s; + + GameVector buckets; + if (!read_vector(static_cast(storage) + A::EventStorage_off_Events, + A::TurnEvents_sizeof, kMaxTurnBuckets, readable, toHost, buckets, + s.scanTruncated)) + return s; + + s.nextId = word_at(storage, A::EventStorage_off_EvNxID); + s.turnsBytes = buckets.bytes; + s.bucketTurns.reserve(buckets.count); + for (std::size_t i = 0; i < buckets.count; ++i) { + const char* b = static_cast(buckets.host) + i * A::TurnEvents_sizeof; + const int bt = word_at(b, A::TurnEvents_off_EvTurn); + s.bucketTurns.push_back(bt); + if (bt != turn) continue; + + // GetOrCreateTurnBucket returns the LAST matching bucket, so a later duplicate bucket + // replaces what an earlier one contributed rather than adding to it. + s.turnBucketExists = true; + s.eventsInTurnBucket = 0; + s.researchEventsInTurnBucket = 0; + + GameVector evs; + if (!read_vector(b + A::TurnEvents_off_Events, A::PlayerEvent_sizeof, kMaxEventsPerBucket, + readable, toHost, evs, s.scanTruncated)) { + s.scanTruncated = true; + continue; + } + s.eventsInTurnBucket = evs.count; + for (std::size_t k = 0; k < evs.count; ++k) { + const char* rec = static_cast(evs.host) + k * A::PlayerEvent_sizeof; + std::string img; + if (!ReadStdStringMapped(rec + A::PlayerEvent_off_EvImg, readable, toHost, img)) { + s.scanTruncated = true; + break; + } + if (is_research_image(img)) ++s.researchEventsInTurnBucket; + } + } + s.ok = true; + return s; +} + +sots::events::EventStorage SeedFromScan(const EventStorageScan& scan) { + sots::events::EventStorage log; + log.set_nextId(scan.nextId); + for (int t : scan.bucketTurns) { + sots::events::TurnEvents b; + b.turn = t; + log.turns().push_back(std::move(b)); + } + return log; +} + +void WriteBackCounts(void* scratchStorage, const sots::events::EventStorage& log, + ReadProbe readable) { + if (!readable(scratchStorage, A::EventStorage_sizeof)) return; + set_word(scratchStorage, A::EventStorage_off_EvNxID, log.nextId()); + std::uint32_t first = addr_at(scratchStorage, A::EventStorage_off_Events); + if (!first) { + // A player who has never seen an event carries {null, null, null}, and two of the four + // players in the reference save do. If our post creates that player's first bucket the + // original allocates and its span becomes non-zero, so writing nothing here would show up + // as a divergence in `turns_bytes` for the wrong reason. The region diff ignores pointer + // *values* (HookPolicy::ptr_exact is false) and compares only the span, so a synthetic + // base carries the meaning without pretending to name a heap block. Nothing dereferences + // it: the scratch copy is a byte buffer the describer only reads through. + if (log.turns().empty()) return; + first = kSyntheticTurnsBase; + set_addr(scratchStorage, A::EventStorage_off_Events, first); + } + set_addr(scratchStorage, A::EventStorage_off_Events + kGamePtrSize, + first + static_cast(log.turns().size() * A::TurnEvents_sizeof)); +} + +} // namespace shim::hooks diff --git a/src/shim/hooks/event_inputs.h b/src/shim/hooks/event_inputs.h new file mode 100644 index 0000000..9971ee1 --- /dev/null +++ b/src/shim/hooks/event_inputs.h @@ -0,0 +1,121 @@ +// Adapter between a live `Game::EventStorage` and the pure `sots::events` model. +// +// The B3 hook needs three things the event module cannot get for itself, because they live in +// the game's own memory: +// +// 1. what the owner's event list looked like **before** the original ran (its `EvNxID`, its +// turn buckets, and whether the current turn's bucket already holds a research event that +// our post could be deduplicated against); +// 2. a model `EventStorage` seeded from that, so `ours` posts into the same shape; +// 3. a way to write the *counts* back into a scratch copy of the storage header, which is what +// the `events` region compares. +// +// It is a separate library from the hook so it can be built and tested on the host: every read +// goes through a caller-supplied bounds probe, so the tests drive it on a plain byte buffer laid +// out with the real offsets and the shim drives it on live game memory with `VirtualQuery`. +// +// Nothing here composes or reads event *text* for the model. `ScanEventStorage` does read each +// record's `EvImg` -- the event-type identifier, an ASCII interface name, not localized prose -- +// because that is the only field that can make a duplicate possible. +// +// Layout facts all come from the generated header (lane E read them off the instruction stream): +// EventStorage +0x04 vector {first,last,end} +0x14 EvNxID sizeof 0x1c +// TurnEvents +0x04 EvTurn +0x08 vector sizeof 0x18 +// PlayerEvent +0x50 EvImg (std::string) sizeof 0x74 +#pragma once + +#include +#include +#include +#include + +#include "game/events/event_log.h" + +namespace shim::hooks { + +// "Are `n` bytes at `p` safe to read?" The shim answers with VirtualQuery; the host tests answer +// against their buffer. A null pointer must answer false. +using ReadProbe = bool (*)(const void* p, std::size_t n); + +// The game is a 32-bit process, so every pointer *inside* its objects is four bytes. The shim is +// built 32-bit and could just dereference them; the host tests are 64-bit and cannot. So every +// game pointer read out of an object is carried as an explicit `uint32` address and turned into +// something readable through this hook -- a cast in the shim, an arena lookup in the tests. +// Keeping the width explicit is not only what makes the adapter host-testable: it is also what +// stops a 64-bit host build from silently reading the vector's `last` word on top of its `first`. +using ToHost = const void* (*)(std::uint32_t gameAddr); + +// The identity mapping the shim uses: in a 32-bit build a game address IS a host address. In a +// 64-bit build it returns null, so a caller that forgot to supply a real mapping fails the read +// probe instead of dereferencing a fabricated pointer. +const void* IdentityToHost(std::uint32_t gameAddr); + +// Width of a pointer inside a game object, always 4 regardless of the host. +inline constexpr std::size_t kGamePtrSize = 4; + +// MSVC 2010 `std::string`: a 16-byte union that is either the inline buffer or a heap pointer, +// then `_Mysize` at +0x10 and `_Myres` at +0x14. `_Myres >= 16` selects the heap pointer. The +// same layout lane E used to reach a TechDef's name through `_Myres` at def+0x54. +inline constexpr std::size_t kStdStringSize = 0x1c; +inline constexpr std::size_t kStdStringOffSize = 0x10; +inline constexpr std::size_t kStdStringOffRes = 0x14; +inline constexpr std::size_t kStdStringInlineCap = 16; + +// Copies the string's bytes out of game memory, handling both the inline and the heap form. +// False (and `out` untouched) when any part of it fails the probe or the length is implausible -- +// a bad read is reported, never guessed at. +bool ReadStdStringMapped(const void* p, ReadProbe readable, ToHost toHost, std::string& out); + +// A read-only view of a live EventStorage, taken BEFORE the original runs. +// +// Why "before" matters: in compare mode `ours` runs *after* the original, so a scan taken then +// would show the original's own freshly posted events and our model would deduplicate against +// them -- silently posting nothing and "matching" for the wrong reason. +struct EventStorageScan { + bool ok = false; // false: the header did not look like an EventStorage + int nextId = 0; // EvNxID as it stood before the call + std::vector bucketTurns; // EvTurn of every TurnEvents, in order + std::size_t turnsBytes = 0; // the outer vector's byte span (last - first) + + // How many events the bucket for the scanned turn already holds, and how many of those carry + // one of the research `EvImg` identifiers. The second number is the **dedup risk**: our model + // rebuilds the buckets but not their contents, so it can only fail to deduplicate against a + // pre-existing record, and only a record with the same `EvImg` can ever be a duplicate. When + // it is 0, a count-only model is exact; when it is not, the count is a lower bound and the + // run should say so rather than quietly assume. + std::size_t eventsInTurnBucket = 0; + std::size_t researchEventsInTurnBucket = 0; + bool turnBucketExists = false; + bool scanTruncated = false; // a vector looked longer than the sanity caps: treat as unknown +}; + +// Loop guards, so a garbage vector header cannot walk the whole address space. +inline constexpr std::size_t kMaxTurnBuckets = 4096; +inline constexpr std::size_t kMaxEventsPerBucket = 4096; + +EventStorageScan ScanEventStorage(const void* storage, int turn, ReadProbe readable, + ToHost toHost); + +// A model storage with the same turn buckets and the same `EvNxID` as the scan, and empty +// buckets. Empty because the records' text is the game's, not ours: see the dedup-risk note +// above for exactly what that costs. +sots::events::EventStorage SeedFromScan(const EventStorageScan& scan); + +// Write what `ours` posted back into a **scratch copy** of the EventStorage header, in the +// storage's own layout, so the `events` region's describer reads it the same way for both sides: +// +// * `EvNxID` <- log.nextId() +// * the turns vector's `last` pointer <- `first + turns * sizeof(TurnEvents)`, so the region's +// `turns_bytes` reflects a bucket our post created. The pointer *values* are the original's +// (the region diff ignores pointers); only their difference carries meaning. +// +// MUST NOT be called on live game memory: the header would then claim an id and a bucket that +// no serialized record backs. Compare mode only. +void WriteBackCounts(void* scratchStorage, const sots::events::EventStorage& log, + ReadProbe readable); + +// Stand-in `first` address used only when the scanned storage held an empty {null,null,null} +// vector and our post created its first bucket -- see WriteBackCounts. +inline constexpr std::uint32_t kSyntheticTurnsBase = 0x10000; + +} // namespace shim::hooks diff --git a/src/shim/hooks/research.cpp b/src/shim/hooks/research.cpp index 2c7f23f..75c95be 100644 --- a/src/shim/hooks/research.cpp +++ b/src/shim/hooks/research.cpp @@ -12,10 +12,13 @@ #include #endif +#include "game/events/event_log.h" +#include "game/events/research_events.h" #include "game/sim/research.h" #include "game/sim/species.h" #include "generated/sots_addresses.h" #include "mars/rng/mt19937.h" +#include "shim/hooks/event_inputs.h" namespace shim::hooks { @@ -40,6 +43,17 @@ constexpr std::size_t kPlayerEventsOff = A::ServerPlayer_off_Events; // 0x29c constexpr std::size_t kEventStorageSize = A::EventStorage_sizeof; // 0x1c constexpr std::size_t kEventsVecOff = A::EventStorage_off_Events; // 0x04 constexpr std::size_t kEventsNextIdOff = A::EventStorage_off_EvNxID; // 0x14 +// The owner's vector (save tag `otch`). Lane R's player guard caught all three of +// its words moving on both completion calls, in both this hook and OnTechResearched -- a third +// serialized list append in the same neighbourhood as the event list, named in no coverage note +// anywhere. Declaring it turns an undeclared write into a check. +constexpr std::size_t kObservedTechsOff = A::ServerPlayer_off_ObservedTechs; // 0x274 +constexpr std::size_t kVectorHeaderSize = 0xc; // {first,last,end} +// The turn every event is filed under: the owner's second StrategyServer base is at +// ServerPlayer+8 and the turn counter (ModCount) sits at +8 in it. Read out of the instruction +// stream twice -- lane E on every research-path post site, and again at 0x00581e10 where +// SetResearched stamps node.turnResearched from the same chain. +constexpr std::size_t kPlayerServerOff = 8; // Whole-object guard spans. ServerPlayer is 0x3e0 and the TechTree header we care about ends // at the order counter (+0x20). Source: findings/control-flow/turn-spine.md object table. constexpr std::size_t kPlayerSize = 0x3e0; @@ -134,12 +148,39 @@ struct CallState { std::size_t rng_region = 0; std::size_t ob_region = 1; int events_region = -1; // the owner's EventStorage, -1 when the owner is unreadable + int otch_region = -1; // the owner's vector + void* scratch_events = nullptr; // compare mode: the scratch copy of the EventStorage header std::vector node_region; // node index -> region index, -1 when the slot is null std::vector names; // stable storage for Region::name std::vector scratch_nodes; }; CallState g_call; +// The pre-call read of the owner's event list. +// +// It lives outside CallState on purpose. `describe_args` runs immediately before `regions()` for +// the same call (hook.h), and `regions()` resets CallState -- so the scan is taken in +// describe_args, where its numbers can also be reported as arguments in *every* mode, and read +// back in regions()/ours(). ProcessResearch is called once per player from the single-threaded +// turn pass and is never re-entrant, the same concession the CallState mapping already makes. +struct EventScan { + bool owner_ok = false; + bool turn_ok = false; + int turn = 0; + EventStorageScan storage; +}; +EventScan g_scan; + +// The turn the events are filed under; `ok` is false when the chain is not walkable. +int current_turn(const void* owner, bool& ok) { + ok = false; + if (!readable(owner, kPlayerServerOff + 4)) return 0; + const void* srv = ptr_at(owner, kPlayerServerOff); + if (!readable(srv, A::StrategyServer_off_ModCount + 4)) return 0; + ok = true; + return word_at(srv, A::StrategyServer_off_ModCount); +} + // The tree's node vector, or false when the header does not look like one. bool tree_nodes(const void* tree, std::vector& out) { out.clear(); @@ -239,14 +280,21 @@ Tv describe_i32(const void* p, std::size_t, unsigned) { // The owner's inline EventStorage. The three vector words are heap pointers (ignored by the // default policy), so what the diff actually compares is `turns` -- how many TurnEvents the -// list holds -- and `next_id`, the counter every posted event bumps. Posting the -// EVENT_RESEARCH_OVERBUDGET message moves `next_id`, so a run where ours does not post it now -// diverges here instead of passing silently. That is the whole point of this region. +// list holds -- and `next_id`, the counter every posted event bumps. +// +// `ours` now posts the research events into a model storage seeded from the pre-call scan and +// writes the resulting counts back into the *scratch* copy of this header, so both sides of the +// diff are produced the same way. Before that wiring, this region reported the defect; now it +// checks it. `turns_bytes` was an element count the harness audit could not compute because the +// TurnEvents stride was unknown -- lane E read it (0x18), so `turns` is emitted too. Tv describe_events(const void* p, std::size_t, unsigned) { Tv s = tv::struct_(); const char* begin = static_cast(ptr_at(p, kEventsVecOff)); const char* end = static_cast(ptr_at(p, kEventsVecOff + 4)); - s.add("turns_bytes", tv::i32(end >= begin ? static_cast(end - begin) : -1)); + const std::int32_t bytes = end >= begin ? static_cast(end - begin) : -1; + s.add("turns_bytes", tv::i32(bytes)); + s.add("turns", tv::i32(bytes >= 0 ? bytes / static_cast(A::TurnEvents_sizeof) + : -1)); s.add("next_id", tv::i32(word_at(p, kEventsNextIdOff))); s.add("vec_begin", tv::ptr(ptr_at(p, kEventsVecOff))); s.add("vec_end", tv::ptr(ptr_at(p, kEventsVecOff + 4))); @@ -254,6 +302,26 @@ Tv describe_events(const void* p, std::size_t, unsigned) { return s; } +// The owner's `vector otch` (ServerPlayer+0x274) -- serialized state that grows on +// every tech completion and that no coverage note in B2 or B3 mentioned. +// +// `ours` cannot append to it: `sizeof(ObservedTech)` and the append call site are both unpinned +// (the element is a Streamable with four ints and a string on disk, which bounds it but does not +// name it). So this region is declared knowing it will diverge on a completion call -- and the +// **byte delta it reports is what measures the stride**: one completion appends one element, so +// `bytes` growing by n names sizeof(ObservedTech) = n. That is the point of declaring it now +// rather than waiting for the element layout. +Tv describe_otch(const void* p, std::size_t, unsigned) { + Tv s = tv::struct_(); + const char* begin = static_cast(ptr_at(p, 0)); + const char* end = static_cast(ptr_at(p, 4)); + s.add("bytes", tv::i32(end >= begin ? static_cast(end - begin) : -1)); + s.add("vec_begin", tv::ptr(ptr_at(p, 0))); + s.add("vec_end", tv::ptr(ptr_at(p, 4))); + s.add("vec_cap", tv::ptr(ptr_at(p, 8))); + return s; +} + // ---- the generator seen by ours ---------------------------------------------------------- struct ShimRandom final : sots::sim::IRandom { @@ -308,6 +376,36 @@ void TechTreeProcessResearchHook::describe_args(std::vector& out, void* tree out.push_back(tv::i32(readable(overbudget, 4) ? word_at(overbudget, 0) : 0).named("overbudget_in")); // The x87 precision mode in force for this call (see fpu_control_word above). out.push_back(tv::u32(fpu_control_word()).named("fpu_cw")); + + // ---- the pre-call read of the owner's event list ---------------------------------------- + // + // Taken here, not in ours(): in compare mode ours runs AFTER the original, so a scan taken + // then would see the original's own posts and our model would deduplicate against them -- + // posting nothing and "agreeing" for exactly the wrong reason. + g_scan = EventScan{}; + g_scan.owner_ok = readable(owner, kPlayerEventsOff + kEventStorageSize); + g_scan.turn = current_turn(owner, g_scan.turn_ok); + if (g_scan.owner_ok && g_scan.turn_ok) { + // The shim is a 32-bit build living in the game's own address space, so a game pointer + // maps to a host pointer by a cast (the adapter static_asserts the width). + g_scan.storage = ScanEventStorage(static_cast(owner) + kPlayerEventsOff, + g_scan.turn, &readable, &IdentityToHost); + } + out.push_back(tv::i32(g_scan.turn_ok ? g_scan.turn : -1).named("turn")); + out.push_back(tv::i32(g_scan.storage.ok ? g_scan.storage.nextId : -1).named("events_next_id_in")); + out.push_back(tv::u32(static_cast(g_scan.storage.bucketTurns.size())) + .named("events_turns_in")); + out.push_back(tv::boolean(g_scan.storage.turnBucketExists).named("events_turn_bucket_exists")); + out.push_back(tv::u32(static_cast(g_scan.storage.eventsInTurnBucket)) + .named("events_in_turn_bucket")); + // THE assumption behind a count-only model, measured instead of assumed. Our model rebuilds + // the turn buckets but not their contents, so it can only ever fail to deduplicate against a + // record that was already there -- and only a record carrying one of the six research + // `EvImg` identifiers can be a duplicate of anything we post. 0 means count-only is exact on + // this call; anything else means our id count is a lower bound and the run must say so. + out.push_back(tv::u32(static_cast(g_scan.storage.researchEventsInTurnBucket)) + .named("events_dedup_risk")); + if (g_scan.storage.scanTruncated) out.push_back(tv::boolean(true).named("events_scan_truncated")); } void TechTreeProcessResearchHook::regions(std::vector& out, void* tree, void* rng, @@ -358,8 +456,9 @@ void TechTreeProcessResearchHook::regions(std::vector& out, void* // The owner's event storage. This is the region whose absence made B3's compare clean while // replace mode diverged: the over-budget branch posts EVENT_RESEARCH_OVERBUDGET, which bumps - // EvNxID, and nothing declared here could see it. `ours` still does not post the event, so - // the divergence this region now reports is real and expected -- visible instead of silent. + // EvNxID, and nothing declared here could see it. `ours` now posts into a model storage and + // writes the counts into this region's scratch copy, so the region is a check and not just an + // alarm -- see ours() and docs/P-events-wiring.md. void* owner = readable(tree, kTreeHeadSize) ? ptr_at(tree, A::TechTree_off_Owner) : nullptr; g_call.events_region = -1; if (readable(owner, kPlayerEventsOff + kEventStorageSize)) { @@ -372,6 +471,18 @@ void TechTreeProcessResearchHook::regions(std::vector& out, void* out.push_back(ev); } + // The owner's vector. Declared, not modelled: see describe_otch. + g_call.otch_region = -1; + if (readable(owner, kObservedTechsOff + kVectorHeaderSize)) { + g_call.otch_region = static_cast(out.size()); + trace::Region ot; + ot.name = "observed_techs"; + ot.ptr = static_cast(owner) + kObservedTechsOff; + ot.size = kVectorHeaderSize; + ot.describe = &describe_otch; + out.push_back(ot); + } + // ---- guards: pushed LAST so every recorded index above is also a Scratch index ---------- // // Coarse spans that no reimplementation writes. Anything the ORIGINAL moves inside them and @@ -407,6 +518,10 @@ TechTreeProcessResearchHook::Args TechTreeProcessResearchHook::rebind(trace::Scr if (g_call.node_region[i] >= 0) g_call.scratch_nodes[i] = s.ptr(static_cast(g_call.node_region[i])); } + // The event storage is not one of the four parameters, so its scratch copy travels the same + // way the node copies do: through CallState, resolved here where Scratch is in hand. + g_call.scratch_events = + g_call.events_region >= 0 ? s.ptr(static_cast(g_call.events_region)) : nullptr; g_call.compare = true; // The tree pointer is passed through unchanged: ours only reads it (owner, node count) and // hands it to the game's own read-only Cost. Every node it writes is a scratch copy. @@ -499,6 +614,58 @@ void TechTreeProcessResearchHook::ours(void* tree, void* rng, void* alloc, int* set_ptr(rng, A::RNG_off_Next, reinterpret_cast(base + static_cast(rand.gen.index()) * 4)); } + + // ---- the events this pass posts (count-only) -------------------------------------------- + // + // Lane E's option (a): `ours` posts into its OWN sots::events::EventStorage, seeded from the + // pre-call scan, and the counts go into the SCRATCH copy of the owner's storage header. The + // game's own EventStorage::PostEvent is never called and no live byte is written, so this + // cannot perturb the VM -- and it is the reason the write below is gated on `compare`. + // + // What that proves and what it does not: it proves the *decision* (did this pass post, how + // many, in what id order) and it makes `region:events` a check. It does not prove the text, + // which the game composes from its own string table -- see docs/P-events-wiring.md. + // + // In replace mode nothing is written: an EvNxID bumped without a serialized record behind it + // would corrupt the save the oracle hashes, which is strictly worse than the missing event. + if (compare && g_call.scratch_events && g_scan.storage.ok && g_scan.turn_ok) { + sots::events::EventStorage log = SeedFromScan(g_scan.storage); + const sots::events::EventText text = sots::events::KeylessEventText(); + + // ProcessResearchTurn emits one step per allocation entry **that was in range**, so the + // steps are not index-aligned with `entries` when the budget names a tech this tree does + // not have. Mirror its filter to recover which node each step belongs to. (Duplicating + // the predicate is the price of not adding a field to the shared sim header while lane M + // is working in src/game/sim.) + std::vector step_node; + step_node.reserve(entries.size()); + for (const ResearchAllocEntry& e : entries) { + if (e.nodeIndex < 0 || static_cast(e.nodeIndex) >= model.size()) continue; + step_node.push_back(e.nodeIndex); + } + if (step_node.size() != r.steps.size()) + throw std::runtime_error("research step/entry mapping is stale"); + + std::vector outcomes; + outcomes.reserve(r.steps.size()); + for (std::size_t i = 0; i < r.steps.size(); ++i) { + sots::events::ResearchPassOutcome o; + // The token only has to separate techs whose messages the original separates; the + // node's own index does that and needs no string table. See KeylessEventText. + char buf[24]; + std::snprintf(buf, sizeof buf, "%d", step_node[i]); + o.techName = buf; + o.overbudgetEvent = r.steps[i].overbudgetEvent; + o.completed = r.steps[i].completed; + o.completedEarly = r.steps[i].completedEarly; + outcomes.push_back(std::move(o)); + } + // nullptr, not an empty list: the EVENT_TECHS_UNLOCKED set comes from SetResearched's + // child-unlock cascade, which `ours` deliberately does not run. Passing an empty list + // would claim we computed the set and found it empty. + sots::events::PostResearchPassEvents(log, text, outcomes, nullptr, g_scan.turn); + WriteBackCounts(g_call.scratch_events, log, &readable); + } } void init_research(std::uintptr_t exe_base, void (*log_line)(const char* line)) { diff --git a/src/shim/hooks/research.h b/src/shim/hooks/research.h index 5b6621a..2e745bc 100644 --- a/src/shim/hooks/research.h +++ b/src/shim/hooks/research.h @@ -13,9 +13,15 @@ // the Zuul double roll in one place, and its RNG consumption is observable. So the declared // regions are // -// rng the whole 0x9cc-byte generator object -- mt[624] plus the stream position -// overbudget the caller's accumulator -// node[i] every non-null TechNode in the tree, 0x34 bytes each +// rng the whole 0x9cc-byte generator object -- mt[624] plus the stream position +// overbudget the caller's accumulator +// node[i] every non-null TechNode in the tree, 0x34 bytes each +// events the owner's inline EventStorage header (ServerPlayer+0x29c): EvNxID and the +// turn-bucket count. `ours` posts the pass's events into its own model storage +// and writes the counts into this region's scratch copy -- never into the game. +// observed_techs the owner's vector header (ServerPlayer+0x274): declared so the +// completion append is a named check rather than an undeclared write, and so +// its byte delta measures the element stride, which is not yet pinned. // // and the compare is run with our own MT19937 seeded by load_state() from the *pre-call* // snapshot, so both implementations read the same stream. If the post-call generator state @@ -54,22 +60,38 @@ struct TechTreeProcessResearchHook { static void ours(void* tree, void* rng, void* alloc, int* overbudget); static trace::HookPolicy policy() { return trace::HookPolicy{}; } static void coverage(trace::Coverage& c) { - // THE B3 DEFECT. Kept as a note even though `events` is now a declared region, because - // ours still does not post the event -- the region makes the gap loud, it does not close - // it. See docs/harness-audit.md. - c.unmodelled("posts EVENT_RESEARCH_OVERBUDGET on the owner's EventStorage in the same " - "branch that sets node.flag = 2", - trace::Risk::High, - "the message text is composed from the tech name, so ours cannot synthesise " - "it; it would have to be posted through the game's own event API. This is " - "the defect that made a clean compare false: replace mode's autosave " - "differed from the oracle by exactly this one event", - "region:events (EvNxID now diverges instead of passing silently)"); - c.unmodelled("posts EVENT_TECHS_UNLOCKED for nodes that became available this turn", + // THE B3 DEFECT, now modelled count-only (lane E option (a); see docs/P-events-wiring.md). + c.unmodelled("posts EVENT_RESEARCH_OVERBUDGET on the owner's EventStorage: ours " + "reproduces the decision and the id sequence, so region:events compares " + "next_id, but the composed EvDsc/EvMsg text is not reproduced and no region " + "can see it", trace::Risk::Medium, - "the trailing unlock loop makes no draw and writes no node, but it does " - "build a names list and post an event", + "text comes from the game's string table, which the engine must not carry; " + "ours posts into its own EventStorage and writes only the counts into the " + "scratch copy, so no live byte moves and replace mode posts nothing at all", "region:events"); + // The one event the count model cannot decide. Its trigger IS pinned -- SetResearched's + // second sweep sets state 2 and stamps turnAvailable, and the tail loop collects + // state==2 && turnAvailable==currentTurn -- but evaluating it needs the cascade `ours` + // does not run, so posting it would be a guess that happens to score. + c.unmodelled("posts EVENT_TECHS_UNLOCKED once after the per-node loop, for the nodes " + "SetResearched made available this turn", + trace::Risk::Medium, + "the set comes from the child-unlock cascade, which ours does not run; the " + "pass driver takes the unlock list as an input and is given `no list` " + "rather than an empty one, so a missing input cannot look like a modelled " + "negative. Expect region:events to under-count next_id by exactly 1 on " + "every call that completes a tech", + "region:events"); + c.unmodelled("appends to the owner's vector (ServerPlayer+0x274) on every " + "tech completion", + trace::Risk::High, + "serialized ServerPlayer state that no coverage note in B2 or B3 mentioned " + "until lane R's guard caught it. sizeof(ObservedTech) and the append call " + "site are both unpinned, so ours cannot produce the bytes; the region " + "reports the byte span, whose growth on a completion call measures the " + "element stride", + "region:observed_techs"); c.unmodelled("TechTree::SetResearched on completion: the turn/order stamps, the child " "unlock cascade, the recursive research of zero-cost children, and the " "owner's OnTechResearched callback", diff --git a/tests/game_events/test_events.cpp b/tests/game_events/test_events.cpp index b39bb19..8435931 100644 --- a/tests/game_events/test_events.cpp +++ b/tests/game_events/test_events.cpp @@ -326,6 +326,161 @@ static void test_real_save_shape() { CHECK_EQ(log.turns()[0].events[0].action, 0); } +// --------------------------------------------------------------------------------------- +// The pass driver: what ONE TechTree::ProcessResearch call posts. +// +// The three cases are the three shapes lane R's guarded recapture actually captured +// (findings/subsystems/golden-trace-recapture.md §2): an over-budget call, a completion call, +// and the zero-spend calls in between. +// --------------------------------------------------------------------------------------- + +static void keys_for_pass() { + table().clear(); + table()["EVENTSUM_RESEARCH_OVERBUDGET"] = "sum-ob"; + table()["EVENTMSG_RESEARCH_OVERBUDGET"] = "ob %s"; + table()["EVENTSUM_RESEARCH_COMPLETE"] = "sum-c"; + table()["EVENTMSG_RESEARCH_COMPLETE"] = "c %s"; + table()["EVENTSUM_RESEARCH_UNDERBUDGET"] = "sum-u"; + table()["EVENTMSG_RESEARCH_UNDERBUDGET"] = "u %s"; + table()["EVENTSUM_UNLOCKEDTECHS"] = "sum-unl"; + table()["EVENTMSG_UNLOCKEDTECHS"] = "unl:"; +} + +// recap-b3-compare call 0: alloc {node 144, 2889 points}, the roll failed on the turn the node +// crossed its cost. The original took EvNxID 3 -> 4. This is harness-audit row 1. +static void test_pass_overbudget_call() { + keys_for_pass(); + EventStorage log; + log.set_nextId(3); + log.turns().push_back(TurnEvents{2, {}}); + log.turns().push_back(TurnEvents{3, {}}); // the bucket EVENT_SHIPS_BUILT already made + + std::vector out(1); + out[0].techName = "144"; + out[0].overbudgetEvent = true; + + const ResearchPassResult r = PostResearchPassEvents(log, text_source(), out, nullptr, 3); + CHECK_EQ(r.overbudgetPosts, 1); + CHECK_EQ(r.completionPosts, 0); + CHECK_EQ(r.techsUnlockedPosts, 0); + CHECK_EQ(r.ids.size(), 1u); + CHECK_EQ(r.ids[0], 3); // the id the record gets + CHECK_EQ(r.nextId, 4); // <- the field the compare reports + CHECK_EQ(log.turns().size(), 2u); // no new bucket: turn 3 already had one + CHECK(log.turns()[1].events[0].image == "EVENT_RESEARCH_OVERBUDGET"); + CHECK_EQ(log.turns()[1].events[0].action, 1); + CHECK(log.turns()[1].events[0].pos.x == FLT_MAX); +} + +// The zero-spend calls (1, 2, 4, 5, 7, 8, 10, 11, 13, 14): nothing happened, nothing posts, +// and EvNxID must not move -- including for a player whose list is empty and whose EvNxID is 0. +static void test_pass_posts_nothing_when_nothing_happened() { + keys_for_pass(); + EventStorage log; + std::vector out(2); + out[0].techName = "90"; + out[1].techName = "9"; + const ResearchPassResult r = PostResearchPassEvents(log, text_source(), out, nullptr, 3); + CHECK_EQ(r.ids.size(), 0u); + CHECK_EQ(r.nextId, 0); // EvNxID is promoted to 1 only by an actual post + CHECK_EQ(log.turns().size(), 0u); + CHECK_EQ(log.total_events(), 0u); +} + +// recap-b3-compare call 3: the completion. The original posted TWO events (EvNxID 5 -> 7): the +// completion event, which this driver models, and EVENT_TECHS_UNLOCKED, whose set comes from the +// unlock cascade the hook does not run. With no unlock list the driver posts one, and that under- +// count is the *predicted* residual -- not a silent miss. +static void test_pass_completion_call() { + keys_for_pass(); + EventStorage log; + log.set_nextId(5); + log.turns().push_back(TurnEvents{4, {}}); + + std::vector out(1); + out[0].techName = "144"; + out[0].completed = true; + out[0].completedEarly = false; + + ResearchPassResult r = PostResearchPassEvents(log, text_source(), out, nullptr, 4); + CHECK_EQ(r.completionPosts, 1); + CHECK_EQ(r.techsUnlockedPosts, 0); + CHECK_EQ(r.nextId, 6); // the original reached 7; the missing one is EVENT_TECHS_UNLOCKED + CHECK(log.turns()[0].events[0].image == "EVENT_RESEARCH_COMPLETE"); + + // Given the unlock list, the same pass reaches 7 -- the model is complete, the input is not. + EventStorage log2; + log2.set_nextId(5); + log2.turns().push_back(TurnEvents{4, {}}); + const std::vector unlocked{"91", "92"}; + r = PostResearchPassEvents(log2, text_source(), out, &unlocked, 4); + CHECK_EQ(r.techsUnlockedPosts, 1); + CHECK_EQ(r.nextId, 7); + CHECK(log2.turns()[0].events[1].image == "EVENT_TECHS_UNLOCKED"); + + // An empty (non-null) list is "computed, and empty": still no post, but a different claim. + EventStorage log3; + log3.set_nextId(5); + const std::vector none; + r = PostResearchPassEvents(log3, text_source(), out, &none, 4); + CHECK_EQ(r.techsUnlockedPosts, 0); + CHECK_EQ(r.nextId, 6); +} + +// A cheap completion takes the UNDERBUDGET image -- and posts exactly one event either way, so +// the id count does not depend on the split. +static void test_pass_early_completion_picks_underbudget() { + keys_for_pass(); + EventStorage log; + std::vector out(1); + out[0].techName = "7"; + out[0].completed = true; + out[0].completedEarly = true; + const ResearchPassResult r = PostResearchPassEvents(log, text_source(), out, nullptr, 1); + CHECK_EQ(r.completionPosts, 1); + CHECK_EQ(r.nextId, 2); + CHECK(log.turns()[0].events[0].image == "EVENT_RESEARCH_UNDERBUDGET"); +} + +// Two different techs going over budget in the same turn differ only in EvMsg -- which IS +// compared -- so both survive; the same tech twice collapses to one id. This is the property +// the shim's keyless lookup has to preserve, so it is pinned through that lookup. +static void test_pass_dedup_across_nodes() { + EventStorage log; + std::vector out(2); + out[0].techName = "144"; + out[0].overbudgetEvent = true; + out[1].techName = "9"; + out[1].overbudgetEvent = true; + ResearchPassResult r = PostResearchPassEvents(log, KeylessEventText(), out, nullptr, 3); + CHECK_EQ(r.ids.size(), 2u); + CHECK_EQ(r.ids[0], 1); + CHECK_EQ(r.ids[1], 2); + CHECK_EQ(r.nextId, 3); + CHECK_EQ(log.total_events(), 2u); + + EventStorage same; + out[1].techName = "144"; // the same node twice: the original's messages would be identical + r = PostResearchPassEvents(same, KeylessEventText(), out, nullptr, 3); + CHECK_EQ(r.ids.size(), 2u); + CHECK_EQ(r.ids[0], 1); + CHECK_EQ(r.ids[1], 1); // the duplicate returns the existing id + CHECK_EQ(r.nextId, 2); // and burns neither an id nor a slot + CHECK_EQ(same.total_events(), 1u); +} + +// The keyless lookup must keep an over-budget and a completion for the SAME node apart: they +// share a message but differ in EvImg, which the comparator also tests. +static void test_keyless_lookup_keeps_images_apart() { + EventStorage log; + PostResearchOverbudget(log, KeylessEventText(), "144", 3); + PostResearchCompleted(log, KeylessEventText(), "144", 1.f, 3); + CHECK_EQ(log.total_events(), 2u); + CHECK_EQ(log.nextId(), 3); + CHECK(log.turns()[0].events[0].message == log.turns()[0].events[1].message); + CHECK(log.turns()[0].events[0].image != log.turns()[0].events[1].image); +} + int main() { test_ids_and_next_id(); test_default_record(); @@ -340,5 +495,11 @@ int main() { test_research_event_posts(); test_completion_message_is_capped(); test_real_save_shape(); + test_pass_overbudget_call(); + test_pass_posts_nothing_when_nothing_happened(); + test_pass_completion_call(); + test_pass_early_completion_picks_underbudget(); + test_pass_dedup_across_nodes(); + test_keyless_lookup_keeps_images_apart(); return simtest::finish("game_events"); } diff --git a/tests/shim_events/CMakeLists.txt b/tests/shim_events/CMakeLists.txt new file mode 100644 index 0000000..0b79233 --- /dev/null +++ b/tests/shim_events/CMakeLists.txt @@ -0,0 +1,6 @@ +# P: the event adapter (a live Game::EventStorage <-> the sots::events model). +add_executable(shim_events_unit_tests unit_tests.cpp) +target_link_libraries(shim_events_unit_tests PRIVATE shim_events) +target_include_directories(shim_events_unit_tests PRIVATE ${CMAKE_SOURCE_DIR}/tests/game_sim) +target_compile_options(shim_events_unit_tests PRIVATE -Wall -Wextra -Werror) +add_test(NAME shim_events_unit COMMAND shim_events_unit_tests) diff --git a/tests/shim_events/unit_tests.cpp b/tests/shim_events/unit_tests.cpp new file mode 100644 index 0000000..d2c73f9 --- /dev/null +++ b/tests/shim_events/unit_tests.cpp @@ -0,0 +1,366 @@ +// P adapter tests: read a live-shaped Game::EventStorage out of raw bytes, seed the model from +// it, and write the counts back the way compare mode does. +// +// The fixture is a byte buffer laid out with the *real* offsets from the generated header +// (EventStorage 0x1c / TurnEvents 0x18 / PlayerEvent 0x74, MSVC-2010 std::string 0x1c), so these +// cases pin the nested turn-bucketed shape and the stride, not just the arithmetic. The shape is +// the one verify/results/saves/turn3-state.sav holds for player 1 and the one +// verify/state-checksum/state_checksum.py walks: EvNxID, then n x {EvTurn, m x PlayerEvent}. +#include "shim/hooks/event_inputs.h" + +#include +#include +#include +#include + +#include "check.h" +#include "game/events/research_events.h" +#include "generated/sots_addresses.h" + +using namespace shim::hooks; +namespace A = sots::addr; + +namespace { + +// ---- a fake address space ----------------------------------------------------------------- +// +// One arena; `probe` answers "is this range inside it?", which is the host stand-in for the +// shim's VirtualQuery. Everything the adapter reads must go through it. +std::vector& arena() { + static std::vector a; + return a; +} + +bool probe(const void* p, std::size_t n) { + if (!p) return false; + if (n == 0) return true; + const char* c = static_cast(p); + const char* lo = arena().data(); + const char* hi = lo + arena().size(); + return c >= lo && c + n <= hi && c + n >= c; +} + +void put_word(std::size_t off, std::int32_t v) { std::memcpy(&arena()[off], &v, sizeof v); } + +// A pointer *inside a game object* is four bytes, and in this fake address space a game address +// is simply an offset into the arena, biased so that 0 stays "null". +constexpr std::uint32_t kArenaBias = 0x40000000; +void put_addr(std::size_t off, std::size_t targetOff) { + const std::uint32_t a = kArenaBias + static_cast(targetOff); + std::memcpy(&arena()[off], &a, sizeof a); +} +void put_addr_raw(std::size_t off, std::uint32_t a) { std::memcpy(&arena()[off], &a, sizeof a); } +void put_null(std::size_t off) { put_addr_raw(off, 0); } + +const void* to_host(std::uint32_t a) { + if (a < kArenaBias) return nullptr; // not one of ours: the probe will reject it + const std::size_t off = a - kArenaBias; + if (off > arena().size()) return nullptr; + return arena().data() + off; +} + +// An MSVC 2010 std::string at `off`, short enough to live in the inline buffer (_Myres 15). +void put_short_string(std::size_t off, const std::string& s) { + CHECK(s.size() < kStdStringInlineCap); + std::memcpy(&arena()[off], s.data(), s.size()); + arena()[off + s.size()] = '\0'; + put_word(off + kStdStringOffSize, static_cast(s.size())); + put_word(off + kStdStringOffRes, static_cast(kStdStringInlineCap - 1)); +} + +// The same string, but heap-allocated: _Myres >= 16 and the first word is the pointer. Every +// EvImg in a real save takes this form -- "EVENT_RESEARCH_OVERBUDGET" is 25 characters -- so it +// is the form that actually matters, and reading the inline buffer instead yields garbage. +void put_long_string(std::size_t off, std::size_t bufOff, const std::string& s) { + std::memcpy(&arena()[bufOff], s.data(), s.size()); + arena()[bufOff + s.size()] = '\0'; + put_addr(off, bufOff); + put_word(off + kStdStringOffSize, static_cast(s.size())); + put_word(off + kStdStringOffRes, static_cast(s.size() + 8)); +} + +// Pick the form the real string would take. +void put_string(std::size_t off, std::size_t bufOff, const std::string& s) { + if (s.size() < kStdStringInlineCap) + put_short_string(off, s); + else + put_long_string(off, bufOff, s); +} + +// ---- the fixture ---------------------------------------------------------------------------- +// +// Layout inside the arena: +// 0x0000 EventStorage (0x1c) +// 0x0100 TurnEvents[] (0x18 each) +// 0x0400 PlayerEvent[] for bucket 1 (0x74 each) +// 0x1000 string heap +constexpr std::size_t kStorage = 0x0000; +constexpr std::size_t kBuckets = 0x0100; +constexpr std::size_t kEvents = 0x0400; +constexpr std::size_t kHeap = 0x1000; + +// `images` is one EvImg per event in the *second* bucket; the first bucket is left empty. +void build(int nextId, const std::vector& bucketTurns, const std::vector& images, + std::size_t whichBucketHasEvents) { + arena().assign(0x2000, 0); + put_word(kStorage + A::EventStorage_off_EvNxID, nextId); + if (bucketTurns.empty()) { + put_null(kStorage + A::EventStorage_off_Events); + put_null(kStorage + A::EventStorage_off_Events + kGamePtrSize); + put_null(kStorage + A::EventStorage_off_Events + 2 * kGamePtrSize); + return; + } + const std::size_t bytes = bucketTurns.size() * A::TurnEvents_sizeof; + put_addr(kStorage + A::EventStorage_off_Events, kBuckets); + put_addr(kStorage + A::EventStorage_off_Events + kGamePtrSize, kBuckets + bytes); + put_addr(kStorage + A::EventStorage_off_Events + 2 * kGamePtrSize, kBuckets + bytes); + + for (std::size_t i = 0; i < bucketTurns.size(); ++i) { + const std::size_t b = kBuckets + i * A::TurnEvents_sizeof; + put_word(b + A::TurnEvents_off_EvTurn, bucketTurns[i]); + if (i != whichBucketHasEvents || images.empty()) { + put_null(b + A::TurnEvents_off_Events); + put_null(b + A::TurnEvents_off_Events + kGamePtrSize); + put_null(b + A::TurnEvents_off_Events + 2 * kGamePtrSize); + continue; + } + const std::size_t evBytes = images.size() * A::PlayerEvent_sizeof; + put_addr(b + A::TurnEvents_off_Events, kEvents); + put_addr(b + A::TurnEvents_off_Events + kGamePtrSize, kEvents + evBytes); + put_addr(b + A::TurnEvents_off_Events + 2 * kGamePtrSize, kEvents + evBytes); + for (std::size_t k = 0; k < images.size(); ++k) { + const std::size_t img = kEvents + k * A::PlayerEvent_sizeof + A::PlayerEvent_off_EvImg; + put_string(img, kHeap + k * 0x40, images[k]); + } + } +} + +const void* storage() { return arena().data() + kStorage; } + +// Read the turns vector's {first, last} back out of a scratch header, as 32-bit game addresses. +std::size_t scratch_turns_bytes(const std::vector& scratch) { + std::uint32_t first = 0, last = 0; + std::memcpy(&first, scratch.data() + A::EventStorage_off_Events, sizeof first); + std::memcpy(&last, scratch.data() + A::EventStorage_off_Events + kGamePtrSize, sizeof last); + CHECK(last >= first); + return last - first; +} +std::uint32_t scratch_turns_first(const std::vector& scratch) { + std::uint32_t first = 0; + std::memcpy(&first, scratch.data() + A::EventStorage_off_Events, sizeof first); + return first; +} +// The write-back runs on a scratch buffer, not on the arena, so it gets its own probe. +bool scratch_probe(const void* p, std::size_t n) { return p != nullptr && n <= 0x100; } + +// ---- cases ------------------------------------------------------------------------------------ + +// turn3-state.sav, player 1: EvNxID 4, buckets for turns 2 and 3, the turn-3 bucket holding +// EVENT_SHIPS_BUILT and EVENT_RESEARCH_OVERBUDGET. +void test_scan_reads_the_real_save_shape() { + build(4, {2, 3}, {"EVENT_SHIPS_BUILT", "EVENT_RESEARCH_OVERBUDGET"}, 1); + const EventStorageScan s = ScanEventStorage(storage(), 3, &probe, &to_host); + CHECK(s.ok); + CHECK(!s.scanTruncated); + CHECK_EQ(s.nextId, 4); + CHECK_EQ(s.bucketTurns.size(), 2u); + CHECK_EQ(s.bucketTurns[0], 2); + CHECK_EQ(s.bucketTurns[1], 3); + CHECK_EQ(s.turnsBytes, 2u * A::TurnEvents_sizeof); + CHECK(s.turnBucketExists); + CHECK_EQ(s.eventsInTurnBucket, 2u); + // Only the research event counts towards the dedup risk; EVENT_SHIPS_BUILT can never be a + // duplicate of anything the research path posts. + CHECK_EQ(s.researchEventsInTurnBucket, 1u); +} + +// The pre-call state of recap-b3-compare call 0: EvNxID 3 and no research event in the bucket +// yet -- so a count-only model is exact on that call. +void test_scan_reports_no_dedup_risk_before_the_post() { + build(3, {2, 3}, {"EVENT_SHIPS_BUILT"}, 1); + const EventStorageScan s = ScanEventStorage(storage(), 3, &probe, &to_host); + CHECK_EQ(s.nextId, 3); + CHECK_EQ(s.researchEventsInTurnBucket, 0u); + CHECK_EQ(s.eventsInTurnBucket, 1u); +} + +// A player who has never seen an event: EvNxID 0 and {null,null,null}. Two of the four players +// in the reference save look exactly like this. +void test_scan_of_an_empty_storage() { + build(0, {}, {}, 0); + const EventStorageScan s = ScanEventStorage(storage(), 3, &probe, &to_host); + CHECK(s.ok); + CHECK_EQ(s.nextId, 0); + CHECK_EQ(s.bucketTurns.size(), 0u); + CHECK_EQ(s.turnsBytes, 0u); + CHECK(!s.turnBucketExists); +} + +// The heap form of std::string must read the same as the inline form. +void test_scan_reads_heap_strings() { + build(4, {3}, {"EVENT_RESEARCH_UNDERBUDGET"}, 0); + const EventStorageScan s = ScanEventStorage(storage(), 3, &probe, &to_host); + CHECK(s.ok); + CHECK_EQ(s.researchEventsInTurnBucket, 1u); +} + +// A header that is not a vector must not be believed, and must not walk off the arena. +void test_scan_rejects_a_bad_header() { + build(4, {3}, {}, 0); + // last < first + put_addr(kStorage + A::EventStorage_off_Events, kBuckets + A::TurnEvents_sizeof); + put_addr(kStorage + A::EventStorage_off_Events + kGamePtrSize, kBuckets); + CHECK(!ScanEventStorage(storage(), 3, &probe, &to_host).ok); + + // a span that is not a whole number of TurnEvents + build(4, {3}, {}, 0); + put_addr(kStorage + A::EventStorage_off_Events + kGamePtrSize, kBuckets + 7); + CHECK(!ScanEventStorage(storage(), 3, &probe, &to_host).ok); + + // a span far larger than the arena + build(4, {3}, {}, 0); + put_addr(kStorage + A::EventStorage_off_Events + kGamePtrSize, + kBuckets + (kMaxTurnBuckets + 1) * A::TurnEvents_sizeof); + const EventStorageScan s = ScanEventStorage(storage(), 3, &probe, &to_host); + CHECK(!s.ok); +} + +// ---- seed + post + write back: the whole compare-mode path ----------------------------------- + +// recap-b3-compare call 0, end to end. THE case: EvNxID 3 -> 4 with one over-budget post, and +// `turns` unchanged because the turn-3 bucket already existed. +void test_overbudget_call_reaches_next_id_4() { + build(3, {2, 3}, {"EVENT_SHIPS_BUILT"}, 1); + const EventStorageScan s = ScanEventStorage(storage(), 3, &probe, &to_host); + CHECK_EQ(s.researchEventsInTurnBucket, 0u); // count-only is exact here + + sots::events::EventStorage log = SeedFromScan(s); + CHECK_EQ(log.nextId(), 3); + CHECK_EQ(log.turns().size(), 2u); + + std::vector out(1); + out[0].techName = "144"; + out[0].overbudgetEvent = true; + const sots::events::ResearchPassResult r = + sots::events::PostResearchPassEvents(log, sots::events::KeylessEventText(), out, nullptr, 3); + CHECK_EQ(r.nextId, 4); + + // The scratch copy the region diff reads is a byte copy of the header. + std::vector scratch(A::EventStorage_sizeof); + std::memcpy(scratch.data(), storage(), scratch.size()); + WriteBackCounts(scratch.data(), log, &scratch_probe); + + std::int32_t nextId = 0; + std::memcpy(&nextId, scratch.data() + A::EventStorage_off_EvNxID, 4); + CHECK_EQ(nextId, 4); + CHECK_EQ(scratch_turns_bytes(scratch), 2u * A::TurnEvents_sizeof); +} + +// The same pass on a turn with no bucket yet: `turns` must grow by one, which is what the +// region's turns_bytes reports. +void test_write_back_grows_turns_when_a_bucket_is_created() { + build(3, {2}, {}, 0); + const EventStorageScan s = ScanEventStorage(storage(), 9, &probe, &to_host); + CHECK(!s.turnBucketExists); + sots::events::EventStorage log = SeedFromScan(s); + std::vector out(1); + out[0].techName = "144"; + out[0].overbudgetEvent = true; + sots::events::PostResearchPassEvents(log, sots::events::KeylessEventText(), out, nullptr, 9); + CHECK_EQ(log.turns().size(), 2u); + + std::vector scratch(A::EventStorage_sizeof); + std::memcpy(scratch.data(), storage(), scratch.size()); + WriteBackCounts(scratch.data(), log, &scratch_probe); + CHECK_EQ(scratch_turns_bytes(scratch), 2u * A::TurnEvents_sizeof); +} + +// A previously empty storage: the header has no pointers to extend, so the write-back has to +// synthesise a base or the span would read 0 where the original allocated one bucket. +void test_write_back_on_a_previously_empty_storage() { + build(0, {}, {}, 0); + const EventStorageScan s = ScanEventStorage(storage(), 3, &probe, &to_host); + sots::events::EventStorage log = SeedFromScan(s); + CHECK_EQ(log.nextId(), 0); + std::vector out(1); + out[0].techName = "144"; + out[0].overbudgetEvent = true; + sots::events::PostResearchPassEvents(log, sots::events::KeylessEventText(), out, nullptr, 3); + + std::vector scratch(A::EventStorage_sizeof); + std::memcpy(scratch.data(), storage(), scratch.size()); + WriteBackCounts(scratch.data(), log, &scratch_probe); + std::int32_t nextId = 0; + std::memcpy(&nextId, scratch.data() + A::EventStorage_off_EvNxID, 4); + CHECK_EQ(nextId, 2); // 0 -> promoted to 1 by the post -> post-incremented + CHECK_EQ(scratch_turns_first(scratch), kSyntheticTurnsBase); + CHECK_EQ(scratch_turns_bytes(scratch), 1u * A::TurnEvents_sizeof); +} + +// A zero-spend call must leave the header byte-identical: no id, no bucket. +void test_write_back_is_a_no_op_when_nothing_posted() { + build(3, {2, 3}, {"EVENT_SHIPS_BUILT"}, 1); + const EventStorageScan s = ScanEventStorage(storage(), 3, &probe, &to_host); + sots::events::EventStorage log = SeedFromScan(s); + std::vector out(2); + out[0].techName = "90"; + out[1].techName = "9"; + sots::events::PostResearchPassEvents(log, sots::events::KeylessEventText(), out, nullptr, 3); + + std::vector before(A::EventStorage_sizeof), scratch(A::EventStorage_sizeof); + std::memcpy(before.data(), storage(), before.size()); + std::memcpy(scratch.data(), storage(), scratch.size()); + WriteBackCounts(scratch.data(), log, &scratch_probe); + CHECK(std::memcmp(before.data(), scratch.data(), before.size()) == 0); +} + +// Both std::string forms read the same, and a misread is reported rather than guessed at. +void test_read_std_string_forms_and_bounds() { + arena().assign(0x200, 0); + std::string out = "untouched"; + put_short_string(0, "EVENT_X"); + CHECK(ReadStdStringMapped(arena().data(), &probe, &to_host, out)); + CHECK(out == "EVENT_X"); + + // The heap form -- what every real EvImg uses. + arena().assign(0x200, 0); + put_long_string(0, 0x80, "EVENT_RESEARCH_OVERBUDGET"); + out = "untouched"; + CHECK(ReadStdStringMapped(arena().data(), &probe, &to_host, out)); + CHECK(out == "EVENT_RESEARCH_OVERBUDGET"); + + // A length past the sanity cap. + put_word(kStdStringOffSize, 0x2000); + out = "untouched"; + CHECK(!ReadStdStringMapped(arena().data(), &probe, &to_host, out)); + CHECK(out == "untouched"); + + // _Myres < _Mysize is impossible in a well-formed string. + arena().assign(0x200, 0); + put_long_string(0, 0x80, "EVENT_RESEARCH_OVERBUDGET"); + put_word(kStdStringOffRes, 4); + CHECK(!ReadStdStringMapped(arena().data(), &probe, &to_host, out)); + + // A heap string whose buffer address is not in the arena at all. + arena().assign(0x200, 0); + put_addr_raw(0, 0xdead0000); + put_word(kStdStringOffSize, 8); + put_word(kStdStringOffRes, 31); + CHECK(!ReadStdStringMapped(arena().data(), &probe, &to_host, out)); +} + +} // namespace + +int main() { + test_scan_reads_the_real_save_shape(); + test_scan_reports_no_dedup_risk_before_the_post(); + test_scan_of_an_empty_storage(); + test_scan_reads_heap_strings(); + test_scan_rejects_a_bad_header(); + test_overbudget_call_reaches_next_id_4(); + test_write_back_grows_turns_when_a_bucket_is_created(); + test_write_back_on_a_previously_empty_storage(); + test_write_back_is_a_no_op_when_nothing_posted(); + test_read_std_string_forms_and_bounds(); + return simtest::finish("shim_events"); +}