diff --git a/CMakeLists.txt b/CMakeLists.txt index a900b38..e2f13da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ add_subdirectory(src/game/sim) # strategic formulas, pure (lib sots add_subdirectory(src/mars/vfs) # .gob ZIP reader + native override (lib mars_vfs, miniz) add_subdirectory(src/mars/stream) # Streamable save format + gzip (lib mars_stream) add_subdirectory(src/mars/rng) # MT19937 (lib mars_rng) +add_subdirectory(src/game/config) # flat KEY/value constants loader (lib sots_game_config) add_subdirectory(src/game/data) # typed catalogs on mars/parse+text (lib game_data) add_subdirectory(src/game/design) # ship-design rules + derived stats (lib game_design) @@ -48,8 +49,13 @@ if(WIN32) third_party/minhook/src/hde/hde32.c) target_include_directories(minhook PUBLIC third_party/minhook/include) + # ---- hooks: one descriptor per hooked game function (src/shim/hooks/*) ---- + add_library(shim_hooks STATIC src/shim/hooks/global_consts.cpp) + target_link_libraries(shim_hooks PUBLIC shim_trace sots_addresses sots_game_config) + target_compile_options(shim_hooks PRIVATE -Wall -Wextra -Werror) + add_library(binkw32 SHARED src/shim/main.cpp src/shim/binkw32.def) - target_link_libraries(binkw32 PRIVATE minhook sots_addresses shim_trace) + target_link_libraries(binkw32 PRIVATE minhook sots_addresses shim_trace shim_hooks) target_compile_definitions(binkw32 PRIVATE SHIM_BUILD_ID="${SHIM_BUILD_ID}" SOTS_ADDR_PROVENANCE="${SOTS_ADDR_PROVENANCE}") @@ -61,7 +67,7 @@ else() add_executable(addr_smoke tests/addr_smoke.cpp) target_link_libraries(addr_smoke PRIVATE sots_addresses) add_test(NAME addr_smoke COMMAND addr_smoke) - foreach(_t mars_parse mars_text game_sim mars_vfs mars_stream shim_trace game_data game_design) + foreach(_t mars_parse game_config game_data game_design game_sim mars_stream mars_text mars_vfs shim_trace) if(EXISTS ${CMAKE_SOURCE_DIR}/tests/${_t}/CMakeLists.txt) add_subdirectory(tests/${_t}) endif() diff --git a/docs/M1.md b/docs/M1.md new file mode 100644 index 0000000..a1484ad --- /dev/null +++ b/docs/M1.md @@ -0,0 +1,106 @@ +# M1 — flat KEY/value config loader (`Mars::GlobalConsts::LoadFile`) old-vs-new on the live game + +**Result (2026-09-07/08):** the first real old-vs-new verification. Trace mode: 19 `LoadFile` calls +(1088 constant slots), `tracecmp.py` exit 0. Compare mode: **19/19 compared, 0 divergences**. +Replace mode: our loader fed the game its constants alone; loading `ref-turn2.sav` and pressing +End Turn reproduced the determinism oracle byte for byte (`(Autosave).sav` = `978041ac…`, +`(Autosave EndTurn).sav` = `bb4fd9ac…`). The offline replay test (golden trace through the +host build) also passes. Game left running on VM140 in `hooks=trace` at the main menu. + +## What was hooked + +`Mars::GlobalConsts::LoadFile(const char* file, GlobalConstMap* consts)` — cdecl, verified, so +the `Hook` template applies directly (`src/shim/hooks/global_consts.{h,cpp}`, +installed from `src/shim/main.cpp` via `InstallTemplateHook<>`). It is called once per data file +from `LoadAll()` inside `Application::Initialize`, before the intro movie; the files it touches +(19, in map order of the file names): + +``` +Data/Combat/{actor,assaultshuttle,camera,drone,gravboat,ortgay,planet,sensors,ship,spyship,terrain,vnhomeworld}.txt +Data/encounters.txt Data/globals.txt Data/options.txt Data/Species.txt +Data/Strategy/AI/AIRebellion.txt Data/Strategy/starmap.txt Data/Strategy/StrategyVars.txt +``` + +`ctechvars.txt`, `shipai.txt`, `damfx*.txt`, `playercolors.txt`, `systemnames.txt` are *not* +GlobalConsts files (other readers). + +### Region model + +The map handed to `LoadFile` is an MSVC 2010 `std::map` holding the +constants registered for that file (`key -> {storage, key, parser, file}`; layouts are now in +`sots_addresses.h`). `regions()` walks it in order before the original runs and declares **one +region per constant, named by the key**, sized and described by the parser the registering stub +pushed: + +| parser (VA) | kind | width | describer | +|---|---|---|---| +| `GlobalConst_ParseInt` 0x008b7000 | `int` | 4 | `{t:"int", v:i32}` | +| `GlobalConst_ParseFloat` 0x008b7020 | `float` | 4 | `{t:"float", v:f32}` | +| `GlobalConst_ParseFloatScaled` 0x008b70e0 | `fscaled` | 4 | `{t:"fscaled", v:f32}` | +| `GlobalConst_ParseColour` 0x008b7110 | `colour` | 16 | `{t:"colour", r,g,b,a:f32}` | +| `GlobalConst_ParseRect` 0x008b7040 **(new)** | `rect` | 16 | `{t:"rect", x,y,w,h:i32}` | +| `GlobalConst_ParseVec3` 0x008b7080 **(new)** | `vec3` | 12 | `{t:"vec3", x,y,z:f32}` | +| `GlobalConst_ParseString` 0x008b7670 **(new)** | `string` | 24 | `{t:"string", v:str}` (decoded MSVC `std::string`) | + +Args: `file` (str), `consts` (ptr), `nkeys` (u32) and `scale` (f64, the scaled-float multiplier) +— the last so a golden log replays offline with nothing from the exe. Return: void. + +Discoveries the RE notes did not have (all fed back into `sots-re/ghidra/addresses.json`, status +`verified-by-trace`, and regenerated into `include/generated/sots_addresses.h`): + +- **Three more parsers.** 146 keys (`*_NAME`, `*_SOUND`, `*_TEXTURENAME`, `DERELICT_SECTION_nn`, + `Data/options.txt` …) are `std::string` slots (0x18 bytes: 16-byte SSO buffer / heap pointer, + size @+0x10, capacity @+0x14). Five `HIVER_SPAWN_*` keys are `float[3]` (`"0 0 50.0"`). Two + `COMBATSETUP_CARD_*_RECT` keys are `int[4]`. +- **The scaled-float constant at 0x00af5210 is a `double`** (pi/180 = 0x3f91df46a2529d39), not a + float; reading it as float gives -2.85e-18. The product is computed in double and stored as + float: `(float)((double)v * scale)`. +- `Data/Strategy/StrategyVars.txt` and `Data/encounters.txt` have no trailing newline, so the + original never applies their last key (`CIVILIAN_BURDEN_RATIO`, `HERALD_SPEECH_MAX_INVERVAL`): + visible in the trace as the only unchanged regions in those files and reproduced by ours + ("expected but not found", `missing 1` in both). + +## `ours` — `src/game/config/config_loader.{h,cpp}` (lib `sots_game_config`) + +- File bytes come from the game's own `gobio::ReadFile` (honours the mount order / mods) and are + released with `RefCounted_Release`; the host tests read plain files. +- Pairs come from `mars::text::parse_flat_kv` (engine-parity: Mars::Script tokenizer, blocks + skipped, trailing pair dropped). `apply()` implements **first-occurrence-wins** on top of the + raw pair list (a slot is consumed on first sight; repeats and unknown keys are logged + "`[file] KEY not recognized or is multiply defined.`", absent slots + "`[file] KEY expected but not found.`", to `shim.log` with a `cfg:` prefix). +- Typed writes mirror `sscanf`: `%d` (CRT wrap-around accumulator, no write when no digit), + `%f` (one rounding from the text via `strtold` → float), colour (`%d %d %d %d`, defaults 255, + `/255.0` in double, clamp, always writes all four), vec3 / rect (components until the first + scan failure), scaled float (scan then multiply, whatever the scan did). +- `String` slots are written through an `ExternWriter`: the shim hands the text to the game's + own `ParseString` (a MinGW-built `std::string` cannot be handed to MSVCR100). In compare mode + that runs on the scratch copy of the object, which starts as the image default (empty short + string), so nothing belonging to the live object is ever freed; long strings leak one heap + block per compare call (startup only). + +## Runs (`/bulk-storage/re-lab/shim/traces/`, build `4f0a9db-dirty-20260908T0205Z`) + +| file | mode | calls | result | +|---|---|---|---| +| `m1-trace-golden.jsonl` | trace | 19 LoadFile + 1 selftest | `tracecmp.py` exit 0, 0 invalid | +| `m1-compare.jsonl` | compare | 19 compared | **0 diverged**, 0 errors, exit 0 | +| replace (no log) | replace | 19 | End Turn from `ref-turn2.sav`: `(Autosave).sav` `978041ac…` / `(Autosave EndTurn).sav` `bb4fd9ac…` = oracle | +| `m1-shim.log` | | | all four runs' banners, `cfg:` stats per file | +| `m1-*.png` | | | main menu (trace, compare, final), turn 2 / turn 3 in replace | + +Offline: `SOTS_M1_TRACE= SOTS_DATA_DIR= build-host/tests/game_config/game_config_replay` +→ `tracecmp.py --replay` exit 0 (19 calls, 0 files missing). Host suite: `ctest` 22/22. + +## Gotchas + +1. `shim.cfg` per-hook line: `hook.Mars::GlobalConsts::LoadFile=compare|replace` (default `hooks=trace`). +2. `Z:` is per-logon; from SSH there is no share. Push the dist to `C:\SOTS\shimdist\` with scp + and run `deploy.ps1 -Dist C:\SOTS\shimdist`. Inline PowerShell over ssh mangles `$` and + quotes — ship `.ps1` files (`C:\SOTS\ui\{status,loginfo,lastrun,relaunch,hashes}.ps1`). +3. Local cmake is 3.22 (< 3.25 required): use `uvx --from cmake cmake` with `Unix Makefiles`. +4. The per-call slot table is kept in statics between `regions()` → `rebind()` → `ours()`; fine + because `LoadFile` is single-threaded at startup, but do not reuse the pattern for re-entrant + hooks. +5. `Data/Combat/ctechvars.txt` style files (a single `name { … }` block) would set nothing + through this loader — they are simply not GlobalConsts files, as the trace confirms. diff --git a/src/game/config/CMakeLists.txt b/src/game/config/CMakeLists.txt new file mode 100644 index 0000000..e445160 --- /dev/null +++ b/src/game/config/CMakeLists.txt @@ -0,0 +1,8 @@ +# sots_game_config -- the flat KEY/value constant loader (GlobalConsts): typed slots, +# first-occurrence-wins, the pull-tokenizer pairing rules. Pure; host + cross. +# Include from the root with add_subdirectory(src/game/config) after src/mars/text. +add_library(sots_game_config STATIC config_loader.cpp) +target_include_directories(sots_game_config PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../..) +target_link_libraries(sots_game_config PUBLIC mars_text) +target_compile_features(sots_game_config PUBLIC cxx_std_17) +target_compile_options(sots_game_config PRIVATE -Wall -Wextra -Werror) diff --git a/src/game/config/config_loader.cpp b/src/game/config/config_loader.cpp new file mode 100644 index 0000000..d120895 --- /dev/null +++ b/src/game/config/config_loader.cpp @@ -0,0 +1,207 @@ +#include "game/config/config_loader.h" + +#include +#include + +#include "mars/text/flat_kv.h" +#include "mars/text/value.h" + +namespace game::config { + +const char* kind_name(Kind k) { + switch (k) { + case Kind::Int: return "int"; + case Kind::Float: return "float"; + case Kind::FloatScaled: return "fscaled"; + case Kind::Colour: return "colour"; + case Kind::Vec3: return "vec3"; + case Kind::Rect: return "rect"; + case Kind::String: return "string"; + case Kind::Unknown: break; + } + return "unknown"; +} + +// ---- scanf-style scalar scanners -------------------------------------------------------------- + +namespace { +// The whitespace set C's isspace() has in the "C" locale (what scanf skips). +bool c_space(char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; } +bool digit(char c) { return c >= '0' && c <= '9'; } +} // namespace + +bool scan_int_at(std::string_view s, std::size_t& i, std::int32_t& out) { + std::size_t p = i; + while (p < s.size() && c_space(s[p])) ++p; + bool neg = false; + if (p < s.size() && (s[p] == '+' || s[p] == '-')) neg = s[p++] == '-'; + if (p >= s.size() || !digit(s[p])) return false; + std::uint32_t acc = 0; // wraps like the CRT accumulator + while (p < s.size() && digit(s[p])) acc = acc * 10u + static_cast(s[p++] - '0'); + if (neg) acc = 0u - acc; + out = static_cast(acc); + i = p; + return true; +} + +bool scan_float_at(std::string_view text, std::size_t& pos, float& out) { + std::size_t i = pos; + const std::size_t n = text.size(); + while (i < n && c_space(text[i])) ++i; + const std::size_t start = i; + if (i < n && (text[i] == '+' || text[i] == '-')) ++i; + std::size_t digits = 0; + while (i < n && digit(text[i])) ++i, ++digits; + if (i < n && text[i] == '.') { + ++i; + while (i < n && digit(text[i])) ++i, ++digits; + } + if (digits == 0) return false; + if (i < n && (text[i] == 'e' || text[i] == 'E')) { + std::size_t j = i + 1; + if (j < n && (text[j] == '+' || text[j] == '-')) ++j; + if (j < n && digit(text[j])) { + while (j < n && digit(text[j])) ++j; + i = j; + } + } + // One rounding from the decimal text: the CRT's float scanner converts the collected + // digits straight to single precision. strtold + cast keeps that (an 80-bit intermediate + // cannot double-round the short literals these files hold). + const std::string prefix(text.substr(start, i - start)); + out = static_cast(std::strtold(prefix.c_str(), nullptr)); + pos = i; + return true; +} + +bool scan_int(std::string_view text, std::int32_t& out) { + std::size_t i = 0; + return scan_int_at(text, i, out); +} + +bool scan_float(std::string_view text, float& out) { + std::size_t i = 0; + return scan_float_at(text, i, out); +} + +void parse_colour(std::string_view text, float rgba[4]) { + std::int32_t c[4] = {255, 255, 255, 255}; + std::size_t i = 0; + for (int k = 0; k < 4; ++k) + if (!scan_int_at(text, i, c[k])) break; + for (int k = 0; k < 4; ++k) { + double f = static_cast(c[k]) / 255.0; + if (f < 0.0) f = 0.0; + if (f > 1.0) f = 1.0; + rgba[k] = static_cast(f); + } +} + +int parse_vec3(std::string_view text, float xyz[3]) { + std::size_t i = 0; + int k = 0; + for (; k < 3; ++k) + if (!scan_float_at(text, i, xyz[k])) break; + return k; +} + +int parse_rect(std::string_view text, std::int32_t rect[4]) { + std::size_t i = 0; + int k = 0; + for (; k < 4; ++k) + if (!scan_int_at(text, i, rect[k])) break; + return k; +} + +float scale_float(float v, double scale) { return static_cast(static_cast(v) * scale); } + +// ---- pairs ----------------------------------------------------------------------------------------- + +std::vector pairs(std::string_view text) { + std::vector out; + const mars::text::Result r = mars::text::parse_flat_kv(text); + out.reserve(r.value.size()); + for (const mars::text::KvEntry& e : r.value.entries()) out.push_back(Pair{e.key, e.value.text}); + return out; +} + +// ---- writes -------------------------------------------------------------------------------------- + +void write_slot(Slot& slot, std::string_view value, double scale, const ExternWriter& ext) { + switch (slot.kind) { + case Kind::Int: { + std::int32_t v = 0; + if (scan_int(value, v)) std::memcpy(slot.storage, &v, sizeof v); + break; + } + case Kind::Float: { + float v = 0; + if (scan_float(value, v)) std::memcpy(slot.storage, &v, sizeof v); + break; + } + case Kind::FloatScaled: { + // The original scans into the word and then multiplies it, whatever the scan did. + float v = 0; + std::memcpy(&v, slot.storage, sizeof v); + scan_float(value, v); + v = scale_float(v, scale); + std::memcpy(slot.storage, &v, sizeof v); + break; + } + case Kind::Colour: { + float rgba[4]; + parse_colour(value, rgba); + std::memcpy(slot.storage, rgba, sizeof rgba); + break; + } + case Kind::Vec3: { + float xyz[3]; + std::memcpy(xyz, slot.storage, sizeof xyz); + parse_vec3(value, xyz); + std::memcpy(slot.storage, xyz, sizeof xyz); + break; + } + case Kind::Rect: { + std::int32_t rect[4]; + std::memcpy(rect, slot.storage, sizeof rect); + parse_rect(value, rect); + std::memcpy(slot.storage, rect, sizeof rect); + break; + } + case Kind::String: + if (ext) ext(slot, value); + break; + case Kind::Unknown: break; + } +} + +Stats apply(std::string_view file, std::string_view text, std::vector& slots, double scale, + const LogFn& log, const ExternWriter& ext) { + Stats st; + const std::string fname(file); + for (const Pair& p : pairs(text)) { + Slot* hit = nullptr; + for (Slot& s : slots) + if (mars::text::equals_fold(s.key, p.key)) { + hit = &s; + break; + } + if (!hit || hit->consumed) { + if (hit) ++st.duplicate; + else ++st.unknown; + if (log) log("[" + fname + "] " + p.key + " not recognized or is multiply defined."); + continue; + } + write_slot(*hit, p.value, scale, ext); + hit->consumed = true; + ++st.applied; + } + for (const Slot& s : slots) + if (!s.consumed) { + ++st.missing; + if (log) log("[" + fname + "] " + s.key + " expected but not found."); + } + return st; +} + +} // namespace game::config diff --git a/src/game/config/config_loader.h b/src/game/config/config_loader.h new file mode 100644 index 0000000..2cfb9ae --- /dev/null +++ b/src/game/config/config_loader.h @@ -0,0 +1,108 @@ +// game::config -- the flat "KEY value" constant tables (Data/globals.txt, +// Data/Strategy/StrategyVars.txt, Data/Species.txt, Data/encounters.txt, Data/Combat/*.txt). +// +// Model of the original loader (findings: loader-prototypes.md section M1): +// +// * Every constant is a static "slot" registered at image start-up with a key, a storage +// word and one typed parser. The *slot* decides the type; the file never does. +// * The file is read with mars::text::parse_flat_kv, which follows the engine's own loop: +// the Mars::Script pull tokenizer, KEY value pairs, `NAME {` blocks skipped whole, a +// lone `}` ignored, and a final pair whose value touches end-of-file dropped. +// * Keys match case-insensitively (ASCII). A slot is consumed on first sight, so the +// first occurrence wins; a repeat -- or an unknown key -- is logged and ignored. Slots +// the file never names keep whatever the image default was. +// +// Slot kinds and their storage (widths are what the hook declares as side-effect regions): +// Int int32 "%d" +// Float float "%f" +// FloatScaled float "%f", then multiplied by a process constant (pi/180 as a double) +// Colour float[4] "%d %d %d %d", each defaulted to 255, /255, clamped to 0..1 +// Vec3 float[3] "%f %f %f" (components not matched stay unchanged) +// Rect int32[4] "%d %d %d %d" (idem) +// String an engine std::string (24 bytes): the text itself. This module cannot build +// the engine's string object, so String slots are written through the +// caller's ExternWriter (the shim hands them to the game's own parser). +// Unknown a parser this code does not model; never written, declared 4 bytes wide. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace game::config { + +enum class Kind : std::uint8_t { Int, Float, FloatScaled, Colour, Vec3, Rect, String, Unknown }; + +constexpr std::size_t width(Kind k) { + switch (k) { + case Kind::Colour: return 16; + case Kind::Vec3: return 12; + case Kind::Rect: return 16; + case Kind::String: return 24; + default: return 4; + } +} +const char* kind_name(Kind k); // "int" | "float" | "fscaled" | "colour" | "vec3" | "rect" | "string" | "unknown" + +// One registered constant: where the value lives and how its text is typed. +struct Slot { + std::string key; // as registered (matched case-insensitively) + Kind kind = Kind::Int; + void* storage = nullptr; // width(kind) bytes + void* native = nullptr; // opaque handle for the ExternWriter (the shim keeps the game parser here) + bool consumed = false; // set by apply(): first occurrence wins +}; + +struct Stats { + unsigned applied = 0; // pairs written to a slot + unsigned duplicate = 0; // repeats of an already consumed key (ignored) + unsigned unknown = 0; // keys with no slot (ignored) + unsigned missing = 0; // slots the file never named +}; + +using LogFn = std::function; +using ExternWriter = std::function; + +// ---- typed value parsers ------------------------------------------------------------------- +// +// scan_int / scan_float follow C `sscanf` "%d" / "%f": leading whitespace skipped, an optional +// sign, then the longest valid prefix; they return false and leave `out` alone when no digit +// is found. Integers wrap modulo 2^32 like the CRT's accumulator; floats are converted with +// one rounding from the decimal text. The *_at forms continue from `pos` (for "%d %d .."). +bool scan_int(std::string_view text, std::int32_t& out); +bool scan_float(std::string_view text, float& out); +bool scan_int_at(std::string_view text, std::size_t& pos, std::int32_t& out); +bool scan_float_at(std::string_view text, std::size_t& pos, float& out); +// "%d %d %d %d" with every component defaulted to 255; each is divided by 255 (double +// arithmetic, then stored as float) and clamped to [0, 1]. Always writes all four. +void parse_colour(std::string_view text, float rgba[4]); +// "%f %f %f" / "%d %d %d %d": components are written left to right until one fails to scan; +// returns how many were written. +int parse_vec3(std::string_view text, float xyz[3]); +int parse_rect(std::string_view text, std::int32_t rect[4]); +// The scaled-float product as the original computes it (double product, stored as float). +float scale_float(float v, double scale); + +// ---- the pairs ------------------------------------------------------------------------------------ + +struct Pair { + std::string key, value; +}; +// Every top-level KEY value pair in file order (duplicates included), per the engine rules. +std::vector pairs(std::string_view text); + +// Writes the value text into the slot according to its kind. `scale` feeds FloatScaled; +// String slots go through `ext` (nothing happens without one). +void write_slot(Slot& slot, std::string_view value, double scale, const ExternWriter& ext); + +// The whole loader for one file: `text` is the file's bytes, `slots` the constants registered +// for that file. Consumed slots are flagged; diagnostics mirror the original's wording: +// "[file] KEY not recognized or is multiply defined." "[file] KEY expected but not found." +// `file` is only used in the log lines. +Stats apply(std::string_view file, std::string_view text, std::vector& slots, double scale, + const LogFn& log, const ExternWriter& ext = nullptr); + +} // namespace game::config diff --git a/src/shim/hooks/global_consts.cpp b/src/shim/hooks/global_consts.cpp new file mode 100644 index 0000000..7855cb9 --- /dev/null +++ b/src/shim/hooks/global_consts.cpp @@ -0,0 +1,325 @@ +#include "shim/hooks/global_consts.h" + +#include +#include +#include +#include +#include + +#include "game/config/config_loader.h" +#include "generated/sots_addresses.h" + +namespace shim::hooks { + +using game::config::Kind; +using game::config::Slot; +using trace::Tv; + +namespace { + +// ---- in-process layouts (32-bit; facts from sots_addresses.h) -------------------------------- + +struct GlobalConst { // one registered constant + void* storage; // +0 int32 / float / float[4] / float[3] / int[4] / std::string + const char* key; // +4 + void* parser; // +8 one of the GlobalConst_Parse* functions + const char* file; // +c +}; +struct MapNode { // MSVC 2010 std::map node + MapNode* left; // +0 + MapNode* parent; // +4 + MapNode* right; // +8 + const char* key; // +c + GlobalConst* value; // +10 + std::uint8_t color; // +14 + std::uint8_t isnil; // +15 set on the head/nil sentinel only +}; +struct MapHeader { // MSVC 2010 std::map object + void* cmp; // +0 + MapNode* head; // +4 sentinel: parent = root, left = leftmost, right = rightmost + std::uint32_t size; // +8 +}; +struct IBuffer { // gobio::Buffer + void* vft; // +0 + std::uint32_t refcount; // +4 + const char* data; // +8 + std::uint32_t size; // +c +}; +struct MsvcString { // MSVC 2010 std::string (0x18 bytes) + union { + char buf[16]; // the text when capacity <= 15 + const char* ptr; // else a heap block + } u; + std::uint32_t size; // +10 + std::uint32_t capacity; // +14 +}; + +#if defined(_WIN32) && defined(__i386__) +#define HOOK_FASTCALL __attribute__((fastcall)) +#else +#define HOOK_FASTCALL +#endif +using ReadFileFn = unsigned char(SHIM_CDECL*)(const char* path, IBuffer** out); // bool in AL +using ReleaseFn = int(HOOK_FASTCALL*)(IBuffer* obj); // ECX = obj +using ParseFn = void(SHIM_CDECL*)(void* storage, const char* text); // the slot parsers + +struct Env { + std::uintptr_t exe_base = 0; + void (*log_line)(const char*) = nullptr; + void* parse_int = nullptr; + void* parse_float = nullptr; + void* parse_float_scaled = nullptr; + void* parse_colour = nullptr; + void* parse_vec3 = nullptr; + void* parse_rect = nullptr; + void* parse_string = nullptr; + const double* scale = nullptr; + ReadFileFn read_file = nullptr; + ReleaseFn release = nullptr; +}; +Env g_env; + +void logf(const char* fmt, ...) { + if (!g_env.log_line) return; + char line[1024]; + va_list ap; + va_start(ap, fmt); + std::vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + g_env.log_line(line); +} + +double scale_constant() { + double s = 0; + if (g_env.scale) std::memcpy(&s, g_env.scale, sizeof s); + return s; +} + +Kind classify(const void* parser) { + if (parser == g_env.parse_int) return Kind::Int; + if (parser == g_env.parse_float) return Kind::Float; + if (parser == g_env.parse_float_scaled) return Kind::FloatScaled; + if (parser == g_env.parse_colour) return Kind::Colour; + if (parser == g_env.parse_vec3) return Kind::Vec3; + if (parser == g_env.parse_rect) return Kind::Rect; + if (parser == g_env.parse_string) return Kind::String; + return Kind::Unknown; +} + +// In-order walk of the map: one Slot per registered constant (map order = stricmp order). +std::vector slots_from_map(const void* consts) { + std::vector out; + const MapHeader* m = static_cast(consts); + if (!m || !m->head) return out; + const MapNode* head = m->head; + const std::size_t cap = static_cast(m->size) + 1; // loop guard + auto is_nil = [head](const MapNode* n) { return !n || n == head || n->isnil; }; + const MapNode* n = head->left; + while (!is_nil(n) && out.size() < cap) { + if (n->value) { + Slot s; + s.key = n->value->key ? n->value->key : (n->key ? n->key : ""); + s.kind = classify(n->value->parser); + s.storage = n->value->storage; + s.native = n->value->parser; + if (s.kind == Kind::Unknown) + logf("cfg: %s has a parser this build does not model (%p)", s.key.c_str(), n->value->parser); + out.push_back(std::move(s)); + } + if (!is_nil(n->right)) { + n = n->right; + while (!is_nil(n->left)) n = n->left; + } else { + const MapNode* p = n->parent; + while (!is_nil(p) && n == p->right) { + n = p; + p = p->parent; + } + n = p; + } + } + return out; +} + +// ---- describers: the region bytes as the typed value the game reads ------------------------ + +Tv typed(const char* kind) { + Tv s = trace::tv::struct_(); + s.add("t", trace::tv::str(kind)); + return s; +} +Tv describe_int(const void* p, std::size_t, unsigned) { + std::int32_t v; + std::memcpy(&v, p, sizeof v); + Tv s = typed("int"); + s.add("v", trace::tv::i32(v)); + return s; +} +Tv describe_float_kind(const void* p, const char* kind) { + float v; + std::memcpy(&v, p, sizeof v); + Tv s = typed(kind); + s.add("v", trace::tv::f32(v)); + return s; +} +Tv describe_float(const void* p, std::size_t, unsigned) { return describe_float_kind(p, "float"); } +Tv describe_fscaled(const void* p, std::size_t, unsigned) { return describe_float_kind(p, "fscaled"); } +Tv describe_colour(const void* p, std::size_t, unsigned) { + float c[4]; + std::memcpy(c, p, sizeof c); + Tv s = typed("colour"); + s.add("r", trace::tv::f32(c[0])); + s.add("g", trace::tv::f32(c[1])); + s.add("b", trace::tv::f32(c[2])); + s.add("a", trace::tv::f32(c[3])); + return s; +} +Tv describe_vec3(const void* p, std::size_t, unsigned) { + float v[3]; + std::memcpy(v, p, sizeof v); + Tv s = typed("vec3"); + s.add("x", trace::tv::f32(v[0])); + s.add("y", trace::tv::f32(v[1])); + s.add("z", trace::tv::f32(v[2])); + return s; +} +Tv describe_rect(const void* p, std::size_t, unsigned) { + std::int32_t v[4]; + std::memcpy(v, p, sizeof v); + Tv s = typed("rect"); + s.add("x", trace::tv::i32(v[0])); + s.add("y", trace::tv::i32(v[1])); + s.add("w", trace::tv::i32(v[2])); + s.add("h", trace::tv::i32(v[3])); + return s; +} +// The region is a copy of the string object; a long string's text still lives in the heap +// block the copied pointer names (same process), so the value is the text either way. +Tv describe_string(const void* p, std::size_t, unsigned) { + MsvcString s; + std::memcpy(&s, p, sizeof s); + const char* text = s.capacity > 15 ? s.u.ptr : s.u.buf; + std::size_t n = s.size; + if (!text || n > 4096) { // never trust a corrupt object + Tv bad = typed("string"); + bad.add("v", trace::tv::null()); + bad.add("size", trace::tv::u32(s.size)); + bad.add("capacity", trace::tv::u32(s.capacity)); + return bad; + } + if (s.capacity <= 15 && n > 15) n = 15; + Tv out = typed("string"); + out.add("v", trace::tv::str(text, n)); + return out; +} + +trace::Region region_for(const Slot& s) { + trace::Region r; + r.name = s.key.c_str(); + r.ptr = s.storage; + r.size = game::config::width(s.kind); + switch (s.kind) { + case Kind::Int: r.describe = &describe_int; break; + case Kind::Float: r.describe = &describe_float; break; + case Kind::FloatScaled: r.describe = &describe_fscaled; break; + case Kind::Colour: r.describe = &describe_colour; break; + case Kind::Vec3: r.describe = &describe_vec3; break; + case Kind::Rect: r.describe = &describe_rect; break; + case Kind::String: r.describe = &describe_string; break; + case Kind::Unknown: r.describe = nullptr; break; // raw bytes + } + return r; +} + +// String slots: hand the text to the game's own parser (std::string assignment inside the +// engine's runtime). On a scratch copy that copy starts as the image default (an empty +// short string), so the assignment never frees anything that belongs to the live object. +void write_string_via_game(Slot& slot, std::string_view value) { + if (!slot.native || !slot.storage) return; + const std::string text(value); + reinterpret_cast(slot.native)(slot.storage, text.c_str()); +} + +// Per-call state. LoadFile is only ever called from LoadAll on the start-up thread, one +// call at a time, so a single pending table is enough: regions() fills it before the +// original runs, rebind() re-aims it at the scratch copies, ours() consumes it. +std::vector g_pending; // slots in region order (live storage) +std::vector g_bound; // the same slots aimed at scratch memory +bool g_have_bound = false; + +} // namespace + +void init_global_consts(std::uintptr_t exe_base, void (*log_line)(const char* line)) { + g_env.exe_base = exe_base; + g_env.log_line = log_line; + auto at = [exe_base](std::uint32_t rva) { return reinterpret_cast(exe_base + rva); }; + g_env.parse_int = at(sots::addr::GlobalConst_ParseInt); + g_env.parse_float = at(sots::addr::GlobalConst_ParseFloat); + g_env.parse_float_scaled = at(sots::addr::GlobalConst_ParseFloatScaled); + g_env.parse_colour = at(sots::addr::GlobalConst_ParseColour); + g_env.parse_vec3 = at(sots::addr::GlobalConst_ParseVec3); + g_env.parse_rect = at(sots::addr::GlobalConst_ParseRect); + g_env.parse_string = at(sots::addr::GlobalConst_ParseString); + g_env.scale = static_cast(at(sots::addr::g_GlobalConstFloatScale)); + g_env.read_file = reinterpret_cast(at(sots::addr::gobio_ReadFile)); + g_env.release = reinterpret_cast(at(sots::addr::RefCounted_Release)); + logf("cfg: GlobalConsts hook ready (scale constant %.17g)", scale_constant()); +} + +void GlobalConstsLoadFileHook::describe_args(std::vector& out, const char* file, void* consts) { + out.push_back(trace::tv::str(file).named("file")); + out.push_back(trace::tv::ptr(consts).named("consts")); + const MapHeader* m = static_cast(consts); + out.push_back(trace::tv::u32(m ? m->size : 0u).named("nkeys")); + // The scaled-float multiplier, so an offline replay of this log needs nothing from the exe. + out.push_back(trace::tv::f64(scale_constant()).named("scale")); +} + +void GlobalConstsLoadFileHook::regions(std::vector& out, const char*, void* consts) { + g_pending = slots_from_map(consts); + g_have_bound = false; + for (const Slot& s : g_pending) out.push_back(region_for(s)); +} + +GlobalConstsLoadFileHook::Args GlobalConstsLoadFileHook::rebind(trace::Scratch& s, const char* file, + void* consts) { + g_bound.clear(); + for (std::size_t i = 0; i < g_pending.size() && i < s.count(); ++i) { + Slot b = g_pending[i]; + b.storage = s.ptr(i); + b.consumed = false; + g_bound.push_back(std::move(b)); + } + g_have_bound = true; + return Args(file, consts); +} + +void GlobalConstsLoadFileHook::ours(const char* file, void* consts) { + std::vector slots; + if (g_have_bound) { + slots = std::move(g_bound); + g_bound.clear(); + g_have_bound = false; + } else { + slots = slots_from_map(consts); // replace mode: the live storage words + } + if (!g_env.read_file) { + logf("cfg: [%s] hook not initialised", file ? file : ""); + return; + } + IBuffer* buf = nullptr; + if (!g_env.read_file(file, &buf) || !buf) { + logf("cfg: GlobalConsts: Could not open %s.", file ? file : ""); + return; + } + const std::string_view text(buf->data ? buf->data : "", buf->data ? buf->size : 0u); + const std::string fname = file ? file : ""; + const game::config::Stats st = game::config::apply( + fname, text, slots, scale_constant(), [](const std::string& line) { logf("cfg: %s", line.c_str()); }, + &write_string_via_game); + logf("cfg: [%s] %u keys: applied %u, unknown %u, duplicate %u, missing %u", fname.c_str(), + static_cast(slots.size()), st.applied, st.unknown, st.duplicate, st.missing); + if (g_env.release) g_env.release(buf); +} + +} // namespace shim::hooks diff --git a/src/shim/hooks/global_consts.h b/src/shim/hooks/global_consts.h new file mode 100644 index 0000000..5b8b8f9 --- /dev/null +++ b/src/shim/hooks/global_consts.h @@ -0,0 +1,43 @@ +// Hook descriptor for Mars::GlobalConsts::LoadFile(const char* file, GlobalConstMap* consts) +// -- the file-level read of the flat KEY/value constant tables (M1). +// +// Side-effect model: the map handed to LoadFile lists every constant registered for that +// file (key -> {storage word, parser, file}). The hook walks the map before the original +// runs and declares one region per constant, named by its key, as wide as its parser's +// type (int / float / scaled float = 4 bytes, colour = float[4] = 16 bytes), with a struct +// describer so a diff names the field ("v", or "r"/"g"/"b"/"a"). In compare mode `ours` +// (game::config::apply over the same file bytes, read through the game's own gobio) fills +// the scratch copies of those words; the tracer diffs them against the original's. +// +// The map itself is a temporary of LoadAll (the original erases consumed keys from it); it +// is not declared as a region and `ours` never touches it. +// +// LoadFile is cdecl with a verified prototype, so the Hook<> template applies directly. +#pragma once + +#include +#include +#include + +#include "shim/trace/hook.h" + +namespace shim::hooks { + +struct GlobalConstsLoadFileHook { + static constexpr const char* name = "Mars::GlobalConsts::LoadFile"; + static constexpr trace::CallConv conv = trace::CallConv::Cdecl; + using Ret = void; + using Args = std::tuple; + + static void describe_args(std::vector& out, const char* file, void* consts); + static void regions(std::vector& out, const char* file, void* consts); + static Args rebind(trace::Scratch& s, const char* file, void* consts); + static void ours(const char* file, void* consts); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } +}; + +// Process facts the hook needs: the exe's load address (parser identification, the scale +// constant, gobio::ReadFile) and a line logger (shim.log). Call once before installing. +void init_global_consts(std::uintptr_t exe_base, void (*log_line)(const char* line)); + +} // namespace shim::hooks diff --git a/src/shim/main.cpp b/src/shim/main.cpp index fe37293..493a80f 100644 --- a/src/shim/main.cpp +++ b/src/shim/main.cpp @@ -15,6 +15,7 @@ #include "MinHook.h" #include "generated/sots_addresses.h" +#include "shim/hooks/global_consts.h" #include "shim/trace/hook.h" #include "shim/trace/selftest.h" #include "shim/trace/tracer.h" @@ -118,9 +119,35 @@ _InitializeDetour: jmp *_g_origInitialize )"); +// Line logger handed to the template hooks (they live outside this TU's anonymous namespace). +void ShimLogLine(const char* line) { Log("%s", line); } + namespace { -void InstallHooks() { +// ---- template hooks (verified cdecl prototypes; docs/shim-trace.md "Declaring a hook") ---- + +template +void InstallTemplateHook(shim::trace::Tracer& tracer, uintptr_t exeBase, uint32_t rva) { + using H = shim::trace::Hook; + H::configure(tracer); + void* target = reinterpret_cast(exeBase + rva); + if (H::mode == shim::trace::Mode::Off) { + Log("hook: %s rva=0x%08x mode=off (not installed)", D::name, rva); + return; + } + MH_STATUS st = MH_CreateHook(target, reinterpret_cast(H::detour()), + reinterpret_cast(&H::original)); + Log("hook: %s rva=0x%08x -> va=%p MH_CreateHook -> %s (trampoline=%p)", D::name, rva, target, + MH_StatusToString(st), reinterpret_cast(H::original)); + if (st != MH_OK) return; + st = MH_EnableHook(target); + Log("hook: %s MH_EnableHook -> %s mode=%s", D::name, MH_StatusToString(st), + shim::trace::mode_name(H::mode)); +} + +using LoadFileHook = shim::trace::Hook; + +void InstallHooks(shim::trace::Tracer& tracer) { const uintptr_t exeBase = reinterpret_cast(GetModuleHandleA(nullptr)); void* target = reinterpret_cast(exeBase + sots::addr::Mars_Application_Initialize); Log("hook: Mars_Application_Initialize rva=0x%08x -> va=%p", sots::addr::Mars_Application_Initialize, @@ -137,6 +164,10 @@ void InstallHooks() { st = MH_EnableHook(target); Log("hook: MH_EnableHook -> %s", MH_StatusToString(st)); + + // M1: GlobalConsts::LoadFile (cdecl, verified) through the template. + shim::hooks::init_global_consts(exeBase, &ShimLogLine); + InstallTemplateHook(tracer, exeBase, sots::addr::GlobalConsts_LoadFile); } // ---- lifecycle ----------------------------------------------------------------------------- @@ -174,6 +205,7 @@ void Shim_Init(HMODULE self) { shim::trace::Tracer& tracer = shim::trace::Tracer::instance(); tracer.configure(cfg.trace); shim::trace::Hook::register_policy(tracer); + LoadFileHook::register_policy(tracer); char exeSha[65] = {}; if (!shim::trace::sha256_file(exePath, exeSha)) Log("trace: could not hash %s", exePath); if (tracer.open(SHIM_BUILD_ID, exeSha)) { @@ -184,7 +216,7 @@ void Shim_Init(HMODULE self) { Log("trace: cannot open %s; template hooks forced off", cfg.trace.path.c_str()); } - InstallHooks(); + InstallHooks(tracer); // Self-test through the hook template (no MinHook involved): one record per launch proves // the emitter/tracer inside the game process. `hook.Shim::SelfTest::Fill=off` silences it. diff --git a/src/shim/shim.cfg b/src/shim/shim.cfg index 572a2e3..e10c173 100644 --- a/src/shim/shim.cfg +++ b/src/shim/shim.cfg @@ -12,6 +12,7 @@ hooks=trace # hook. = off | trace | compare | replace # per-hook override; is the hook's record name (the "hook" field in the log) #hook.Shim::SelfTest::Fill=off +#hook.Mars::GlobalConsts::LoadFile=compare # M1: the flat KEY/value constants loader (docs/M1.md) # trace.path = default: shim.trace.jsonl next to the DLL (overwritten each run) # trace.inline_max = regions up to this size are logged as hex (default 256) diff --git a/tests/game_config/CMakeLists.txt b/tests/game_config/CMakeLists.txt new file mode 100644 index 0000000..c0f6510 --- /dev/null +++ b/tests/game_config/CMakeLists.txt @@ -0,0 +1,16 @@ +# game/config tests: hand-written samples for the constant loader's typing, first-wins, +# colour, tokenizer and block rules, plus an offline replay of a golden LoadFile trace +# (skips unless SOTS_DATA_DIR and a trace path are given). +add_executable(game_config_unit_tests unit_tests.cpp) +target_link_libraries(game_config_unit_tests PRIVATE sots_game_config) +target_compile_options(game_config_unit_tests PRIVATE -Wall -Wextra -Werror) +add_test(NAME game_config_unit COMMAND game_config_unit_tests) + +add_executable(game_config_replay replay_trace.cpp) +target_link_libraries(game_config_replay PRIVATE sots_game_config shim_trace) +target_include_directories(game_config_replay PRIVATE ${CMAKE_SOURCE_DIR}/tests/game_sim ${CMAKE_SOURCE_DIR}/tests/shim_trace) +set(SOTS_TRACECMP_DIR "$ENV{HOME}/sots-re/verify/harness/compare" CACHE PATH + "Directory holding tracecmp.py (python oracle for the trace tests)") +target_compile_definitions(game_config_replay PRIVATE SOTS_TRACECMP_DIR="${SOTS_TRACECMP_DIR}") +target_compile_options(game_config_replay PRIVATE -Wall -Wextra -Werror) +add_test(NAME game_config_replay COMMAND game_config_replay) # SKIPs without SOTS_DATA_DIR / SOTS_M1_TRACE diff --git a/tests/game_config/replay_trace.cpp b/tests/game_config/replay_trace.cpp new file mode 100644 index 0000000..d8fafe4 --- /dev/null +++ b/tests/game_config/replay_trace.cpp @@ -0,0 +1,278 @@ +// Offline replay of a golden Mars::GlobalConsts::LoadFile trace through game::config. +// +// For every LoadFile record in $SOTS_M1_TRACE the test rebuilds the registered slots from +// the record's `side..before` values (the region model: one struct per key, typed by +// its "t" field), reads the file named in args from $SOTS_DATA_DIR (case-insensitively -- +// the game's VFS is), runs apply() with the scale constant carried in args, and emits the +// results as replay input (TRACE_FORMAT.md section 8). tracecmp.py --replay then diffs them +// against the original's `after` values. SKIPs cleanly when either variable is unset. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "game/config/config_loader.h" +#include "harness.h" +#include "mini_json.h" +#include "shim/trace/emitter.h" + +namespace fs = std::filesystem; +using namespace game::config; +using shim::trace::Tv; + +static const char* kHook = "Mars::GlobalConsts::LoadFile"; + +static std::string getenv_str(const char* k) { + const char* v = std::getenv(k); + return v ? v : ""; +} + +// Resolve `rel` (forward slashes, any case) under `root`, one component at a time. +static bool resolve_ci(const fs::path& root, const std::string& rel, fs::path& out) { + fs::path cur = root; + std::stringstream ss(rel); + std::string part; + while (std::getline(ss, part, '/')) { + if (part.empty()) continue; + bool found = false; + std::error_code ec; + for (const fs::directory_entry& e : fs::directory_iterator(cur, ec)) { + const std::string name = e.path().filename().string(); + if (name.size() == part.size() && std::equal(name.begin(), name.end(), part.begin(), [](char a, char b) { + return std::tolower(static_cast(a)) == std::tolower(static_cast(b)); + })) { + cur = e.path(); + found = true; + break; + } + } + if (!found) return false; + } + out = cur; + return true; +} + +static bool read_file(const fs::path& p, std::string& out) { + std::ifstream f(p, std::ios::binary); + if (!f) return false; + std::ostringstream ss; + ss << f.rdbuf(); + out = ss.str(); + return true; +} + +// ---- the region model, host side (mirrors src/shim/hooks/global_consts.cpp) --------------- + +static Kind kind_of(const std::string& t) { + if (t == "int") return Kind::Int; + if (t == "float") return Kind::Float; + if (t == "fscaled") return Kind::FloatScaled; + if (t == "colour") return Kind::Colour; + if (t == "vec3") return Kind::Vec3; + if (t == "rect") return Kind::Rect; + if (t == "string") return Kind::String; + return Kind::Unknown; +} + +static float fld(const minijson::Value* v, const char* name); +static std::int32_t ild(const minijson::Value* v, const char* name); + +static float fval(const minijson::Value* v) { + if (!v) return 0; + if (v->kind == minijson::Value::String) { + if (v->str == "nan") return std::nanf(""); + if (v->str == "inf") return HUGE_VALF; + if (v->str == "-inf") return -HUGE_VALF; + } + return static_cast(v->number()); +} + +// A struct tv {t:{..,v:"int"}, v:{..}} -> slot kind + storage bytes. +static bool slot_from_tv(const std::string& key, const minijson::Value& tv, Slot& slot, std::vector& store) { + const minijson::Value* t = tv.get("t"); + if (!t || t->string() != "struct") return false; + const minijson::Value* v = tv.get("v"); + if (!v || !v->isObject()) return false; + const minijson::Value* kind = v->get("t"); + if (!kind) return false; + slot.key = key; + slot.kind = kind_of(kind->get("v") ? kind->get("v")->string() : ""); + store.assign(width(slot.kind), 0); + if (slot.kind == Kind::String) { + // host side: the slot's storage is a std::string owned by the test (see main) + return true; + } + if (slot.kind == Kind::Vec3) { + float c[3] = {fld(v, "x"), fld(v, "y"), fld(v, "z")}; + std::memcpy(store.data(), c, sizeof c); + } else if (slot.kind == Kind::Rect) { + std::int32_t c[4] = {ild(v, "x"), ild(v, "y"), ild(v, "w"), ild(v, "h")}; + std::memcpy(store.data(), c, sizeof c); + } else if (slot.kind == Kind::Colour) { + float c[4] = {fval(v->get("r") ? v->get("r")->get("v") : nullptr), fval(v->get("g") ? v->get("g")->get("v") : nullptr), + fval(v->get("b") ? v->get("b")->get("v") : nullptr), fval(v->get("a") ? v->get("a")->get("v") : nullptr)}; + std::memcpy(store.data(), c, sizeof c); + } else if (slot.kind == Kind::Int) { + const std::int32_t i = static_cast(v->get("v") && v->get("v")->get("v") ? v->get("v")->get("v")->number() : 0); + std::memcpy(store.data(), &i, sizeof i); + } else { + const float f = fval(v->get("v") ? v->get("v")->get("v") : nullptr); + std::memcpy(store.data(), &f, sizeof f); + } + slot.storage = store.data(); + return true; +} + +static float fld(const minijson::Value* v, const char* name) { + const minijson::Value* f = v ? v->get(name) : nullptr; + return fval(f ? f->get("v") : nullptr); +} +static std::int32_t ild(const minijson::Value* v, const char* name) { + const minijson::Value* f = v ? v->get(name) : nullptr; + return static_cast(f && f->get("v") ? f->get("v")->number() : 0); +} + +static Tv describe(const Slot& s) { + Tv out = shim::trace::tv::struct_(); + out.add("t", shim::trace::tv::str(kind_name(s.kind))); + if (s.kind == Kind::String) { + const std::string* str = static_cast(s.storage); + out.add("v", shim::trace::tv::str(str->data(), str->size())); + } else if (s.kind == Kind::Vec3) { + float c[3]; + std::memcpy(c, s.storage, sizeof c); + out.add("x", shim::trace::tv::f32(c[0])); + out.add("y", shim::trace::tv::f32(c[1])); + out.add("z", shim::trace::tv::f32(c[2])); + } else if (s.kind == Kind::Rect) { + std::int32_t c[4]; + std::memcpy(c, s.storage, sizeof c); + out.add("x", shim::trace::tv::i32(c[0])); + out.add("y", shim::trace::tv::i32(c[1])); + out.add("w", shim::trace::tv::i32(c[2])); + out.add("h", shim::trace::tv::i32(c[3])); + } else if (s.kind == Kind::Colour) { + float c[4]; + std::memcpy(c, s.storage, sizeof c); + out.add("r", shim::trace::tv::f32(c[0])); + out.add("g", shim::trace::tv::f32(c[1])); + out.add("b", shim::trace::tv::f32(c[2])); + out.add("a", shim::trace::tv::f32(c[3])); + } else if (s.kind == Kind::Int) { + std::int32_t i; + std::memcpy(&i, s.storage, sizeof i); + out.add("v", shim::trace::tv::i32(i)); + } else { + float f; + std::memcpy(&f, s.storage, sizeof f); + out.add("v", shim::trace::tv::f32(f)); + } + return out; +} + +int main(int argc, char** argv) { + const std::string trace = getenv_str("SOTS_M1_TRACE"); + const std::string data = getenv_str("SOTS_DATA_DIR"); + if (trace.empty() || data.empty()) { + std::printf("SKIP: set SOTS_M1_TRACE (golden LoadFile log) and SOTS_DATA_DIR (game data root)\n"); + return 0; + } + const std::string out_path = argc > 1 ? argv[1] : "m1-replay-impl.jsonl"; + + std::ifstream in(trace); + if (!in) { + std::printf("FAIL: cannot open %s\n", trace.c_str()); + return 1; + } + std::FILE* out = std::fopen(out_path.c_str(), "wb"); + if (!out) { + std::printf("FAIL: cannot write %s\n", out_path.c_str()); + return 1; + } + + unsigned calls = 0, files_missing = 0; + std::string line; + while (std::getline(in, line)) { + minijson::Value rec; + if (!minijson::Parser(line).parse(rec)) continue; + const minijson::Value* hook = rec.get("hook"); + if (!hook || hook->string() != kHook) continue; + const minijson::Value* args = rec.get("args"); + const minijson::Value* side = rec.get("side"); + if (!args || !args->isArray() || args->arr.empty() || !side || !side->isObject()) continue; + + std::string file; + double scale = 0; + for (const minijson::Value& a : args->arr) { + const minijson::Value* n = a.get("n"); + const std::string name = n ? n->string() : ""; + if (name == "file" && a.get("v")) file = a.get("v")->string(); + if (name == "scale" && a.get("v")) scale = a.get("v")->number(); + } + + // Slots from the `before` snapshots, in the record's (map) order. + std::vector slots; + std::vector> stores; + std::vector strings; // String slots' storage on the host + slots.reserve(side->obj.size()); + stores.reserve(side->obj.size()); + strings.reserve(side->obj.size()); + for (const auto& kv : side->obj) { + const minijson::Value* before = kv.second.get("before"); + Slot s; + stores.emplace_back(); + if (!before || !slot_from_tv(kv.first, *before, s, stores.back())) { + std::printf("FAIL: call %g region %s is not a typed struct\n", rec.get("call_id")->number(), kv.first.c_str()); + return 1; + } + if (s.kind == Kind::String) { + const minijson::Value* bv = before->get("v"); + const minijson::Value* txt = bv ? bv->get("v") : nullptr; + strings.push_back(txt && txt->get("v") ? txt->get("v")->string() : ""); + s.storage = &strings.back(); + } + slots.push_back(std::move(s)); + } + const ExternWriter ext = [](Slot& sl, std::string_view v) { *static_cast(sl.storage) = std::string(v); }; + + shim::trace::Record r; + r.hook = kHook; + r.call_id = static_cast(rec.get("call_id")->number()); + r.mode = shim::trace::Mode::Trace; + + fs::path path; + std::string text; + if (resolve_ci(data, file, path) && read_file(path, text)) { + apply(file, text, slots, scale, nullptr, ext); + } else { + ++files_missing; + std::printf(" note: %s not under %s (leaving defaults, as the original does)\n", file.c_str(), data.c_str()); + } + for (const Slot& s : slots) { + shim::trace::SideEntry e; + e.name = s.key; + e.before_kind = shim::trace::SideEntry::Absent; + e.after = describe(s); + r.side.push_back(std::move(e)); + } + shim::trace::Buf b; + shim::trace::emit_record(b, r); + std::fwrite(b.data(), 1, b.size(), out); + ++calls; + } + std::fclose(out); + std::printf("replayed %u LoadFile calls (%u files missing) -> %s\n", calls, files_missing, out_path.c_str()); + if (calls == 0) { + std::printf("FAIL: no %s records in %s\n", kHook, trace.c_str()); + return 1; + } + const int rc = tracetest::run_tracecmp(trace, "--hook " + std::string(kHook) + " --replay " + out_path); + if (rc == tracetest::kSkipped) return 0; + std::printf("tracecmp --replay exit %d\n", rc); + return rc == 0 ? 0 : 1; +} diff --git a/tests/game_config/unit_tests.cpp b/tests/game_config/unit_tests.cpp new file mode 100644 index 0000000..fd09cfe --- /dev/null +++ b/tests/game_config/unit_tests.cpp @@ -0,0 +1,258 @@ +// game::config unit tests: typed scanners, colour/vec3/rect rules, the pair stream (through +// mars::text::parse_flat_kv), first-occurrence-wins and the diagnostics. All samples are +// hand-written. +#include +#include +#include +#include +#include + +#include "game/config/config_loader.h" + +using namespace game::config; + +static int g_fail = 0, g_pass = 0; +#define CHECK(cond) \ + do { \ + if (cond) ++g_pass; \ + else { ++g_fail; std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); } \ + } while (0) + +static bool same_bits(float a, float b) { return std::memcmp(&a, &b, sizeof a) == 0; } +static const double kScale = 0.017453292519943295; // pi/180 + +static void test_scan_int() { + std::int32_t v = 77; + CHECK(scan_int("10", v) && v == 10); + CHECK(scan_int(" -5", v) && v == -5); + CHECK(scan_int("10.7", v) && v == 10); // %d stops at the dot + CHECK(scan_int("+3x", v) && v == 3); + CHECK(scan_int("\t\n42", v) && v == 42); + v = 77; + CHECK(!scan_int("abc", v) && v == 77); // no digits: untouched + CHECK(!scan_int("", v) && v == 77); + CHECK(!scan_int("-", v) && v == 77); + CHECK(!scan_int(".5", v) && v == 77); + CHECK(scan_int("4294967296", v) && v == 0); // wraps like the CRT accumulator + CHECK(scan_int("2147483648", v) && v == -2147483647 - 1); + std::size_t pos = 0; + CHECK(scan_int_at("1 2", pos, v) && v == 1 && pos == 1); + CHECK(scan_int_at("1 2", pos, v) && v == 2 && pos == 3); + CHECK(!scan_int_at("1 2", pos, v) && pos == 3); +} + +static void test_scan_float() { + float f = 9; + CHECK(scan_float("0.5", f) && f == 0.5f); + CHECK(scan_float("1e3", f) && f == 1000.0f); + CHECK(scan_float(".5", f) && f == 0.5f); + CHECK(scan_float("5.", f) && f == 5.0f); + CHECK(scan_float("-.25", f) && f == -0.25f); + CHECK(scan_float(" 7", f) && f == 7.0f); + CHECK(scan_float("1e", f) && f == 1.0f); // dangling exponent: mantissa only + CHECK(scan_float("2E+2z", f) && f == 200.0f); + CHECK(scan_float("0.1", f) && same_bits(f, 0.1f)); + CHECK(scan_float("3.14159265358979", f) && same_bits(f, 3.14159265358979f)); + CHECK(scan_float("1.5abc", f) && f == 1.5f); + f = 9; + CHECK(!scan_float("abc", f) && f == 9); + CHECK(!scan_float(".", f) && f == 9); + CHECK(!scan_float("", f) && f == 9); + CHECK(!scan_float("-e5", f) && f == 9); +} + +static float c255(int v) { return static_cast(static_cast(v) / 255.0); } + +static void test_colour() { + float c[4]; + parse_colour("48 29 2", c); + CHECK(same_bits(c[0], c255(48)) && same_bits(c[1], c255(29)) && same_bits(c[2], c255(2)) && c[3] == 1.0f); + parse_colour("", c); + CHECK(c[0] == 1.0f && c[1] == 1.0f && c[2] == 1.0f && c[3] == 1.0f); // all default 255 + parse_colour("300 -5 0 128", c); + CHECK(c[0] == 1.0f && c[1] == 0.0f && c[2] == 0.0f && same_bits(c[3], c255(128))); // clamped + parse_colour("12 34", c); + CHECK(same_bits(c[0], c255(12)) && same_bits(c[1], c255(34)) && c[2] == 1.0f && c[3] == 1.0f); + parse_colour(" 0\t190 195 255", c); + CHECK(c[0] == 0.0f && same_bits(c[1], c255(190)) && same_bits(c[2], c255(195)) && c[3] == 1.0f); + parse_colour("1 2 x 4", c); // the scan stops at the first non-number; the rest default + CHECK(same_bits(c[0], c255(1)) && same_bits(c[1], c255(2)) && c[2] == 1.0f && c[3] == 1.0f); + float prev = -1; + bool mono = true; + for (int i = 0; i <= 255; ++i) { + parse_colour(std::to_string(i), c); + if (!(c[0] >= 0 && c[0] <= 1 && c[0] > prev)) mono = false; + prev = c[0]; + } + CHECK(mono); +} + +static void test_vec3_rect() { + float v[3] = {9, 9, 9}; + CHECK(parse_vec3("0 0 50.0", v) == 3 && v[0] == 0 && v[1] == 0 && v[2] == 50.0f); + float w[3] = {9, 9, 9}; + CHECK(parse_vec3("1.5 x", w) == 1 && w[0] == 1.5f && w[1] == 9 && w[2] == 9); // the rest untouched + CHECK(parse_vec3("", w) == 0); + std::int32_t r[4] = {9, 9, 9, 9}; + CHECK(parse_rect("16 6\t122 40", r) == 4 && r[0] == 16 && r[1] == 6 && r[2] == 122 && r[3] == 40); + std::int32_t q[4] = {9, 9, 9, 9}; + CHECK(parse_rect("1 2.5 3 4", q) == 2 && q[0] == 1 && q[1] == 2 && q[2] == 9); // ".5" stops %d +} + +static void test_scale() { + CHECK(same_bits(scale_float(90.0f, kScale), static_cast(90.0 * kScale))); + CHECK(same_bits(scale_float(3.0f, kScale), 0.052359879f)); + CHECK(same_bits(scale_float(85.0f, kScale), 1.48352981f)); + CHECK(same_bits(scale_float(-60.0f, kScale), -1.04719758f)); + CHECK(scale_float(0.0f, kScale) == 0.0f); + CHECK(scale_float(2.0f, 0.5) == 1.0f); +} + +static void test_pairs() { + auto eq = [](const std::vector& p, std::vector> want) { + if (p.size() != want.size()) return false; + for (std::size_t i = 0; i < p.size(); ++i) + if (p[i].key != want[i].first || p[i].value != want[i].second) return false; + return true; + }; + CHECK(eq(pairs("A 1\nB 2\n"), {{"A", "1"}, {"B", "2"}})); + CHECK(eq(pairs("A 1\r\nB 2\r\n"), {{"A", "1"}, {"B", "2"}})); + CHECK(eq(pairs("// c\nA 1 // t\n\n\t B\t\"x y\"\n"), {{"A", "1"}, {"B", "x y"}})); + // duplicates are all delivered, in order (first-wins is applied by apply()) + CHECK(eq(pairs("A 1\nA 2\na 3\n"), {{"A", "1"}, {"A", "2"}, {"a", "3"}})); + // the last pair touches end-of-file: lost; a trailing blank or comment keeps it + CHECK(eq(pairs("A 1\nB 2"), {{"A", "1"}})); + CHECK(eq(pairs("A 1\nB 2 "), {{"A", "1"}, {"B", "2"}})); + CHECK(eq(pairs("A 1\nB 2 // c"), {{"A", "1"}, {"B", "2"}})); + // a key alone takes the next line's key as its value (the stream is not line-based) + CHECK(eq(pairs("A\nB 2\nC 3\n"), {{"A", "B"}, {"2", "C"}})); + // three tokens on a line: the third starts the next pair + CHECK(eq(pairs("A 1 2\nB 3\n"), {{"A", "1"}, {"2", "B"}})); + // blocks are skipped whole, nesting included; unclosed runs to the end; stray } ignored + CHECK(eq(pairs("A 1\nblk {\n X 1\n sub {\n Y 2\n }\n Z 3\n}\nB 2\n"), {{"A", "1"}, {"B", "2"}})); + CHECK(eq(pairs("A 1\nblk {\n X 1\n"), {{"A", "1"}})); + CHECK(eq(pairs("}\nA 1\n}\nB 2\n"), {{"A", "1"}, {"B", "2"}})); + CHECK(eq(pairs("vars\n{\nA 1\nB 2\n}\n"), {})); + // a glued comment after a quoted value is still a comment + CHECK(eq(pairs("COL \"0 0 0\"// \" 92 76 20\"\n"), {{"COL", "0 0 0"}})); + CHECK(eq(pairs("E \"\"\n"), {{"E", ""}})); +} + +struct Table { + std::int32_t i = 7; + float f = 1.5f; + float s = 2.0f; + float c[4] = {0, 0, 0, 0}; + float v[3] = {0, 0, 0}; + std::int32_t r[4] = {0, 0, 0, 0}; + std::string str; + std::vector slots() { + return {Slot{"COUNT", Kind::Int, &i, nullptr, false}, Slot{"RATE", Kind::Float, &f, nullptr, false}, + Slot{"ANGLE", Kind::FloatScaled, &s, nullptr, false}, Slot{"TINT", Kind::Colour, c, nullptr, false}, + Slot{"OFFSET", Kind::Vec3, v, nullptr, false}, Slot{"BOX", Kind::Rect, r, nullptr, false}, + Slot{"NAME", Kind::String, &str, nullptr, false}}; + } +}; +static const unsigned kSlots = 7; + +static void test_apply() { + std::vector log; + auto lg = [&log](const std::string& l) { log.push_back(l); }; + ExternWriter ext = [](Slot& s, std::string_view v) { *static_cast(s.storage) = std::string(v); }; + + { + Table t; + std::vector sl = t.slots(); + Stats st = apply("Data/x.txt", + "count 3\nRATE 0.25\nAngle 90\nTINT \"255 0 128\"\nOFFSET \"0 0 50.0\"\nBOX \"16 6 122 40\"\nname \"spy_node\"\n", + sl, 0.5, lg, ext); + CHECK(st.applied == kSlots && st.unknown == 0 && st.duplicate == 0 && st.missing == 0); + CHECK(t.i == 3 && t.f == 0.25f && t.s == 45.0f); + CHECK(t.c[0] == 1.0f && t.c[1] == 0.0f && same_bits(t.c[2], c255(128)) && t.c[3] == 1.0f); + CHECK(t.v[2] == 50.0f && t.r[2] == 122 && t.str == "spy_node"); + CHECK(log.empty()); + for (const Slot& s : sl) CHECK(s.consumed); + } + { // first occurrence wins; the repeat is logged as multiply defined + Table t; + std::vector sl = t.slots(); + log.clear(); + Stats st = apply("f", "COUNT 1\nCOUNT 2\ncount 3\n", sl, 1, lg); + CHECK(t.i == 1 && st.applied == 1 && st.duplicate == 2 && st.missing == kSlots - 1); + CHECK(log.size() == 2 + kSlots - 1 && log[0] == "[f] COUNT not recognized or is multiply defined." && + log[1] == "[f] count not recognized or is multiply defined."); + CHECK(log[2] == "[f] RATE expected but not found."); + } + { // unknown keys are ignored and logged; absent keys keep their defaults + Table t; + std::vector sl = t.slots(); + log.clear(); + Stats st = apply("f", "NOPE 5\nRATE 2\n", sl, 1, lg); + CHECK(st.unknown == 1 && st.applied == 1 && st.missing == kSlots - 1); + CHECK(t.i == 7 && t.f == 2.0f && t.s == 2.0f && t.c[0] == 0.0f); + CHECK(log[0] == "[f] NOPE not recognized or is multiply defined."); + } + { // a value the scanner rejects leaves an int/float word alone but still counts as consumed + Table t; + std::vector sl = t.slots(); + Stats st = apply("f", "COUNT abc\nRATE x\n", sl, 1, nullptr); + CHECK(t.i == 7 && t.f == 1.5f && st.applied == 2); + CHECK(sl[0].consumed && sl[1].consumed); + } + { // scaled float: the word is multiplied even when the scan fails + Table t; + std::vector sl = t.slots(); + apply("f", "ANGLE x\n", sl, 0.25, nullptr); + CHECK(t.s == 0.5f); + } + { // string slots without an ExternWriter are left alone (but consumed) + Table t; + std::vector sl = t.slots(); + Stats st = apply("f", "NAME foo\n", sl, 1, nullptr); + CHECK(t.str.empty() && st.applied == 1 && sl[6].consumed); + } + { // colour with a bare (unquoted) value: only the first token is the value, the rest + // become keys -- exactly the failure mode the data avoids by quoting colours + Table t; + std::vector sl = t.slots(); + log.clear(); + Stats st = apply("f", "TINT 10 20 30\nCOUNT 1\n", sl, 1, lg); + CHECK(same_bits(t.c[0], c255(10)) && t.c[1] == 1.0f); + CHECK(st.unknown == 1 && log[0] == "[f] 20 not recognized or is multiply defined."); // "20 30" pair + CHECK(t.i == 1); + } + { // no trailing newline: the last pair never arrives + Table t; + std::vector sl = t.slots(); + Stats st = apply("f", "COUNT 1\nRATE 0.5", sl, 1, nullptr); + CHECK(t.i == 1 && t.f == 1.5f && st.missing == kSlots - 1); + } + { // a file that is one big block sets nothing + Table t; + std::vector sl = t.slots(); + Stats st = apply("f", "vars\n{\nCOUNT 1\nRATE 0.5\n}\n", sl, 1, nullptr); + CHECK(st.applied == 0 && st.missing == kSlots && t.i == 7); + } + { // Unknown-kind slot: never written, consumed when present + std::int32_t w = 5; + std::vector sl = {Slot{"W", Kind::Unknown, &w, nullptr, false}}; + Stats st = apply("f", "W 9\n", sl, 1, nullptr); + CHECK(w == 5 && st.applied == 1); + } + CHECK(width(Kind::Colour) == 16 && width(Kind::Int) == 4 && width(Kind::FloatScaled) == 4); + CHECK(width(Kind::Vec3) == 12 && width(Kind::Rect) == 16 && width(Kind::String) == 24 && width(Kind::Unknown) == 4); + CHECK(std::string(kind_name(Kind::Colour)) == "colour" && std::string(kind_name(Kind::FloatScaled)) == "fscaled"); + CHECK(std::string(kind_name(Kind::String)) == "string" && std::string(kind_name(Kind::Vec3)) == "vec3"); +} + +int main() { + test_scan_int(); + test_scan_float(); + test_colour(); + test_vec3_rect(); + test_scale(); + test_pairs(); + test_apply(); + std::printf("game_config unit tests: %d passed, %d failed\n", g_pass, g_fail); + return g_fail ? 1 : 0; +}