alex/sots-engine
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge m1 config loader (cmake union)

Browse source
This commit is contained in:
alex 2026-09-07 22:16:55 -04:00
commit 8b231c0a86
12 changed files with 1392 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
CMakeLists.txt
View file

@ -24,6 +24,7 @@ add_subdirectory(src/game/sim) # strategic formulas, pure (lib sots
add_subdirectory(src/mars/vfs) # .gob ZIP reader + native override (lib mars_vfs, miniz) add_subdirectory(src/mars/vfs) # .gob ZIP reader + native override (lib mars_vfs, miniz)
add_subdirectory(src/mars/stream) # Streamable save format + gzip (lib mars_stream) add_subdirectory(src/mars/stream) # Streamable save format + gzip (lib mars_stream)
add_subdirectory(src/mars/rng) # MT19937 (lib mars_rng) add_subdirectory(src/mars/rng) # MT19937 (lib mars_rng)
add_subdirectory(src/game/config) # flat KEY/value constants loader (lib sots_game_config)
add_subdirectory(src/game/data) # typed catalogs on mars/parse+text (lib game_data) add_subdirectory(src/game/data) # typed catalogs on mars/parse+text (lib game_data)
add_subdirectory(src/game/design) # ship-design rules + derived stats (lib game_design) add_subdirectory(src/game/design) # ship-design rules + derived stats (lib game_design)
@ -48,8 +49,13 @@ if(WIN32)
third_party/minhook/src/hde/hde32.c) third_party/minhook/src/hde/hde32.c)
target_include_directories(minhook PUBLIC third_party/minhook/include) target_include_directories(minhook PUBLIC third_party/minhook/include)
# ---- hooks: one descriptor per hooked game function (src/shim/hooks/*) ----
add_library(shim_hooks STATIC src/shim/hooks/global_consts.cpp)
target_link_libraries(shim_hooks PUBLIC shim_trace sots_addresses sots_game_config)
target_compile_options(shim_hooks PRIVATE -Wall -Wextra -Werror)
add_library(binkw32 SHARED src/shim/main.cpp src/shim/binkw32.def) add_library(binkw32 SHARED src/shim/main.cpp src/shim/binkw32.def)
target_link_libraries(binkw32 PRIVATE minhook sots_addresses shim_trace) target_link_libraries(binkw32 PRIVATE minhook sots_addresses shim_trace shim_hooks)
target_compile_definitions(binkw32 PRIVATE target_compile_definitions(binkw32 PRIVATE
SHIM_BUILD_ID="${SHIM_BUILD_ID}" SHIM_BUILD_ID="${SHIM_BUILD_ID}"
SOTS_ADDR_PROVENANCE="${SOTS_ADDR_PROVENANCE}") SOTS_ADDR_PROVENANCE="${SOTS_ADDR_PROVENANCE}")
@ -61,7 +67,7 @@ else()
add_executable(addr_smoke tests/addr_smoke.cpp) add_executable(addr_smoke tests/addr_smoke.cpp)
target_link_libraries(addr_smoke PRIVATE sots_addresses) target_link_libraries(addr_smoke PRIVATE sots_addresses)
add_test(NAME addr_smoke COMMAND addr_smoke) add_test(NAME addr_smoke COMMAND addr_smoke)
foreach(_t mars_parse mars_text game_sim mars_vfs mars_stream shim_trace game_data game_design) foreach(_t mars_parse game_config game_data game_design game_sim mars_stream mars_text mars_vfs shim_trace)
if(EXISTS ${CMAKE_SOURCE_DIR}/tests/${_t}/CMakeLists.txt) if(EXISTS ${CMAKE_SOURCE_DIR}/tests/${_t}/CMakeLists.txt)
add_subdirectory(tests/${_t}) add_subdirectory(tests/${_t})
endif() endif()

106
docs/M1.md Normal file
View file

@ -0,0 +1,106 @@
# M1 — flat KEY/value config loader (`Mars::GlobalConsts::LoadFile`) old-vs-new on the live game
**Result (2026-09-07/08):** the first real old-vs-new verification. Trace mode: 19 `LoadFile` calls
(1088 constant slots), `tracecmp.py` exit 0. Compare mode: **19/19 compared, 0 divergences**.
Replace mode: our loader fed the game its constants alone; loading `ref-turn2.sav` and pressing
End Turn reproduced the determinism oracle byte for byte (`(Autosave).sav` = `978041ac…`,
`(Autosave EndTurn).sav` = `bb4fd9ac…`). The offline replay test (golden trace through the
host build) also passes. Game left running on VM140 in `hooks=trace` at the main menu.
## What was hooked
`Mars::GlobalConsts::LoadFile(const char* file, GlobalConstMap* consts)` — cdecl, verified, so
the `Hook<Descriptor>` template applies directly (`src/shim/hooks/global_consts.{h,cpp}`,
installed from `src/shim/main.cpp` via `InstallTemplateHook<>`). It is called once per data file
from `LoadAll()` inside `Application::Initialize`, before the intro movie; the files it touches
(19, in map order of the file names):
```
Data/Combat/{actor,assaultshuttle,camera,drone,gravboat,ortgay,planet,sensors,ship,spyship,terrain,vnhomeworld}.txt
Data/encounters.txt Data/globals.txt Data/options.txt Data/Species.txt
Data/Strategy/AI/AIRebellion.txt Data/Strategy/starmap.txt Data/Strategy/StrategyVars.txt
```
`ctechvars.txt`, `shipai.txt`, `damfx*.txt`, `playercolors.txt`, `systemnames.txt` are *not*
GlobalConsts files (other readers).
### Region model
The map handed to `LoadFile` is an MSVC 2010 `std::map<const char*, GlobalConst*>` holding the
constants registered for that file (`key -> {storage, key, parser, file}`; layouts are now in
`sots_addresses.h`). `regions()` walks it in order before the original runs and declares **one
region per constant, named by the key**, sized and described by the parser the registering stub
pushed:
| parser (VA) | kind | width | describer |
|---|---|---|---|
| `GlobalConst_ParseInt` 0x008b7000 | `int` | 4 | `{t:"int", v:i32}` |
| `GlobalConst_ParseFloat` 0x008b7020 | `float` | 4 | `{t:"float", v:f32}` |
| `GlobalConst_ParseFloatScaled` 0x008b70e0 | `fscaled` | 4 | `{t:"fscaled", v:f32}` |
| `GlobalConst_ParseColour` 0x008b7110 | `colour` | 16 | `{t:"colour", r,g,b,a:f32}` |
| `GlobalConst_ParseRect` 0x008b7040 **(new)** | `rect` | 16 | `{t:"rect", x,y,w,h:i32}` |
| `GlobalConst_ParseVec3` 0x008b7080 **(new)** | `vec3` | 12 | `{t:"vec3", x,y,z:f32}` |
| `GlobalConst_ParseString` 0x008b7670 **(new)** | `string` | 24 | `{t:"string", v:str}` (decoded MSVC `std::string`) |
Args: `file` (str), `consts` (ptr), `nkeys` (u32) and `scale` (f64, the scaled-float multiplier)
— the last so a golden log replays offline with nothing from the exe. Return: void.
Discoveries the RE notes did not have (all fed back into `sots-re/ghidra/addresses.json`, status
`verified-by-trace`, and regenerated into `include/generated/sots_addresses.h`):
- **Three more parsers.** 146 keys (`*_NAME`, `*_SOUND`, `*_TEXTURENAME`, `DERELICT_SECTION_nn`,
`Data/options.txt` …) are `std::string` slots (0x18 bytes: 16-byte SSO buffer / heap pointer,
size @+0x10, capacity @+0x14). Five `HIVER_SPAWN_*` keys are `float[3]` (`"0 0 50.0"`). Two
`COMBATSETUP_CARD_*_RECT` keys are `int[4]`.
- **The scaled-float constant at 0x00af5210 is a `double`** (pi/180 = 0x3f91df46a2529d39), not a
float; reading it as float gives -2.85e-18. The product is computed in double and stored as
float: `(float)((double)v * scale)`.
- `Data/Strategy/StrategyVars.txt` and `Data/encounters.txt` have no trailing newline, so the
original never applies their last key (`CIVILIAN_BURDEN_RATIO`, `HERALD_SPEECH_MAX_INVERVAL`):
visible in the trace as the only unchanged regions in those files and reproduced by ours
("expected but not found", `missing 1` in both).
## `ours` — `src/game/config/config_loader.{h,cpp}` (lib `sots_game_config`)
- File bytes come from the game's own `gobio::ReadFile` (honours the mount order / mods) and are
released with `RefCounted_Release`; the host tests read plain files.
- Pairs come from `mars::text::parse_flat_kv` (engine-parity: Mars::Script tokenizer, blocks
skipped, trailing pair dropped). `apply()` implements **first-occurrence-wins** on top of the
raw pair list (a slot is consumed on first sight; repeats and unknown keys are logged
"`[file] KEY not recognized or is multiply defined.`", absent slots
"`[file] KEY expected but not found.`", to `shim.log` with a `cfg:` prefix).
- Typed writes mirror `sscanf`: `%d` (CRT wrap-around accumulator, no write when no digit),
`%f` (one rounding from the text via `strtold` → float), colour (`%d %d %d %d`, defaults 255,
`/255.0` in double, clamp, always writes all four), vec3 / rect (components until the first
scan failure), scaled float (scan then multiply, whatever the scan did).
- `String` slots are written through an `ExternWriter`: the shim hands the text to the game's
own `ParseString` (a MinGW-built `std::string` cannot be handed to MSVCR100). In compare mode
that runs on the scratch copy of the object, which starts as the image default (empty short
string), so nothing belonging to the live object is ever freed; long strings leak one heap
block per compare call (startup only).
## Runs (`/bulk-storage/re-lab/shim/traces/`, build `4f0a9db-dirty-20260908T0205Z`)
| file | mode | calls | result |
|---|---|---|---|
| `m1-trace-golden.jsonl` | trace | 19 LoadFile + 1 selftest | `tracecmp.py` exit 0, 0 invalid |
| `m1-compare.jsonl` | compare | 19 compared | **0 diverged**, 0 errors, exit 0 |
| replace (no log) | replace | 19 | End Turn from `ref-turn2.sav`: `(Autosave).sav` `978041ac…` / `(Autosave EndTurn).sav` `bb4fd9ac…` = oracle |
| `m1-shim.log` | | | all four runs' banners, `cfg:` stats per file |
| `m1-*.png` | | | main menu (trace, compare, final), turn 2 / turn 3 in replace |
Offline: `SOTS_M1_TRACE=<golden> SOTS_DATA_DIR=<extract root> build-host/tests/game_config/game_config_replay`
→ `tracecmp.py --replay` exit 0 (19 calls, 0 files missing). Host suite: `ctest` 22/22.
## Gotchas
1. `shim.cfg` per-hook line: `hook.Mars::GlobalConsts::LoadFile=compare|replace` (default `hooks=trace`).
2. `Z:` is per-logon; from SSH there is no share. Push the dist to `C:\SOTS\shimdist\` with scp
and run `deploy.ps1 -Dist C:\SOTS\shimdist`. Inline PowerShell over ssh mangles `$` and
quotes — ship `.ps1` files (`C:\SOTS\ui\{status,loginfo,lastrun,relaunch,hashes}.ps1`).
3. Local cmake is 3.22 (< 3.25 required): use `uvx --from cmake cmake` with `Unix Makefiles`.
4. The per-call slot table is kept in statics between `regions()` → `rebind()` → `ours()`; fine
because `LoadFile` is single-threaded at startup, but do not reuse the pattern for re-entrant
hooks.
5. `Data/Combat/ctechvars.txt` style files (a single `name { … }` block) would set nothing
through this loader — they are simply not GlobalConsts files, as the trace confirms.

8
src/game/config/CMakeLists.txt Normal file
View file

@ -0,0 +1,8 @@
# sots_game_config -- the flat KEY/value constant loader (GlobalConsts): typed slots,
# first-occurrence-wins, the pull-tokenizer pairing rules. Pure; host + cross.
# Include from the root with add_subdirectory(src/game/config) after src/mars/text.
add_library(sots_game_config STATIC config_loader.cpp)
target_include_directories(sots_game_config PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
target_link_libraries(sots_game_config PUBLIC mars_text)
target_compile_features(sots_game_config PUBLIC cxx_std_17)
target_compile_options(sots_game_config PRIVATE -Wall -Wextra -Werror)

207
src/game/config/config_loader.cpp Normal file
View file

@ -0,0 +1,207 @@
#include "game/config/config_loader.h"
#include <cstdlib>
#include <cstring>
#include "mars/text/flat_kv.h"
#include "mars/text/value.h"
namespace game::config {
const char* kind_name(Kind k) {
switch (k) {
case Kind::Int: return "int";
case Kind::Float: return "float";
case Kind::FloatScaled: return "fscaled";
case Kind::Colour: return "colour";
case Kind::Vec3: return "vec3";
case Kind::Rect: return "rect";
case Kind::String: return "string";
case Kind::Unknown: break;
}
return "unknown";
}
// ---- scanf-style scalar scanners --------------------------------------------------------------
namespace {
// The whitespace set C's isspace() has in the "C" locale (what scanf skips).
bool c_space(char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; }
bool digit(char c) { return c >= '0' && c <= '9'; }
} // namespace
bool scan_int_at(std::string_view s, std::size_t& i, std::int32_t& out) {
std::size_t p = i;
while (p < s.size() && c_space(s[p])) ++p;
bool neg = false;
if (p < s.size() && (s[p] == '+' || s[p] == '-')) neg = s[p++] == '-';
if (p >= s.size() || !digit(s[p])) return false;
std::uint32_t acc = 0; // wraps like the CRT accumulator
while (p < s.size() && digit(s[p])) acc = acc * 10u + static_cast<std::uint32_t>(s[p++] - '0');
if (neg) acc = 0u - acc;
out = static_cast<std::int32_t>(acc);
i = p;
return true;
}
bool scan_float_at(std::string_view text, std::size_t& pos, float& out) {
std::size_t i = pos;
const std::size_t n = text.size();
while (i < n && c_space(text[i])) ++i;
const std::size_t start = i;
if (i < n && (text[i] == '+' || text[i] == '-')) ++i;
std::size_t digits = 0;
while (i < n && digit(text[i])) ++i, ++digits;
if (i < n && text[i] == '.') {
++i;
while (i < n && digit(text[i])) ++i, ++digits;
}
if (digits == 0) return false;
if (i < n && (text[i] == 'e' || text[i] == 'E')) {
std::size_t j = i + 1;
if (j < n && (text[j] == '+' || text[j] == '-')) ++j;
if (j < n && digit(text[j])) {
while (j < n && digit(text[j])) ++j;
i = j;
}
}
// One rounding from the decimal text: the CRT's float scanner converts the collected
// digits straight to single precision. strtold + cast keeps that (an 80-bit intermediate
// cannot double-round the short literals these files hold).
const std::string prefix(text.substr(start, i - start));
out = static_cast<float>(std::strtold(prefix.c_str(), nullptr));
pos = i;
return true;
}
bool scan_int(std::string_view text, std::int32_t& out) {
std::size_t i = 0;
return scan_int_at(text, i, out);
}
bool scan_float(std::string_view text, float& out) {
std::size_t i = 0;
return scan_float_at(text, i, out);
}
void parse_colour(std::string_view text, float rgba[4]) {
std::int32_t c[4] = {255, 255, 255, 255};
std::size_t i = 0;
for (int k = 0; k < 4; ++k)
if (!scan_int_at(text, i, c[k])) break;
for (int k = 0; k < 4; ++k) {
double f = static_cast<double>(c[k]) / 255.0;
if (f < 0.0) f = 0.0;
if (f > 1.0) f = 1.0;
rgba[k] = static_cast<float>(f);
}
}
int parse_vec3(std::string_view text, float xyz[3]) {
std::size_t i = 0;
int k = 0;
for (; k < 3; ++k)
if (!scan_float_at(text, i, xyz[k])) break;
return k;
}
int parse_rect(std::string_view text, std::int32_t rect[4]) {
std::size_t i = 0;
int k = 0;
for (; k < 4; ++k)
if (!scan_int_at(text, i, rect[k])) break;
return k;
}
float scale_float(float v, double scale) { return static_cast<float>(static_cast<double>(v) * scale); }
// ---- pairs -----------------------------------------------------------------------------------------
std::vector<Pair> pairs(std::string_view text) {
std::vector<Pair> out;
const mars::text::Result<mars::text::FlatKV> r = mars::text::parse_flat_kv(text);
out.reserve(r.value.size());
for (const mars::text::KvEntry& e : r.value.entries()) out.push_back(Pair{e.key, e.value.text});
return out;
}
// ---- writes --------------------------------------------------------------------------------------
void write_slot(Slot& slot, std::string_view value, double scale, const ExternWriter& ext) {
switch (slot.kind) {
case Kind::Int: {
std::int32_t v = 0;
if (scan_int(value, v)) std::memcpy(slot.storage, &v, sizeof v);
break;
}
case Kind::Float: {
float v = 0;
if (scan_float(value, v)) std::memcpy(slot.storage, &v, sizeof v);
break;
}
case Kind::FloatScaled: {
// The original scans into the word and then multiplies it, whatever the scan did.
float v = 0;
std::memcpy(&v, slot.storage, sizeof v);
scan_float(value, v);
v = scale_float(v, scale);
std::memcpy(slot.storage, &v, sizeof v);
break;
}
case Kind::Colour: {
float rgba[4];
parse_colour(value, rgba);
std::memcpy(slot.storage, rgba, sizeof rgba);
break;
}
case Kind::Vec3: {
float xyz[3];
std::memcpy(xyz, slot.storage, sizeof xyz);
parse_vec3(value, xyz);
std::memcpy(slot.storage, xyz, sizeof xyz);
break;
}
case Kind::Rect: {
std::int32_t rect[4];
std::memcpy(rect, slot.storage, sizeof rect);
parse_rect(value, rect);
std::memcpy(slot.storage, rect, sizeof rect);
break;
}
case Kind::String:
if (ext) ext(slot, value);
break;
case Kind::Unknown: break;
}
}
Stats apply(std::string_view file, std::string_view text, std::vector<Slot>& slots, double scale,
const LogFn& log, const ExternWriter& ext) {
Stats st;
const std::string fname(file);
for (const Pair& p : pairs(text)) {
Slot* hit = nullptr;
for (Slot& s : slots)
if (mars::text::equals_fold(s.key, p.key)) {
hit = &s;
break;
}
if (!hit || hit->consumed) {
if (hit) ++st.duplicate;
else ++st.unknown;
if (log) log("[" + fname + "] " + p.key + " not recognized or is multiply defined.");
continue;
}
write_slot(*hit, p.value, scale, ext);
hit->consumed = true;
++st.applied;
}
for (const Slot& s : slots)
if (!s.consumed) {
++st.missing;
if (log) log("[" + fname + "] " + s.key + " expected but not found.");
}
return st;
}
} // namespace game::config

108
src/game/config/config_loader.h Normal file
View file

@ -0,0 +1,108 @@
// game::config -- the flat "KEY value" constant tables (Data/globals.txt,
// Data/Strategy/StrategyVars.txt, Data/Species.txt, Data/encounters.txt, Data/Combat/*.txt).
//
// Model of the original loader (findings: loader-prototypes.md section M1):
//
// * Every constant is a static "slot" registered at image start-up with a key, a storage
// word and one typed parser. The *slot* decides the type; the file never does.
// * The file is read with mars::text::parse_flat_kv, which follows the engine's own loop:
// the Mars::Script pull tokenizer, KEY value pairs, `NAME {` blocks skipped whole, a
// lone `}` ignored, and a final pair whose value touches end-of-file dropped.
// * Keys match case-insensitively (ASCII). A slot is consumed on first sight, so the
// first occurrence wins; a repeat -- or an unknown key -- is logged and ignored. Slots
// the file never names keep whatever the image default was.
//
// Slot kinds and their storage (widths are what the hook declares as side-effect regions):
// Int int32 "%d"
// Float float "%f"
// FloatScaled float "%f", then multiplied by a process constant (pi/180 as a double)
// Colour float[4] "%d %d %d %d", each defaulted to 255, /255, clamped to 0..1
// Vec3 float[3] "%f %f %f" (components not matched stay unchanged)
// Rect int32[4] "%d %d %d %d" (idem)
// String an engine std::string (24 bytes): the text itself. This module cannot build
// the engine's string object, so String slots are written through the
// caller's ExternWriter (the shim hands them to the game's own parser).
// Unknown a parser this code does not model; never written, declared 4 bytes wide.
#pragma once
#include <cstddef>
#include <cstdint>
#include <functional>
#include <string>
#include <string_view>
#include <vector>
namespace game::config {
enum class Kind : std::uint8_t { Int, Float, FloatScaled, Colour, Vec3, Rect, String, Unknown };
constexpr std::size_t width(Kind k) {
switch (k) {
case Kind::Colour: return 16;
case Kind::Vec3: return 12;
case Kind::Rect: return 16;
case Kind::String: return 24;
default: return 4;
}
}
const char* kind_name(Kind k); // "int" | "float" | "fscaled" | "colour" | "vec3" | "rect" | "string" | "unknown"
// One registered constant: where the value lives and how its text is typed.
struct Slot {
std::string key; // as registered (matched case-insensitively)
Kind kind = Kind::Int;
void* storage = nullptr; // width(kind) bytes
void* native = nullptr; // opaque handle for the ExternWriter (the shim keeps the game parser here)
bool consumed = false; // set by apply(): first occurrence wins
};
struct Stats {
unsigned applied = 0; // pairs written to a slot
unsigned duplicate = 0; // repeats of an already consumed key (ignored)
unsigned unknown = 0; // keys with no slot (ignored)
unsigned missing = 0; // slots the file never named
};
using LogFn = std::function<void(const std::string& line)>;
using ExternWriter = std::function<void(Slot& slot, std::string_view value)>;
// ---- typed value parsers -------------------------------------------------------------------
//
// scan_int / scan_float follow C `sscanf` "%d" / "%f": leading whitespace skipped, an optional
// sign, then the longest valid prefix; they return false and leave `out` alone when no digit
// is found. Integers wrap modulo 2^32 like the CRT's accumulator; floats are converted with
// one rounding from the decimal text. The *_at forms continue from `pos` (for "%d %d ..").
bool scan_int(std::string_view text, std::int32_t& out);
bool scan_float(std::string_view text, float& out);
bool scan_int_at(std::string_view text, std::size_t& pos, std::int32_t& out);
bool scan_float_at(std::string_view text, std::size_t& pos, float& out);
// "%d %d %d %d" with every component defaulted to 255; each is divided by 255 (double
// arithmetic, then stored as float) and clamped to [0, 1]. Always writes all four.
void parse_colour(std::string_view text, float rgba[4]);
// "%f %f %f" / "%d %d %d %d": components are written left to right until one fails to scan;
// returns how many were written.
int parse_vec3(std::string_view text, float xyz[3]);
int parse_rect(std::string_view text, std::int32_t rect[4]);
// The scaled-float product as the original computes it (double product, stored as float).
float scale_float(float v, double scale);
// ---- the pairs ------------------------------------------------------------------------------------
struct Pair {
std::string key, value;
};
// Every top-level KEY value pair in file order (duplicates included), per the engine rules.
std::vector<Pair> pairs(std::string_view text);
// Writes the value text into the slot according to its kind. `scale` feeds FloatScaled;
// String slots go through `ext` (nothing happens without one).
void write_slot(Slot& slot, std::string_view value, double scale, const ExternWriter& ext);
// The whole loader for one file: `text` is the file's bytes, `slots` the constants registered
// for that file. Consumed slots are flagged; diagnostics mirror the original's wording:
// "[file] KEY not recognized or is multiply defined." "[file] KEY expected but not found."
// `file` is only used in the log lines.
Stats apply(std::string_view file, std::string_view text, std::vector<Slot>& slots, double scale,
const LogFn& log, const ExternWriter& ext = nullptr);
} // namespace game::config

325
src/shim/hooks/global_consts.cpp Normal file
View file

@ -0,0 +1,325 @@
#include "shim/hooks/global_consts.h"
#include <cstdarg>
#include <cstdio>
#include <cstring>
#include <string>
#include <string_view>
#include "game/config/config_loader.h"
#include "generated/sots_addresses.h"
namespace shim::hooks {
using game::config::Kind;
using game::config::Slot;
using trace::Tv;
namespace {
// ---- in-process layouts (32-bit; facts from sots_addresses.h) --------------------------------
struct GlobalConst { // one registered constant
void* storage; // +0 int32 / float / float[4] / float[3] / int[4] / std::string
const char* key; // +4
void* parser; // +8 one of the GlobalConst_Parse* functions
const char* file; // +c
};
struct MapNode { // MSVC 2010 std::map node
MapNode* left; // +0
MapNode* parent; // +4
MapNode* right; // +8
const char* key; // +c
GlobalConst* value; // +10
std::uint8_t color; // +14
std::uint8_t isnil; // +15 set on the head/nil sentinel only
};
struct MapHeader { // MSVC 2010 std::map object
void* cmp; // +0
MapNode* head; // +4 sentinel: parent = root, left = leftmost, right = rightmost
std::uint32_t size; // +8
};
struct IBuffer { // gobio::Buffer
void* vft; // +0
std::uint32_t refcount; // +4
const char* data; // +8
std::uint32_t size; // +c
};
struct MsvcString { // MSVC 2010 std::string (0x18 bytes)
union {
char buf[16]; // the text when capacity <= 15
const char* ptr; // else a heap block
} u;
std::uint32_t size; // +10
std::uint32_t capacity; // +14
};
#if defined(_WIN32) && defined(__i386__)
#define HOOK_FASTCALL __attribute__((fastcall))
#else
#define HOOK_FASTCALL
#endif
using ReadFileFn = unsigned char(SHIM_CDECL*)(const char* path, IBuffer** out); // bool in AL
using ReleaseFn = int(HOOK_FASTCALL*)(IBuffer* obj); // ECX = obj
using ParseFn = void(SHIM_CDECL*)(void* storage, const char* text); // the slot parsers
struct Env {
std::uintptr_t exe_base = 0;
void (*log_line)(const char*) = nullptr;
void* parse_int = nullptr;
void* parse_float = nullptr;
void* parse_float_scaled = nullptr;
void* parse_colour = nullptr;
void* parse_vec3 = nullptr;
void* parse_rect = nullptr;
void* parse_string = nullptr;
const double* scale = nullptr;
ReadFileFn read_file = nullptr;
ReleaseFn release = nullptr;
};
Env g_env;
void logf(const char* fmt, ...) {
if (!g_env.log_line) return;
char line[1024];
va_list ap;
va_start(ap, fmt);
std::vsnprintf(line, sizeof line, fmt, ap);
va_end(ap);
g_env.log_line(line);
}
double scale_constant() {
double s = 0;
if (g_env.scale) std::memcpy(&s, g_env.scale, sizeof s);
return s;
}
Kind classify(const void* parser) {
if (parser == g_env.parse_int) return Kind::Int;
if (parser == g_env.parse_float) return Kind::Float;
if (parser == g_env.parse_float_scaled) return Kind::FloatScaled;
if (parser == g_env.parse_colour) return Kind::Colour;
if (parser == g_env.parse_vec3) return Kind::Vec3;
if (parser == g_env.parse_rect) return Kind::Rect;
if (parser == g_env.parse_string) return Kind::String;
return Kind::Unknown;
}
// In-order walk of the map: one Slot per registered constant (map order = stricmp order).
std::vector<Slot> slots_from_map(const void* consts) {
std::vector<Slot> out;
const MapHeader* m = static_cast<const MapHeader*>(consts);
if (!m || !m->head) return out;
const MapNode* head = m->head;
const std::size_t cap = static_cast<std::size_t>(m->size) + 1; // loop guard
auto is_nil = [head](const MapNode* n) { return !n || n == head || n->isnil; };
const MapNode* n = head->left;
while (!is_nil(n) && out.size() < cap) {
if (n->value) {
Slot s;
s.key = n->value->key ? n->value->key : (n->key ? n->key : "");
s.kind = classify(n->value->parser);
s.storage = n->value->storage;
s.native = n->value->parser;
if (s.kind == Kind::Unknown)
logf("cfg: %s has a parser this build does not model (%p)", s.key.c_str(), n->value->parser);
out.push_back(std::move(s));
}
if (!is_nil(n->right)) {
n = n->right;
while (!is_nil(n->left)) n = n->left;
} else {
const MapNode* p = n->parent;
while (!is_nil(p) && n == p->right) {
n = p;
p = p->parent;
}
n = p;
}
}
return out;
}
// ---- describers: the region bytes as the typed value the game reads ------------------------
Tv typed(const char* kind) {
Tv s = trace::tv::struct_();
s.add("t", trace::tv::str(kind));
return s;
}
Tv describe_int(const void* p, std::size_t, unsigned) {
std::int32_t v;
std::memcpy(&v, p, sizeof v);
Tv s = typed("int");
s.add("v", trace::tv::i32(v));
return s;
}
Tv describe_float_kind(const void* p, const char* kind) {
float v;
std::memcpy(&v, p, sizeof v);
Tv s = typed(kind);
s.add("v", trace::tv::f32(v));
return s;
}
Tv describe_float(const void* p, std::size_t, unsigned) { return describe_float_kind(p, "float"); }
Tv describe_fscaled(const void* p, std::size_t, unsigned) { return describe_float_kind(p, "fscaled"); }
Tv describe_colour(const void* p, std::size_t, unsigned) {
float c[4];
std::memcpy(c, p, sizeof c);
Tv s = typed("colour");
s.add("r", trace::tv::f32(c[0]));
s.add("g", trace::tv::f32(c[1]));
s.add("b", trace::tv::f32(c[2]));
s.add("a", trace::tv::f32(c[3]));
return s;
}
Tv describe_vec3(const void* p, std::size_t, unsigned) {
float v[3];
std::memcpy(v, p, sizeof v);
Tv s = typed("vec3");
s.add("x", trace::tv::f32(v[0]));
s.add("y", trace::tv::f32(v[1]));
s.add("z", trace::tv::f32(v[2]));
return s;
}
Tv describe_rect(const void* p, std::size_t, unsigned) {
std::int32_t v[4];
std::memcpy(v, p, sizeof v);
Tv s = typed("rect");
s.add("x", trace::tv::i32(v[0]));
s.add("y", trace::tv::i32(v[1]));
s.add("w", trace::tv::i32(v[2]));
s.add("h", trace::tv::i32(v[3]));
return s;
}
// The region is a copy of the string object; a long string's text still lives in the heap
// block the copied pointer names (same process), so the value is the text either way.
Tv describe_string(const void* p, std::size_t, unsigned) {
MsvcString s;
std::memcpy(&s, p, sizeof s);
const char* text = s.capacity > 15 ? s.u.ptr : s.u.buf;
std::size_t n = s.size;
if (!text || n > 4096) { // never trust a corrupt object
Tv bad = typed("string");
bad.add("v", trace::tv::null());
bad.add("size", trace::tv::u32(s.size));
bad.add("capacity", trace::tv::u32(s.capacity));
return bad;
}
if (s.capacity <= 15 && n > 15) n = 15;
Tv out = typed("string");
out.add("v", trace::tv::str(text, n));
return out;
}
trace::Region region_for(const Slot& s) {
trace::Region r;
r.name = s.key.c_str();
r.ptr = s.storage;
r.size = game::config::width(s.kind);
switch (s.kind) {
case Kind::Int: r.describe = &describe_int; break;
case Kind::Float: r.describe = &describe_float; break;
case Kind::FloatScaled: r.describe = &describe_fscaled; break;
case Kind::Colour: r.describe = &describe_colour; break;
case Kind::Vec3: r.describe = &describe_vec3; break;
case Kind::Rect: r.describe = &describe_rect; break;
case Kind::String: r.describe = &describe_string; break;
case Kind::Unknown: r.describe = nullptr; break; // raw bytes
}
return r;
}
// String slots: hand the text to the game's own parser (std::string assignment inside the
// engine's runtime). On a scratch copy that copy starts as the image default (an empty
// short string), so the assignment never frees anything that belongs to the live object.
void write_string_via_game(Slot& slot, std::string_view value) {
if (!slot.native || !slot.storage) return;
const std::string text(value);
reinterpret_cast<ParseFn>(slot.native)(slot.storage, text.c_str());
}
// Per-call state. LoadFile is only ever called from LoadAll on the start-up thread, one
// call at a time, so a single pending table is enough: regions() fills it before the
// original runs, rebind() re-aims it at the scratch copies, ours() consumes it.
std::vector<Slot> g_pending; // slots in region order (live storage)
std::vector<Slot> g_bound; // the same slots aimed at scratch memory
bool g_have_bound = false;
} // namespace
void init_global_consts(std::uintptr_t exe_base, void (*log_line)(const char* line)) {
g_env.exe_base = exe_base;
g_env.log_line = log_line;
auto at = [exe_base](std::uint32_t rva) { return reinterpret_cast<void*>(exe_base + rva); };
g_env.parse_int = at(sots::addr::GlobalConst_ParseInt);
g_env.parse_float = at(sots::addr::GlobalConst_ParseFloat);
g_env.parse_float_scaled = at(sots::addr::GlobalConst_ParseFloatScaled);
g_env.parse_colour = at(sots::addr::GlobalConst_ParseColour);
g_env.parse_vec3 = at(sots::addr::GlobalConst_ParseVec3);
g_env.parse_rect = at(sots::addr::GlobalConst_ParseRect);
g_env.parse_string = at(sots::addr::GlobalConst_ParseString);
g_env.scale = static_cast<const double*>(at(sots::addr::g_GlobalConstFloatScale));
g_env.read_file = reinterpret_cast<ReadFileFn>(at(sots::addr::gobio_ReadFile));
g_env.release = reinterpret_cast<ReleaseFn>(at(sots::addr::RefCounted_Release));
logf("cfg: GlobalConsts hook ready (scale constant %.17g)", scale_constant());
}
void GlobalConstsLoadFileHook::describe_args(std::vector<Tv>& out, const char* file, void* consts) {
out.push_back(trace::tv::str(file).named("file"));
out.push_back(trace::tv::ptr(consts).named("consts"));
const MapHeader* m = static_cast<const MapHeader*>(consts);
out.push_back(trace::tv::u32(m ? m->size : 0u).named("nkeys"));
// The scaled-float multiplier, so an offline replay of this log needs nothing from the exe.
out.push_back(trace::tv::f64(scale_constant()).named("scale"));
}
void GlobalConstsLoadFileHook::regions(std::vector<trace::Region>& out, const char*, void* consts) {
g_pending = slots_from_map(consts);
g_have_bound = false;
for (const Slot& s : g_pending) out.push_back(region_for(s));
}
GlobalConstsLoadFileHook::Args GlobalConstsLoadFileHook::rebind(trace::Scratch& s, const char* file,
void* consts) {
g_bound.clear();
for (std::size_t i = 0; i < g_pending.size() && i < s.count(); ++i) {
Slot b = g_pending[i];
b.storage = s.ptr(i);
b.consumed = false;
g_bound.push_back(std::move(b));
}
g_have_bound = true;
return Args(file, consts);
}
void GlobalConstsLoadFileHook::ours(const char* file, void* consts) {
std::vector<Slot> slots;
if (g_have_bound) {
slots = std::move(g_bound);
g_bound.clear();
g_have_bound = false;
} else {
slots = slots_from_map(consts); // replace mode: the live storage words
}
if (!g_env.read_file) {
logf("cfg: [%s] hook not initialised", file ? file : "");
return;
}
IBuffer* buf = nullptr;
if (!g_env.read_file(file, &buf) || !buf) {
logf("cfg: GlobalConsts: Could not open %s.", file ? file : "");
return;
}
const std::string_view text(buf->data ? buf->data : "", buf->data ? buf->size : 0u);
const std::string fname = file ? file : "";
const game::config::Stats st = game::config::apply(
fname, text, slots, scale_constant(), [](const std::string& line) { logf("cfg: %s", line.c_str()); },
&write_string_via_game);
logf("cfg: [%s] %u keys: applied %u, unknown %u, duplicate %u, missing %u", fname.c_str(),
static_cast<unsigned>(slots.size()), st.applied, st.unknown, st.duplicate, st.missing);
if (g_env.release) g_env.release(buf);
}
} // namespace shim::hooks

43
src/shim/hooks/global_consts.h Normal file
View file

@ -0,0 +1,43 @@
// Hook descriptor for Mars::GlobalConsts::LoadFile(const char* file, GlobalConstMap* consts)
// -- the file-level read of the flat KEY/value constant tables (M1).
//
// Side-effect model: the map handed to LoadFile lists every constant registered for that
// file (key -> {storage word, parser, file}). The hook walks the map before the original
// runs and declares one region per constant, named by its key, as wide as its parser's
// type (int / float / scaled float = 4 bytes, colour = float[4] = 16 bytes), with a struct
// describer so a diff names the field ("v", or "r"/"g"/"b"/"a"). In compare mode `ours`
// (game::config::apply over the same file bytes, read through the game's own gobio) fills
// the scratch copies of those words; the tracer diffs them against the original's.
//
// The map itself is a temporary of LoadAll (the original erases consumed keys from it); it
// is not declared as a region and `ours` never touches it.
//
// LoadFile is cdecl with a verified prototype, so the Hook<> template applies directly.
#pragma once
#include <cstdint>
#include <tuple>
#include <vector>
#include "shim/trace/hook.h"
namespace shim::hooks {
struct GlobalConstsLoadFileHook {
static constexpr const char* name = "Mars::GlobalConsts::LoadFile";
static constexpr trace::CallConv conv = trace::CallConv::Cdecl;
using Ret = void;
using Args = std::tuple<const char*, void*>;
static void describe_args(std::vector<trace::Tv>& out, const char* file, void* consts);
static void regions(std::vector<trace::Region>& out, const char* file, void* consts);
static Args rebind(trace::Scratch& s, const char* file, void* consts);
static void ours(const char* file, void* consts);
static trace::HookPolicy policy() { return trace::HookPolicy{}; }
};
// Process facts the hook needs: the exe's load address (parser identification, the scale
// constant, gobio::ReadFile) and a line logger (shim.log). Call once before installing.
void init_global_consts(std::uintptr_t exe_base, void (*log_line)(const char* line));
} // namespace shim::hooks

36
src/shim/main.cpp
View file

@ -15,6 +15,7 @@
#include "MinHook.h" #include "MinHook.h"
#include "generated/sots_addresses.h" #include "generated/sots_addresses.h"
#include "shim/hooks/global_consts.h"
#include "shim/trace/hook.h" #include "shim/trace/hook.h"
#include "shim/trace/selftest.h" #include "shim/trace/selftest.h"
#include "shim/trace/tracer.h" #include "shim/trace/tracer.h"
@ -118,9 +119,35 @@ _InitializeDetour:
jmp *_g_origInitialize jmp *_g_origInitialize
)"); )");
// Line logger handed to the template hooks (they live outside this TU's anonymous namespace).
void ShimLogLine(const char* line) { Log("%s", line); }
namespace { namespace {
void InstallHooks() { // ---- template hooks (verified cdecl prototypes; docs/shim-trace.md "Declaring a hook") ----
template <class D>
void InstallTemplateHook(shim::trace::Tracer& tracer, uintptr_t exeBase, uint32_t rva) {
using H = shim::trace::Hook<D>;
H::configure(tracer);
void* target = reinterpret_cast<void*>(exeBase + rva);
if (H::mode == shim::trace::Mode::Off) {
Log("hook: %s rva=0x%08x mode=off (not installed)", D::name, rva);
return;
}
MH_STATUS st = MH_CreateHook(target, reinterpret_cast<void*>(H::detour()),
reinterpret_cast<void**>(&H::original));
Log("hook: %s rva=0x%08x -> va=%p MH_CreateHook -> %s (trampoline=%p)", D::name, rva, target,
MH_StatusToString(st), reinterpret_cast<void*>(H::original));
if (st != MH_OK) return;
st = MH_EnableHook(target);
Log("hook: %s MH_EnableHook -> %s mode=%s", D::name, MH_StatusToString(st),
shim::trace::mode_name(H::mode));
}
using LoadFileHook = shim::trace::Hook<shim::hooks::GlobalConstsLoadFileHook>;
void InstallHooks(shim::trace::Tracer& tracer) {
const uintptr_t exeBase = reinterpret_cast<uintptr_t>(GetModuleHandleA(nullptr)); const uintptr_t exeBase = reinterpret_cast<uintptr_t>(GetModuleHandleA(nullptr));
void* target = reinterpret_cast<void*>(exeBase + sots::addr::Mars_Application_Initialize); void* target = reinterpret_cast<void*>(exeBase + sots::addr::Mars_Application_Initialize);
Log("hook: Mars_Application_Initialize rva=0x%08x -> va=%p", sots::addr::Mars_Application_Initialize, Log("hook: Mars_Application_Initialize rva=0x%08x -> va=%p", sots::addr::Mars_Application_Initialize,
@ -137,6 +164,10 @@ void InstallHooks() {
st = MH_EnableHook(target); st = MH_EnableHook(target);
Log("hook: MH_EnableHook -> %s", MH_StatusToString(st)); Log("hook: MH_EnableHook -> %s", MH_StatusToString(st));
// M1: GlobalConsts::LoadFile (cdecl, verified) through the template.
shim::hooks::init_global_consts(exeBase, &ShimLogLine);
InstallTemplateHook<shim::hooks::GlobalConstsLoadFileHook>(tracer, exeBase, sots::addr::GlobalConsts_LoadFile);
} }
// ---- lifecycle ----------------------------------------------------------------------------- // ---- lifecycle -----------------------------------------------------------------------------
@ -174,6 +205,7 @@ void Shim_Init(HMODULE self) {
shim::trace::Tracer& tracer = shim::trace::Tracer::instance(); shim::trace::Tracer& tracer = shim::trace::Tracer::instance();
tracer.configure(cfg.trace); tracer.configure(cfg.trace);
shim::trace::Hook<shim::selftest::FillHook>::register_policy(tracer); shim::trace::Hook<shim::selftest::FillHook>::register_policy(tracer);
LoadFileHook::register_policy(tracer);
char exeSha[65] = {}; char exeSha[65] = {};
if (!shim::trace::sha256_file(exePath, exeSha)) Log("trace: could not hash %s", exePath); if (!shim::trace::sha256_file(exePath, exeSha)) Log("trace: could not hash %s", exePath);
if (tracer.open(SHIM_BUILD_ID, exeSha)) { if (tracer.open(SHIM_BUILD_ID, exeSha)) {
@ -184,7 +216,7 @@ void Shim_Init(HMODULE self) {
Log("trace: cannot open %s; template hooks forced off", cfg.trace.path.c_str()); Log("trace: cannot open %s; template hooks forced off", cfg.trace.path.c_str());
} }
InstallHooks(); InstallHooks(tracer);
// Self-test through the hook template (no MinHook involved): one record per launch proves // Self-test through the hook template (no MinHook involved): one record per launch proves
// the emitter/tracer inside the game process. `hook.Shim::SelfTest::Fill=off` silences it. // the emitter/tracer inside the game process. `hook.Shim::SelfTest::Fill=off` silences it.

1
src/shim/shim.cfg
View file

@ -12,6 +12,7 @@ hooks=trace
# hook.<Name> = off | trace | compare | replace # hook.<Name> = off | trace | compare | replace
# per-hook override; <Name> is the hook's record name (the "hook" field in the log) # per-hook override; <Name> is the hook's record name (the "hook" field in the log)
#hook.Shim::SelfTest::Fill=off #hook.Shim::SelfTest::Fill=off
#hook.Mars::GlobalConsts::LoadFile=compare # M1: the flat KEY/value constants loader (docs/M1.md)
# trace.path = <file> default: shim.trace.jsonl next to the DLL (overwritten each run) # trace.path = <file> default: shim.trace.jsonl next to the DLL (overwritten each run)
# trace.inline_max = <bytes> regions up to this size are logged as hex (default 256) # trace.inline_max = <bytes> regions up to this size are logged as hex (default 256)

16
tests/game_config/CMakeLists.txt Normal file
View file

@ -0,0 +1,16 @@
# game/config tests: hand-written samples for the constant loader's typing, first-wins,
# colour, tokenizer and block rules, plus an offline replay of a golden LoadFile trace
# (skips unless SOTS_DATA_DIR and a trace path are given).
add_executable(game_config_unit_tests unit_tests.cpp)
target_link_libraries(game_config_unit_tests PRIVATE sots_game_config)
target_compile_options(game_config_unit_tests PRIVATE -Wall -Wextra -Werror)
add_test(NAME game_config_unit COMMAND game_config_unit_tests)
add_executable(game_config_replay replay_trace.cpp)
target_link_libraries(game_config_replay PRIVATE sots_game_config shim_trace)
target_include_directories(game_config_replay PRIVATE ${CMAKE_SOURCE_DIR}/tests/game_sim ${CMAKE_SOURCE_DIR}/tests/shim_trace)
set(SOTS_TRACECMP_DIR "$ENV{HOME}/sots-re/verify/harness/compare" CACHE PATH
"Directory holding tracecmp.py (python oracle for the trace tests)")
target_compile_definitions(game_config_replay PRIVATE SOTS_TRACECMP_DIR="${SOTS_TRACECMP_DIR}")
target_compile_options(game_config_replay PRIVATE -Wall -Wextra -Werror)
add_test(NAME game_config_replay COMMAND game_config_replay) # SKIPs without SOTS_DATA_DIR / SOTS_M1_TRACE

278
tests/game_config/replay_trace.cpp Normal file
View file

@ -0,0 +1,278 @@
// Offline replay of a golden Mars::GlobalConsts::LoadFile trace through game::config.
//
// For every LoadFile record in $SOTS_M1_TRACE the test rebuilds the registered slots from
// the record's `side.<KEY>.before` values (the region model: one struct per key, typed by
// its "t" field), reads the file named in args from $SOTS_DATA_DIR (case-insensitively --
// the game's VFS is), runs apply() with the scale constant carried in args, and emits the
// results as replay input (TRACE_FORMAT.md section 8). tracecmp.py --replay then diffs them
// against the original's `after` values. SKIPs cleanly when either variable is unset.
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include "game/config/config_loader.h"
#include "harness.h"
#include "mini_json.h"
#include "shim/trace/emitter.h"
namespace fs = std::filesystem;
using namespace game::config;
using shim::trace::Tv;
static const char* kHook = "Mars::GlobalConsts::LoadFile";
static std::string getenv_str(const char* k) {
const char* v = std::getenv(k);
return v ? v : "";
}
// Resolve `rel` (forward slashes, any case) under `root`, one component at a time.
static bool resolve_ci(const fs::path& root, const std::string& rel, fs::path& out) {
fs::path cur = root;
std::stringstream ss(rel);
std::string part;
while (std::getline(ss, part, '/')) {
if (part.empty()) continue;
bool found = false;
std::error_code ec;
for (const fs::directory_entry& e : fs::directory_iterator(cur, ec)) {
const std::string name = e.path().filename().string();
if (name.size() == part.size() && std::equal(name.begin(), name.end(), part.begin(), [](char a, char b) {
return std::tolower(static_cast<unsigned char>(a)) == std::tolower(static_cast<unsigned char>(b));
})) {
cur = e.path();
found = true;
break;
}
}
if (!found) return false;
}
out = cur;
return true;
}
static bool read_file(const fs::path& p, std::string& out) {
std::ifstream f(p, std::ios::binary);
if (!f) return false;
std::ostringstream ss;
ss << f.rdbuf();
out = ss.str();
return true;
}
// ---- the region model, host side (mirrors src/shim/hooks/global_consts.cpp) ---------------
static Kind kind_of(const std::string& t) {
if (t == "int") return Kind::Int;
if (t == "float") return Kind::Float;
if (t == "fscaled") return Kind::FloatScaled;
if (t == "colour") return Kind::Colour;
if (t == "vec3") return Kind::Vec3;
if (t == "rect") return Kind::Rect;
if (t == "string") return Kind::String;
return Kind::Unknown;
}
static float fld(const minijson::Value* v, const char* name);
static std::int32_t ild(const minijson::Value* v, const char* name);
static float fval(const minijson::Value* v) {
if (!v) return 0;
if (v->kind == minijson::Value::String) {
if (v->str == "nan") return std::nanf("");
if (v->str == "inf") return HUGE_VALF;
if (v->str == "-inf") return -HUGE_VALF;
}
return static_cast<float>(v->number());
}
// A struct tv {t:{..,v:"int"}, v:{..}} -> slot kind + storage bytes.
static bool slot_from_tv(const std::string& key, const minijson::Value& tv, Slot& slot, std::vector<std::uint8_t>& store) {
const minijson::Value* t = tv.get("t");
if (!t || t->string() != "struct") return false;
const minijson::Value* v = tv.get("v");
if (!v || !v->isObject()) return false;
const minijson::Value* kind = v->get("t");
if (!kind) return false;
slot.key = key;
slot.kind = kind_of(kind->get("v") ? kind->get("v")->string() : "");
store.assign(width(slot.kind), 0);
if (slot.kind == Kind::String) {
// host side: the slot's storage is a std::string owned by the test (see main)
return true;
}
if (slot.kind == Kind::Vec3) {
float c[3] = {fld(v, "x"), fld(v, "y"), fld(v, "z")};
std::memcpy(store.data(), c, sizeof c);
} else if (slot.kind == Kind::Rect) {
std::int32_t c[4] = {ild(v, "x"), ild(v, "y"), ild(v, "w"), ild(v, "h")};
std::memcpy(store.data(), c, sizeof c);
} else if (slot.kind == Kind::Colour) {
float c[4] = {fval(v->get("r") ? v->get("r")->get("v") : nullptr), fval(v->get("g") ? v->get("g")->get("v") : nullptr),
fval(v->get("b") ? v->get("b")->get("v") : nullptr), fval(v->get("a") ? v->get("a")->get("v") : nullptr)};
std::memcpy(store.data(), c, sizeof c);
} else if (slot.kind == Kind::Int) {
const std::int32_t i = static_cast<std::int32_t>(v->get("v") && v->get("v")->get("v") ? v->get("v")->get("v")->number() : 0);
std::memcpy(store.data(), &i, sizeof i);
} else {
const float f = fval(v->get("v") ? v->get("v")->get("v") : nullptr);
std::memcpy(store.data(), &f, sizeof f);
}
slot.storage = store.data();
return true;
}
static float fld(const minijson::Value* v, const char* name) {
const minijson::Value* f = v ? v->get(name) : nullptr;
return fval(f ? f->get("v") : nullptr);
}
static std::int32_t ild(const minijson::Value* v, const char* name) {
const minijson::Value* f = v ? v->get(name) : nullptr;
return static_cast<std::int32_t>(f && f->get("v") ? f->get("v")->number() : 0);
}
static Tv describe(const Slot& s) {
Tv out = shim::trace::tv::struct_();
out.add("t", shim::trace::tv::str(kind_name(s.kind)));
if (s.kind == Kind::String) {
const std::string* str = static_cast<const std::string*>(s.storage);
out.add("v", shim::trace::tv::str(str->data(), str->size()));
} else if (s.kind == Kind::Vec3) {
float c[3];
std::memcpy(c, s.storage, sizeof c);
out.add("x", shim::trace::tv::f32(c[0]));
out.add("y", shim::trace::tv::f32(c[1]));
out.add("z", shim::trace::tv::f32(c[2]));
} else if (s.kind == Kind::Rect) {
std::int32_t c[4];
std::memcpy(c, s.storage, sizeof c);
out.add("x", shim::trace::tv::i32(c[0]));
out.add("y", shim::trace::tv::i32(c[1]));
out.add("w", shim::trace::tv::i32(c[2]));
out.add("h", shim::trace::tv::i32(c[3]));
} else if (s.kind == Kind::Colour) {
float c[4];
std::memcpy(c, s.storage, sizeof c);
out.add("r", shim::trace::tv::f32(c[0]));
out.add("g", shim::trace::tv::f32(c[1]));
out.add("b", shim::trace::tv::f32(c[2]));
out.add("a", shim::trace::tv::f32(c[3]));
} else if (s.kind == Kind::Int) {
std::int32_t i;
std::memcpy(&i, s.storage, sizeof i);
out.add("v", shim::trace::tv::i32(i));
} else {
float f;
std::memcpy(&f, s.storage, sizeof f);
out.add("v", shim::trace::tv::f32(f));
}
return out;
}
int main(int argc, char** argv) {
const std::string trace = getenv_str("SOTS_M1_TRACE");
const std::string data = getenv_str("SOTS_DATA_DIR");
if (trace.empty() || data.empty()) {
std::printf("SKIP: set SOTS_M1_TRACE (golden LoadFile log) and SOTS_DATA_DIR (game data root)\n");
return 0;
}
const std::string out_path = argc > 1 ? argv[1] : "m1-replay-impl.jsonl";
std::ifstream in(trace);
if (!in) {
std::printf("FAIL: cannot open %s\n", trace.c_str());
return 1;
}
std::FILE* out = std::fopen(out_path.c_str(), "wb");
if (!out) {
std::printf("FAIL: cannot write %s\n", out_path.c_str());
return 1;
}
unsigned calls = 0, files_missing = 0;
std::string line;
while (std::getline(in, line)) {
minijson::Value rec;
if (!minijson::Parser(line).parse(rec)) continue;
const minijson::Value* hook = rec.get("hook");
if (!hook || hook->string() != kHook) continue;
const minijson::Value* args = rec.get("args");
const minijson::Value* side = rec.get("side");
if (!args || !args->isArray() || args->arr.empty() || !side || !side->isObject()) continue;
std::string file;
double scale = 0;
for (const minijson::Value& a : args->arr) {
const minijson::Value* n = a.get("n");
const std::string name = n ? n->string() : "";
if (name == "file" && a.get("v")) file = a.get("v")->string();
if (name == "scale" && a.get("v")) scale = a.get("v")->number();
}
// Slots from the `before` snapshots, in the record's (map) order.
std::vector<Slot> slots;
std::vector<std::vector<std::uint8_t>> stores;
std::vector<std::string> strings; // String slots' storage on the host
slots.reserve(side->obj.size());
stores.reserve(side->obj.size());
strings.reserve(side->obj.size());
for (const auto& kv : side->obj) {
const minijson::Value* before = kv.second.get("before");
Slot s;
stores.emplace_back();
if (!before || !slot_from_tv(kv.first, *before, s, stores.back())) {
std::printf("FAIL: call %g region %s is not a typed struct\n", rec.get("call_id")->number(), kv.first.c_str());
return 1;
}
if (s.kind == Kind::String) {
const minijson::Value* bv = before->get("v");
const minijson::Value* txt = bv ? bv->get("v") : nullptr;
strings.push_back(txt && txt->get("v") ? txt->get("v")->string() : "");
s.storage = &strings.back();
}
slots.push_back(std::move(s));
}
const ExternWriter ext = [](Slot& sl, std::string_view v) { *static_cast<std::string*>(sl.storage) = std::string(v); };
shim::trace::Record r;
r.hook = kHook;
r.call_id = static_cast<std::uint32_t>(rec.get("call_id")->number());
r.mode = shim::trace::Mode::Trace;
fs::path path;
std::string text;
if (resolve_ci(data, file, path) && read_file(path, text)) {
apply(file, text, slots, scale, nullptr, ext);
} else {
++files_missing;
std::printf(" note: %s not under %s (leaving defaults, as the original does)\n", file.c_str(), data.c_str());
}
for (const Slot& s : slots) {
shim::trace::SideEntry e;
e.name = s.key;
e.before_kind = shim::trace::SideEntry::Absent;
e.after = describe(s);
r.side.push_back(std::move(e));
}
shim::trace::Buf b;
shim::trace::emit_record(b, r);
std::fwrite(b.data(), 1, b.size(), out);
++calls;
}
std::fclose(out);
std::printf("replayed %u LoadFile calls (%u files missing) -> %s\n", calls, files_missing, out_path.c_str());
if (calls == 0) {
std::printf("FAIL: no %s records in %s\n", kHook, trace.c_str());
return 1;
}
const int rc = tracetest::run_tracecmp(trace, "--hook " + std::string(kHook) + " --replay " + out_path);
if (rc == tracetest::kSkipped) return 0;
std::printf("tracecmp --replay exit %d\n", rc);
return rc == 0 ? 0 : 1;
}

258
tests/game_config/unit_tests.cpp Normal file
View file

@ -0,0 +1,258 @@
// game::config unit tests: typed scanners, colour/vec3/rect rules, the pair stream (through
// mars::text::parse_flat_kv), first-occurrence-wins and the diagnostics. All samples are
// hand-written.
#include <cmath>
#include <cstdio>
#include <cstring>
#include <string>
#include <vector>
#include "game/config/config_loader.h"
using namespace game::config;
static int g_fail = 0, g_pass = 0;
#define CHECK(cond) \
do { \
if (cond) ++g_pass; \
else { ++g_fail; std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); } \
} while (0)
static bool same_bits(float a, float b) { return std::memcmp(&a, &b, sizeof a) == 0; }
static const double kScale = 0.017453292519943295; // pi/180
static void test_scan_int() {
std::int32_t v = 77;
CHECK(scan_int("10", v) && v == 10);
CHECK(scan_int(" -5", v) && v == -5);
CHECK(scan_int("10.7", v) && v == 10); // %d stops at the dot
CHECK(scan_int("+3x", v) && v == 3);
CHECK(scan_int("\t\n42", v) && v == 42);
v = 77;
CHECK(!scan_int("abc", v) && v == 77); // no digits: untouched
CHECK(!scan_int("", v) && v == 77);
CHECK(!scan_int("-", v) && v == 77);
CHECK(!scan_int(".5", v) && v == 77);
CHECK(scan_int("4294967296", v) && v == 0); // wraps like the CRT accumulator
CHECK(scan_int("2147483648", v) && v == -2147483647 - 1);
std::size_t pos = 0;
CHECK(scan_int_at("1 2", pos, v) && v == 1 && pos == 1);
CHECK(scan_int_at("1 2", pos, v) && v == 2 && pos == 3);
CHECK(!scan_int_at("1 2", pos, v) && pos == 3);
}
static void test_scan_float() {
float f = 9;
CHECK(scan_float("0.5", f) && f == 0.5f);
CHECK(scan_float("1e3", f) && f == 1000.0f);
CHECK(scan_float(".5", f) && f == 0.5f);
CHECK(scan_float("5.", f) && f == 5.0f);
CHECK(scan_float("-.25", f) && f == -0.25f);
CHECK(scan_float(" 7", f) && f == 7.0f);
CHECK(scan_float("1e", f) && f == 1.0f); // dangling exponent: mantissa only
CHECK(scan_float("2E+2z", f) && f == 200.0f);
CHECK(scan_float("0.1", f) && same_bits(f, 0.1f));
CHECK(scan_float("3.14159265358979", f) && same_bits(f, 3.14159265358979f));
CHECK(scan_float("1.5abc", f) && f == 1.5f);
f = 9;
CHECK(!scan_float("abc", f) && f == 9);
CHECK(!scan_float(".", f) && f == 9);
CHECK(!scan_float("", f) && f == 9);
CHECK(!scan_float("-e5", f) && f == 9);
}
static float c255(int v) { return static_cast<float>(static_cast<double>(v) / 255.0); }
static void test_colour() {
float c[4];
parse_colour("48 29 2", c);
CHECK(same_bits(c[0], c255(48)) && same_bits(c[1], c255(29)) && same_bits(c[2], c255(2)) && c[3] == 1.0f);
parse_colour("", c);
CHECK(c[0] == 1.0f && c[1] == 1.0f && c[2] == 1.0f && c[3] == 1.0f); // all default 255
parse_colour("300 -5 0 128", c);
CHECK(c[0] == 1.0f && c[1] == 0.0f && c[2] == 0.0f && same_bits(c[3], c255(128))); // clamped
parse_colour("12 34", c);
CHECK(same_bits(c[0], c255(12)) && same_bits(c[1], c255(34)) && c[2] == 1.0f && c[3] == 1.0f);
parse_colour(" 0\t190 195 255", c);
CHECK(c[0] == 0.0f && same_bits(c[1], c255(190)) && same_bits(c[2], c255(195)) && c[3] == 1.0f);
parse_colour("1 2 x 4", c); // the scan stops at the first non-number; the rest default
CHECK(same_bits(c[0], c255(1)) && same_bits(c[1], c255(2)) && c[2] == 1.0f && c[3] == 1.0f);
float prev = -1;
bool mono = true;
for (int i = 0; i <= 255; ++i) {
parse_colour(std::to_string(i), c);
if (!(c[0] >= 0 && c[0] <= 1 && c[0] > prev)) mono = false;
prev = c[0];
}
CHECK(mono);
}
static void test_vec3_rect() {
float v[3] = {9, 9, 9};
CHECK(parse_vec3("0 0 50.0", v) == 3 && v[0] == 0 && v[1] == 0 && v[2] == 50.0f);
float w[3] = {9, 9, 9};
CHECK(parse_vec3("1.5 x", w) == 1 && w[0] == 1.5f && w[1] == 9 && w[2] == 9); // the rest untouched
CHECK(parse_vec3("", w) == 0);
std::int32_t r[4] = {9, 9, 9, 9};
CHECK(parse_rect("16 6\t122 40", r) == 4 && r[0] == 16 && r[1] == 6 && r[2] == 122 && r[3] == 40);
std::int32_t q[4] = {9, 9, 9, 9};
CHECK(parse_rect("1 2.5 3 4", q) == 2 && q[0] == 1 && q[1] == 2 && q[2] == 9); // ".5" stops %d
}
static void test_scale() {
CHECK(same_bits(scale_float(90.0f, kScale), static_cast<float>(90.0 * kScale)));
CHECK(same_bits(scale_float(3.0f, kScale), 0.052359879f));
CHECK(same_bits(scale_float(85.0f, kScale), 1.48352981f));
CHECK(same_bits(scale_float(-60.0f, kScale), -1.04719758f));
CHECK(scale_float(0.0f, kScale) == 0.0f);
CHECK(scale_float(2.0f, 0.5) == 1.0f);
}
static void test_pairs() {
auto eq = [](const std::vector<Pair>& p, std::vector<std::pair<std::string, std::string>> want) {
if (p.size() != want.size()) return false;
for (std::size_t i = 0; i < p.size(); ++i)
if (p[i].key != want[i].first || p[i].value != want[i].second) return false;
return true;
};
CHECK(eq(pairs("A 1\nB 2\n"), {{"A", "1"}, {"B", "2"}}));
CHECK(eq(pairs("A 1\r\nB 2\r\n"), {{"A", "1"}, {"B", "2"}}));
CHECK(eq(pairs("// c\nA 1 // t\n\n\t B\t\"x y\"\n"), {{"A", "1"}, {"B", "x y"}}));
// duplicates are all delivered, in order (first-wins is applied by apply())
CHECK(eq(pairs("A 1\nA 2\na 3\n"), {{"A", "1"}, {"A", "2"}, {"a", "3"}}));
// the last pair touches end-of-file: lost; a trailing blank or comment keeps it
CHECK(eq(pairs("A 1\nB 2"), {{"A", "1"}}));
CHECK(eq(pairs("A 1\nB 2 "), {{"A", "1"}, {"B", "2"}}));
CHECK(eq(pairs("A 1\nB 2 // c"), {{"A", "1"}, {"B", "2"}}));
// a key alone takes the next line's key as its value (the stream is not line-based)
CHECK(eq(pairs("A\nB 2\nC 3\n"), {{"A", "B"}, {"2", "C"}}));
// three tokens on a line: the third starts the next pair
CHECK(eq(pairs("A 1 2\nB 3\n"), {{"A", "1"}, {"2", "B"}}));
// blocks are skipped whole, nesting included; unclosed runs to the end; stray } ignored
CHECK(eq(pairs("A 1\nblk {\n X 1\n sub {\n Y 2\n }\n Z 3\n}\nB 2\n"), {{"A", "1"}, {"B", "2"}}));
CHECK(eq(pairs("A 1\nblk {\n X 1\n"), {{"A", "1"}}));
CHECK(eq(pairs("}\nA 1\n}\nB 2\n"), {{"A", "1"}, {"B", "2"}}));
CHECK(eq(pairs("vars\n{\nA 1\nB 2\n}\n"), {}));
// a glued comment after a quoted value is still a comment
CHECK(eq(pairs("COL \"0 0 0\"// \" 92 76 20\"\n"), {{"COL", "0 0 0"}}));
CHECK(eq(pairs("E \"\"\n"), {{"E", ""}}));
}
struct Table {
std::int32_t i = 7;
float f = 1.5f;
float s = 2.0f;
float c[4] = {0, 0, 0, 0};
float v[3] = {0, 0, 0};
std::int32_t r[4] = {0, 0, 0, 0};
std::string str;
std::vector<Slot> slots() {
return {Slot{"COUNT", Kind::Int, &i, nullptr, false}, Slot{"RATE", Kind::Float, &f, nullptr, false},
Slot{"ANGLE", Kind::FloatScaled, &s, nullptr, false}, Slot{"TINT", Kind::Colour, c, nullptr, false},
Slot{"OFFSET", Kind::Vec3, v, nullptr, false}, Slot{"BOX", Kind::Rect, r, nullptr, false},
Slot{"NAME", Kind::String, &str, nullptr, false}};
}
};
static const unsigned kSlots = 7;
static void test_apply() {
std::vector<std::string> log;
auto lg = [&log](const std::string& l) { log.push_back(l); };
ExternWriter ext = [](Slot& s, std::string_view v) { *static_cast<std::string*>(s.storage) = std::string(v); };
{
Table t;
std::vector<Slot> sl = t.slots();
Stats st = apply("Data/x.txt",
"count 3\nRATE 0.25\nAngle 90\nTINT \"255 0 128\"\nOFFSET \"0 0 50.0\"\nBOX \"16 6 122 40\"\nname \"spy_node\"\n",
sl, 0.5, lg, ext);
CHECK(st.applied == kSlots && st.unknown == 0 && st.duplicate == 0 && st.missing == 0);
CHECK(t.i == 3 && t.f == 0.25f && t.s == 45.0f);
CHECK(t.c[0] == 1.0f && t.c[1] == 0.0f && same_bits(t.c[2], c255(128)) && t.c[3] == 1.0f);
CHECK(t.v[2] == 50.0f && t.r[2] == 122 && t.str == "spy_node");
CHECK(log.empty());
for (const Slot& s : sl) CHECK(s.consumed);
}
{ // first occurrence wins; the repeat is logged as multiply defined
Table t;
std::vector<Slot> sl = t.slots();
log.clear();
Stats st = apply("f", "COUNT 1\nCOUNT 2\ncount 3\n", sl, 1, lg);
CHECK(t.i == 1 && st.applied == 1 && st.duplicate == 2 && st.missing == kSlots - 1);
CHECK(log.size() == 2 + kSlots - 1 && log[0] == "[f] COUNT not recognized or is multiply defined." &&
log[1] == "[f] count not recognized or is multiply defined.");
CHECK(log[2] == "[f] RATE expected but not found.");
}
{ // unknown keys are ignored and logged; absent keys keep their defaults
Table t;
std::vector<Slot> sl = t.slots();
log.clear();
Stats st = apply("f", "NOPE 5\nRATE 2\n", sl, 1, lg);
CHECK(st.unknown == 1 && st.applied == 1 && st.missing == kSlots - 1);
CHECK(t.i == 7 && t.f == 2.0f && t.s == 2.0f && t.c[0] == 0.0f);
CHECK(log[0] == "[f] NOPE not recognized or is multiply defined.");
}
{ // a value the scanner rejects leaves an int/float word alone but still counts as consumed
Table t;
std::vector<Slot> sl = t.slots();
Stats st = apply("f", "COUNT abc\nRATE x\n", sl, 1, nullptr);
CHECK(t.i == 7 && t.f == 1.5f && st.applied == 2);
CHECK(sl[0].consumed && sl[1].consumed);
}
{ // scaled float: the word is multiplied even when the scan fails
Table t;
std::vector<Slot> sl = t.slots();
apply("f", "ANGLE x\n", sl, 0.25, nullptr);
CHECK(t.s == 0.5f);
}
{ // string slots without an ExternWriter are left alone (but consumed)
Table t;
std::vector<Slot> sl = t.slots();
Stats st = apply("f", "NAME foo\n", sl, 1, nullptr);
CHECK(t.str.empty() && st.applied == 1 && sl[6].consumed);
}
{ // colour with a bare (unquoted) value: only the first token is the value, the rest
// become keys -- exactly the failure mode the data avoids by quoting colours
Table t;
std::vector<Slot> sl = t.slots();
log.clear();
Stats st = apply("f", "TINT 10 20 30\nCOUNT 1\n", sl, 1, lg);
CHECK(same_bits(t.c[0], c255(10)) && t.c[1] == 1.0f);
CHECK(st.unknown == 1 && log[0] == "[f] 20 not recognized or is multiply defined."); // "20 30" pair
CHECK(t.i == 1);
}
{ // no trailing newline: the last pair never arrives
Table t;
std::vector<Slot> sl = t.slots();
Stats st = apply("f", "COUNT 1\nRATE 0.5", sl, 1, nullptr);
CHECK(t.i == 1 && t.f == 1.5f && st.missing == kSlots - 1);
}
{ // a file that is one big block sets nothing
Table t;
std::vector<Slot> sl = t.slots();
Stats st = apply("f", "vars\n{\nCOUNT 1\nRATE 0.5\n}\n", sl, 1, nullptr);
CHECK(st.applied == 0 && st.missing == kSlots && t.i == 7);
}
{ // Unknown-kind slot: never written, consumed when present
std::int32_t w = 5;
std::vector<Slot> sl = {Slot{"W", Kind::Unknown, &w, nullptr, false}};
Stats st = apply("f", "W 9\n", sl, 1, nullptr);
CHECK(w == 5 && st.applied == 1);
}
CHECK(width(Kind::Colour) == 16 && width(Kind::Int) == 4 && width(Kind::FloatScaled) == 4);
CHECK(width(Kind::Vec3) == 12 && width(Kind::Rect) == 16 && width(Kind::String) == 24 && width(Kind::Unknown) == 4);
CHECK(std::string(kind_name(Kind::Colour)) == "colour" && std::string(kind_name(Kind::FloatScaled)) == "fscaled");
CHECK(std::string(kind_name(Kind::String)) == "string" && std::string(kind_name(Kind::Vec3)) == "vec3");
}
int main() {
test_scan_int();
test_scan_float();
test_colour();
test_vec3_rect();
test_scale();
test_pairs();
test_apply();
std::printf("game_config unit tests: %d passed, %d failed\n", g_pass, g_fail);
return g_fail ? 1 : 0;
}