diff --git a/CMakeLists.txt b/CMakeLists.txt index 60b59ca..c8deee4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,18 @@ add_subdirectory(src/mars/parse) # brace-block + .effect readers (lib mars_p add_subdirectory(src/mars/text) # flat-kv, id-manifest, csv (lib mars_text) add_subdirectory(src/game/sim) # strategic formulas, pure (lib sots_game_sim) +# ---- shim trace/compare infrastructure (host-testable; linked into binkw32) ---- +add_library(shim_trace STATIC + src/shim/trace/sha256.cpp + src/shim/trace/emitter.cpp + src/shim/trace/tracer.cpp + src/shim/trace/selftest.cpp) +target_include_directories(shim_trace PUBLIC ${CMAKE_SOURCE_DIR}/src) +target_compile_options(shim_trace PRIVATE -Wall -Wextra -Werror) +if(MINGW) + target_compile_definitions(shim_trace PUBLIC __USE_MINGW_ANSI_STDIO=1) # C99 %lld/%.17g +endif() + if(WIN32) # ---- shim: proxy binkw32.dll that the original game loads (Phase 2 frontend) ---- add_library(minhook STATIC @@ -32,7 +44,7 @@ if(WIN32) target_include_directories(minhook PUBLIC third_party/minhook/include) add_library(binkw32 SHARED src/shim/main.cpp src/shim/binkw32.def) - target_link_libraries(binkw32 PRIVATE minhook sots_addresses) + target_link_libraries(binkw32 PRIVATE minhook sots_addresses shim_trace) target_compile_definitions(binkw32 PRIVATE SHIM_BUILD_ID="${SHIM_BUILD_ID}" SOTS_ADDR_PROVENANCE="${SOTS_ADDR_PROVENANCE}") @@ -44,7 +56,7 @@ else() add_executable(addr_smoke tests/addr_smoke.cpp) target_link_libraries(addr_smoke PRIVATE sots_addresses) add_test(NAME addr_smoke COMMAND addr_smoke) - foreach(_t mars_parse mars_text game_sim) + foreach(_t mars_parse mars_text game_sim shim_trace) if(EXISTS ${CMAKE_SOURCE_DIR}/tests/${_t}/CMakeLists.txt) add_subdirectory(tests/${_t}) endif() diff --git a/docs/shim-trace.md b/docs/shim-trace.md new file mode 100644 index 0000000..c1d1f0b --- /dev/null +++ b/docs/shim-trace.md @@ -0,0 +1,193 @@ +# Shim trace / compare infrastructure (`src/shim/trace/`) + +Hooks installed by the proxy `binkw32.dll` can run in one of four **modes** and write a +JSON-Lines log that the harness in `sots-re/verify/harness/compare/` (`tracecmp.py`) validates +and diffs. The wire format is `TRACE_FORMAT.md` (v1) in that directory; the C++ emitter here is +a byte-exact port of its reference emitter (`mkfixture.py`) and is tested against it. + +| file | what | +|---|---| +| `platform.h` | mutex (CRITICAL_SECTION / pthread), monotonic µs, thread id, UTC stamp, `SHIM_CDECL`/`SHIM_STDCALL` | +| `sha256.{h,cpp}` | FIPS 180-4 SHA-256, no allocation | +| `emitter.{h,cpp}` | typed values (`Tv`), `Record`, `Meta`, `Buf`, `emit_record`/`emit_meta` (spec §3–5) | +| `tracer.{h,cpp}` | `Config` (shim.cfg keys), `Tracer` (locked writer, call ids, depth), `Region`/`Snapshot`/`Scratch`, `diff_tv`/`diff_outputs` (spec §6) | +| `hook.h` | `Hook`: the mode dispatch around a hooked C function | +| `selftest.{h,cpp}` | worked example hook over the shim's own `Fill()`; run once per launch | + +Host tests: `tests/shim_trace/` (ctest `shim_trace_*`). They build the same sources with +`-Wall -Wextra -Werror` on Linux; the shim cross-build links the same static library. + +## Modes + +| mode | what runs | what is logged | caller gets | +|---|---|---|---| +| `off` | original | nothing | original's result | +| `trace` | original | `args`, `ret`, `side` (before/after of every declared region) | original's result | +| `compare` | original, then **ours on a copy of the pre-call state** | trace fields + `ours{ret,side}`, `diverged`, `diff` | original's result | +| `replace` | ours | nothing | ours' result | + +`compare` never lets ours touch game memory: the declared regions are snapshotted *before* the +original runs, those snapshots are copied into scratch buffers, the arguments are rebound onto +the scratch buffers (`rebind`), and ours runs there. The diff compares the original's `after` +with the scratch buffers' contents and both return values. + +A throw from ours, or from any describe/regions/rebind callback, is caught inside the hook and +becomes `err` on the record (which the harness counts as a divergence). The original is always +invoked in `off`/`trace`/`compare`, whatever happened during capture. Nothing propagates across +the hook boundary. + +## Configuration (`shim.cfg`, next to the DLL) + +``` +hooks=trace # default mode for every hook; off = install nothing at all +hook.Game::CfgVar_RegisterKey=compare # per-hook override, keyed by the record's "hook" name +hook.Shim::SelfTest::Fill=off +trace.path=C:\SOTS\shim.trace.jsonl # default: shim.trace.jsonl beside the DLL; overwritten per run +trace.inline_max=256 # regions <= this many bytes are logged as hex, else sha256 + 32-byte head +trace.flush=always # always (default) | lazy (flush only on divergence / err / exit) +``` + +Unusable values are logged to `shim.log` (`config: key=value rejected (...)`) and ignored. +`hooks=off` keeps M0 behaviour: no MinHook, no trace file. If the trace file cannot be opened, +every template hook reports mode `off` (the game runs un-instrumented, never half-instrumented). + +The M0 `Mars::Application::Initialize` hook is an asm stub, not a template hook: it is installed +whenever `hooks` is not `off`, and it only logs to `shim.log`. See "thiscall" below. + +## Log + +`shim.trace.jsonl`: line 1 is the `meta` record (`build`, `exe_sha256` of the running exe, +`started`, `inline_max`, and a `hooks` policy map with one entry per registered template hook); +every later line is one call record. `call_id` is process-global and taken *before* the original +runs, so nested hooks get higher ids and `depth` says who is inside whom. `ts` is microseconds +since the tracer opened. The file is ASCII, LF-terminated; a crash mid-run leaves at most one +truncated last line, which the harness reports as one invalid record (`--skip-invalid` drops it). + +`shim.log` gets one `trace:` line at startup (path, default mode) and one `selftest:` line +(the self-test hook's mode and checksum), and `shutdown: trace records=N` on detach. + +## Declaring a hook + +A hook is a descriptor struct; `Hook` generates the detour. Everything the template needs is +static, so a descriptor is a header-only declaration plus a few small functions. + +```cpp +#include "shim/trace/hook.h" +using namespace shim::trace; + +struct CfgVarRegisterKeyHook { + static constexpr const char* name = "Game::CfgVar_RegisterKey"; // the record's "hook" + static constexpr CallConv conv = CallConv::Cdecl; // Cdecl | Stdcall + using Ret = bool; // void is allowed + using Args = std::tuple; // the parameter list + + // inputs, in declaration order (never compared by the harness; use .named("x")) + static void describe_args(std::vector& out, CfgTable* t, const char* key, int value) { + out.push_back(tv::ptr(t).named("table")); + out.push_back(tv::str(key).named("key")); // raw cp1252 bytes; nullptr -> ptr 0x0 + out.push_back(tv::i32(value).named("value")); + } + static Tv describe_ret(bool r) { return tv::boolean(r); } // omit when Ret is void + + // declared side-effect regions: name, address, size, optional structured describer + static void regions(std::vector& out, CfgTable* t, const char*, int) { + out.push_back(Region{"cfg_table", t, sizeof(CfgTable), &describe_table}); + } + static Tv describe_table(const void* p, std::size_t, unsigned /*inline_max*/) { + const CfgTable* t = static_cast(p); // a *copy* of the region + Tv s = tv::struct_(); + s.add("count", tv::u32(t->count)); + return s; // diffs then name the field + } + + // the same call, aimed at the scratch copies (regions in declaration order) + static Args rebind(Scratch& s, CfgTable*, const char* key, int value) { + return Args(s.as(0), key, value); + } + + static bool ours(CfgTable* t, const char* key, int value); // the reimplementation + static HookPolicy policy() { return HookPolicy{}; } // ftol / ftol_kind / ptr_exact / unordered +}; +``` + +Install (in `InstallHooks`, after `Tracer::open`): + +```cpp +using H = Hook; +H::register_policy(tracer); // BEFORE tracer.open(): lands in meta.hooks +... +H::configure(tracer); // reads hook. / hooks from the config +MH_CreateHook(target, reinterpret_cast(H::detour()), reinterpret_cast(&H::original)); +MH_EnableHook(target); +``` + +Rules of the road: + +- **Regions are copies.** `describe` callbacks receive the snapshot buffer, not live memory; a + region that contains pointers into other memory is only compared by the pointer values + (ignored by default policy) unless you declare the pointed-to memory as another region. +- **Region sizes must be known at call time.** A region whose length is only known after the + call cannot be snapshotted "before"; declare an upper bound or split the hook. +- **Prefer `struct` describers** over raw `bytes` for anything with fields: the harness then + points at `side.cfg_table.after.v.count` instead of a byte offset. +- **Floats**: `tv::f32` stores at float32 width and prints `%.9g`; the diff rounds both sides + to float32 first (spec §6). Never describe a `float` as `f64`. +- **Big ints**: `i64`/`u64` at or beyond 2^53 in magnitude are emitted as decimal strings. +- **`json` values** are your own canonical text (spec §7); the shim compares them as text. +- **Policies** only affect the shim's advisory `diff`/`diverged`; `tracecmp.py` recomputes from + the meta policy plus CLI flags and warns when the shim's verdict disagrees. +- **Re-entrancy**: a hook calling into another hooked function is fine (depth, ids). A hook + calling *itself* through ours in compare mode would recurse into the tracer; do not do that. +- **Cost**: trace/compare allocate (Tv trees, snapshots). Fine for M1-scale hooks; do not put + a per-frame hot path in `trace` without thinking about `trace.flush=lazy`. + +### thiscall / unverified prototypes + +The template's detour is a real C++ function with a fixed prototype. That is only safe when the +prototype is known. M0 proved that a C++ wrapper around an `[unverified]` `thiscall` corrupts +the game (`docs/M0.md`, gotcha 1). So: + +- `__thiscall` hooks and any hook whose entry in `sots_addresses.h` is `[unverified]` stay + **asm stubs**: `pushfl/pushal`, call a C logger with ECX, `popal/popfl`, `jmp` trampoline. + They are trace-only and cannot capture the return value. Their logger may still build a + `Record` and hand it to `Tracer::write` (mode `trace`, `ret` null, regions snapshotted by hand). +- Once a prototype is verified, `thiscall` can be adapted to the template by declaring the + detour as a `Stdcall` hook that takes `this` as its first explicit parameter behind a tiny asm + thunk (`push ecx` then jump) — not provided here; do it only with a verified prototype. +- `cdecl`/`stdcall` with verified prototypes use the template directly (`CallConv::Cdecl`, + `CallConv::Stdcall`). + +## Running a report + +``` +# on the game VM: play; then copy C:\SOTS\shim.trace.jsonl to verify/traces/.jsonl +python3 verify/harness/compare/tracecmp.py verify/traces/m1-cfgvar-20260907.jsonl \ + --json-out verify/results/compare/m1-cfgvar-20260907.json +# exit 0 clean, 1 divergences, 2 invalid record(s) (--skip-invalid tolerates a truncated tail) +# --hook NAME one hook only +# --tolerance NAME=abs:1e-6 / rel:1e-5 / ulp:2 float policy (overrides meta.hooks) +# --ptr exact compare pointer values too +# --replay IMPL.jsonl offline: a golden trace vs a host implementation's {call_id, ret, side} lines +``` + +## Host tests + +``` +cmake -S . -B build-host -G Ninja -DCMAKE_BUILD_TYPE=Debug # or: cmake --preset host +cmake --build build-host && ctest --test-dir build-host +``` + +- `shim_trace_sha256` — FIPS vectors, chunked updates, padding edges. +- `shim_trace_emitter` — golden strings for §4 escaping, number forms, bytes inline/head, records, + meta; writes `out/emitter_oracle.jsonl`, then `tests/shim_trace/oracle_emit.py` rebuilds the + same records with `mkfixture.emit_record` and compares byte for byte, and `tracecmp.py` must + read it back (exit 1: it carries one injected divergence). +- `shim_trace_diff` — every §6 rule (type, ptr, nan/inf, f32 rounding, abs/rel/ulp, bytes + offset, set vs list, unordered policy, struct missing/extra, side names), snapshots, config. +- `shim_trace_hook` — the self-test hook through every mode: clean log → `tracecmp` exit 0, + wrong/throwing ours → exit 1 (and exit 0 with `--hook` on the clean hook), truncated file → + exit 2 / 0 with `--skip-invalid`, plus a second thread and the startup `run_once` path. + +The Python steps use `/usr/bin/python3` and the harness dir from `-DSOTS_TRACECMP_DIR` (default +`$HOME/sots-re/verify/harness/compare`) or the `SOTS_TRACECMP_DIR` environment variable; when +`tracecmp.py` is not there they print `SKIP` and the test still passes. diff --git a/src/shim/main.cpp b/src/shim/main.cpp index 5b67d8a..fe37293 100644 --- a/src/shim/main.cpp +++ b/src/shim/main.cpp @@ -11,9 +11,13 @@ #include #include #include +#include #include "MinHook.h" #include "generated/sots_addresses.h" +#include "shim/trace/hook.h" +#include "shim/trace/selftest.h" +#include "shim/trace/tracer.h" namespace { @@ -37,21 +41,27 @@ void Log(const char* fmt, ...) { } // ---- config (shim.cfg next to the DLL; "key=value" per line) ------------------------------ +// +// `hooks=`, `hook.=`, `trace.*` are owned by the tracer (src/shim/trace/tracer.h); +// `hooks=off` additionally means "install nothing" (the M0 asm-stub hook included). struct Config { - bool hooks = true; // hooks=trace (default) | off + bool hooks = true; // hooks != off + shim::trace::Config trace; }; Config ReadConfig() { Config cfg; char path[MAX_PATH]; + std::snprintf(path, sizeof path, "%s\\shim.trace.jsonl", g_dir); + cfg.trace.path = path; std::snprintf(path, sizeof path, "%s\\shim.cfg", g_dir); FILE* f = std::fopen(path, "r"); if (!f) { Log("config: %s not found, using defaults", path); return cfg; } - char line[256]; + char line[512]; while (std::fgets(line, sizeof line, f)) { char* p = line; while (*p == ' ' || *p == '\t') ++p; @@ -61,14 +71,16 @@ Config ReadConfig() { *eq = '\0'; char* val = eq + 1; val[std::strcspn(val, "\r\n")] = '\0'; - if (std::strcmp(p, "hooks") == 0) { - cfg.hooks = std::strcmp(val, "off") != 0; - Log("config: hooks=%s", val); + std::string err; + if (cfg.trace.apply(p, val, &err)) { + if (!err.empty()) Log("config: %s=%s rejected (%s)", p, val, err.c_str()); + else Log("config: %s=%s", p, val); } else { Log("config: ignoring unknown key '%s'", p); } } std::fclose(f); + cfg.hooks = cfg.trace.default_mode != shim::trace::Mode::Off; return cfg; } @@ -151,14 +163,46 @@ void Shim_Init(HMODULE self) { Log("addresses: %s", SOTS_ADDR_PROVENANCE); const Config cfg = ReadConfig(); - if (cfg.hooks) { - InstallHooks(); - } else { + if (!cfg.hooks) { Log("hook: disabled by config"); + return; + } + + // Trace log: register every template hook's policy (goes into the meta line), then open. + // Hooks read their mode from the tracer after open(); a tracer that failed to open reports + // Off for everything, so the game runs un-instrumented rather than half-instrumented. + shim::trace::Tracer& tracer = shim::trace::Tracer::instance(); + tracer.configure(cfg.trace); + shim::trace::Hook::register_policy(tracer); + char exeSha[65] = {}; + if (!shim::trace::sha256_file(exePath, exeSha)) Log("trace: could not hash %s", exePath); + if (tracer.open(SHIM_BUILD_ID, exeSha)) { + Log("trace: %s (default mode %s, inline_max %u, flush %s)", cfg.trace.path.c_str(), + shim::trace::mode_name(cfg.trace.default_mode), cfg.trace.inline_max, + cfg.trace.flush_always ? "always" : "lazy"); + } else { + Log("trace: cannot open %s; template hooks forced off", cfg.trace.path.c_str()); + } + + InstallHooks(); + + // Self-test through the hook template (no MinHook involved): one record per launch proves + // the emitter/tracer inside the game process. `hook.Shim::SelfTest::Fill=off` silences it. + { + using H = shim::trace::Hook; + H::configure(tracer); + const unsigned sum = shim::selftest::run_once(H::mode); + Log("selftest: %s mode=%s checksum=%08x records=%lu", shim::selftest::FillHook::name, + shim::trace::mode_name(H::mode), sum, static_cast(tracer.records_written())); } } void Shim_Shutdown() { + shim::trace::Tracer& tracer = shim::trace::Tracer::instance(); + if (tracer.is_open()) { + Log("shutdown: trace records=%lu", static_cast(tracer.records_written())); + tracer.close(); + } if (g_minhookUp) { MH_STATUS st = MH_Uninitialize(); Log("shutdown: MH_Uninitialize -> %s", MH_StatusToString(st)); diff --git a/src/shim/shim.cfg b/src/shim/shim.cfg index c6df8d1..572a2e3 100644 --- a/src/shim/shim.cfg +++ b/src/shim/shim.cfg @@ -1,7 +1,21 @@ # sots-engine shim configuration. Read once when binkw32.dll (the proxy) loads. -# One key=value per line; '#' starts a comment. +# One key=value per line; '#' starts a comment. See docs/shim-trace.md. # -# hooks = trace | off -# trace install the hooks and log them to shim.log (default) -# off forward Bink calls only; touch nothing in the exe +# hooks = off | trace | compare | replace +# default mode for every hook +# off forward Bink calls only; touch nothing in the exe (nothing is installed) +# trace run the original, log args/ret/side effects to shim.trace.jsonl +# compare run the original AND our reimplementation on the same inputs, log both + diff +# replace run our reimplementation instead of the original (nothing logged) hooks=trace + +# hook. = off | trace | compare | replace +# per-hook override; is the hook's record name (the "hook" field in the log) +#hook.Shim::SelfTest::Fill=off + +# trace.path = default: shim.trace.jsonl next to the DLL (overwritten each run) +# trace.inline_max = regions up to this size are logged as hex (default 256) +# trace.flush = always | lazy fflush every record (default) or only on divergence/err/exit +#trace.path=C:\SOTS\shim.trace.jsonl +#trace.inline_max=256 +#trace.flush=always diff --git a/src/shim/trace/emitter.cpp b/src/shim/trace/emitter.cpp new file mode 100644 index 0000000..1e9264a --- /dev/null +++ b/src/shim/trace/emitter.cpp @@ -0,0 +1,482 @@ +#include "shim/trace/emitter.h" + +#include +#include +#include +#include +#include + +#include "shim/trace/sha256.h" + +namespace shim::trace { + +// ---- modes ---------------------------------------------------------------------------------- + +const char* mode_name(Mode m) { + switch (m) { + case Mode::Off: return "off"; + case Mode::Trace: return "trace"; + case Mode::Compare: return "compare"; + case Mode::Replace: return "replace"; + } + return "off"; +} + +bool parse_mode(const char* text, Mode& out) { + if (!text) return false; + if (std::strcmp(text, "off") == 0) { out = Mode::Off; return true; } + if (std::strcmp(text, "trace") == 0) { out = Mode::Trace; return true; } + if (std::strcmp(text, "compare") == 0) { out = Mode::Compare; return true; } + if (std::strcmp(text, "replace") == 0) { out = Mode::Replace; return true; } + return false; +} + +// ---- typed values ----------------------------------------------------------------------------- + +const char* Tv::type_name() const { + switch (kind) { + case Bool: return "bool"; + case Int: return itype; + case Float: return f32 ? "f32" : "f64"; + case Str: return "str"; + case WStr: return "wstr"; + case Ptr: return "ptr"; + case Enum: return "enum"; + case Null: return "null"; + case Bytes: return "bytes"; + case List: return "list"; + case Set: return "set"; + case Struct: return "struct"; + case Json: return "json"; + } + return "null"; +} + +Tv& Tv::add(const char* key, Tv v) { + keys.emplace_back(key); + items.push_back(std::move(v)); + return *this; +} + +namespace tv { + +namespace { +Tv make_int(const char* t, long long v) { + Tv r; + r.kind = Tv::Int; + r.itype = t; + r.i = v; + return r; +} +Tv make_uint(const char* t, unsigned long long v) { + Tv r; + r.kind = Tv::Int; + r.itype = t; + r.is_unsigned = true; + r.u = v; + return r; +} +} // namespace + +Tv boolean(bool v) { Tv r; r.kind = Tv::Bool; r.b = v; return r; } +Tv i8(std::int8_t v) { return make_int("i8", v); } +Tv i16(std::int16_t v) { return make_int("i16", v); } +Tv i32(std::int32_t v) { return make_int("i32", v); } +Tv i64(std::int64_t v) { return make_int("i64", v); } +Tv u8(std::uint8_t v) { return make_uint("u8", v); } +Tv u16(std::uint16_t v) { return make_uint("u16", v); } +Tv u32(std::uint32_t v) { return make_uint("u32", v); } +Tv u64(std::uint64_t v) { return make_uint("u64", v); } +Tv f32(float v) { Tv r; r.kind = Tv::Float; r.f32 = true; r.f = v; return r; } +Tv f64(double v) { Tv r; r.kind = Tv::Float; r.f = v; return r; } + +Tv str(const char* s) { + if (!s) return ptr(static_cast(0)); + return str(s, std::strlen(s)); +} +Tv str(const void* bytes, std::size_t n) { + Tv r; + r.kind = Tv::Str; + r.s.assign(static_cast(bytes), n); + return r; +} +Tv wstr(const std::uint16_t* units, std::size_t n) { + Tv r; + r.kind = Tv::WStr; + r.w.assign(units, units + n); + return r; +} +Tv wstr(const wchar_t* s) { + if (!s) return ptr(static_cast(0)); + Tv r; + r.kind = Tv::WStr; + for (; *s; ++s) r.w.push_back(static_cast(*s)); + return r; +} +Tv ptr(const void* p) { return ptr(reinterpret_cast(p)); } +Tv ptr(std::uintptr_t p) { Tv r; r.kind = Tv::Ptr; r.p = p; return r; } +Tv enum_(long long v, const char* symbolic) { + Tv r; + r.kind = Tv::Enum; + r.i = v; + if (symbolic) r.s = symbolic; + return r; +} +Tv null() { return Tv(); } + +Tv bytes(const void* data, std::size_t n, std::size_t inline_max) { + Tv r; + r.kind = Tv::Bytes; + r.blen = n; + Sha256::digest_hex(data, n, r.sha); + const std::uint8_t* p = static_cast(data); + if (n <= inline_max) { + r.inline_full = true; + r.data.assign(p, p + n); + } else { + r.data.assign(p, p + 32); + } + return r; +} +Tv list(std::vector items) { Tv r; r.kind = Tv::List; r.items = std::move(items); return r; } +Tv set(std::vector items) { Tv r; r.kind = Tv::Set; r.items = std::move(items); return r; } +Tv struct_() { Tv r; r.kind = Tv::Struct; return r; } +Tv json(const char* canonical_text) { Tv r; r.kind = Tv::Json; r.s = canonical_text; return r; } + +} // namespace tv + +// ---- output buffer -------------------------------------------------------------------------------- + +Buf::Buf() { grow(4096); } +Buf::~Buf() { std::free(data_); } + +bool Buf::grow(std::size_t need) { + if (need <= cap_) return true; + std::size_t ncap = cap_ ? cap_ : 4096; + while (ncap < need) ncap *= 2; + char* nd = static_cast(std::realloc(data_, ncap)); + if (!nd) { + overflow_ = true; + return false; + } + data_ = nd; + cap_ = ncap; + return true; +} + +void Buf::put(char c) { + if (len_ + 1 > cap_ && !grow(len_ + 1)) return; + data_[len_++] = c; +} + +void Buf::put(const void* bytes, std::size_t n) { + if (!n) return; + if (len_ + n > cap_ && !grow(len_ + n)) return; + std::memcpy(data_ + len_, bytes, n); + len_ += n; +} + +void Buf::puts(const char* s) { put(s, std::strlen(s)); } + +void Buf::printf(const char* fmt, ...) { + char tmp[128]; + va_list ap; + va_start(ap, fmt); + const int n = std::vsnprintf(tmp, sizeof tmp, fmt, ap); + va_end(ap); + if (n < 0) { + overflow_ = true; + return; + } + if (static_cast(n) < sizeof tmp) { + put(tmp, static_cast(n)); + return; + } + if (!grow(len_ + static_cast(n) + 1)) return; + va_start(ap, fmt); + std::vsnprintf(data_ + len_, static_cast(n) + 1, fmt, ap); + va_end(ap); + len_ += static_cast(n); +} + +void Buf::clear() { + len_ = 0; + overflow_ = false; +} + +// ---- escaping / numbers ------------------------------------------------------------------------- + +namespace { + +const char* const kHex = "0123456789abcdef"; + +inline void put_u_escape(Buf& out, unsigned unit) { + char e[6] = {'\\', 'u', kHex[(unit >> 12) & 15], kHex[(unit >> 8) & 15], kHex[(unit >> 4) & 15], kHex[unit & 15]}; + out.put(e, 6); +} + +// One unit of either string flavour (bytes for str, UTF-16 units for wstr): mkfixture.esc(). +inline void put_unit(Buf& out, unsigned unit) { + switch (unit) { + case '"': out.put("\\\"", 2); return; + case '\\': out.put("\\\\", 2); return; + case '\n': out.put("\\n", 2); return; + case '\r': out.put("\\r", 2); return; + case '\t': out.put("\\t", 2); return; + default: + if (unit < 0x20 || unit >= 0x7f) put_u_escape(out, unit); + else out.put(static_cast(unit)); + } +} + +void put_hex(Buf& out, const std::uint8_t* p, std::size_t n) { + for (std::size_t i = 0; i < n; ++i) { + out.put(kHex[p[i] >> 4]); + out.put(kHex[p[i] & 15]); + } +} + +const long long kTwo53 = 1LL << 53; + +} // namespace + +void emit_esc(Buf& out, const char* bytes, std::size_t n) { + out.put('"'); + for (std::size_t i = 0; i < n; ++i) put_unit(out, static_cast(bytes[i])); + out.put('"'); +} + +void emit_esc(Buf& out, const std::string& s) { emit_esc(out, s.data(), s.size()); } + +void emit_esc_w(Buf& out, const std::uint16_t* units, std::size_t n) { + out.put('"'); + for (std::size_t i = 0; i < n; ++i) put_unit(out, units[i]); + out.put('"'); +} + +// Integers above 2^53 in magnitude go out as decimal strings (section 3): mkfixture.u64(). +void emit_int(Buf& out, long long v) { + const bool quote = v >= kTwo53 || v <= -kTwo53; + if (quote) out.put('"'); + out.printf("%lld", v); + if (quote) out.put('"'); +} + +void emit_uint(Buf& out, unsigned long long v) { + const bool quote = v >= static_cast(kTwo53); + if (quote) out.put('"'); + out.printf("%llu", v); + if (quote) out.put('"'); +} + +// mkfixture.fmt_num() for floats. +void emit_float(Buf& out, double v, bool f32) { + if (std::isnan(v)) { out.puts("\"nan\""); return; } + if (std::isinf(v)) { out.puts(v > 0 ? "\"inf\"" : "\"-inf\""); return; } + out.printf(f32 ? "%.9g" : "%.17g", v); +} + +// ---- typed values ---------------------------------------------------------------------------------- + +// mkfixture.emit_tv(): {"t":..., ("n","sha256","hex"|"head") | "v":... [,"name":...]} [,"n":...] +void emit_tv(Buf& out, const Tv& v) { + out.puts("{\"t\":"); + emit_esc(out, v.type_name(), std::strlen(v.type_name())); + if (v.kind == Tv::Bytes) { + out.printf(",\"n\":%llu,\"sha256\":\"", static_cast(v.blen)); + out.puts(v.sha); + out.puts(v.inline_full ? "\",\"hex\":\"" : "\",\"head\":\""); + put_hex(out, v.data.data(), v.data.size()); + out.puts("\"}"); + return; + } + out.puts(",\"v\":"); + switch (v.kind) { + case Tv::Bool: out.puts(v.b ? "true" : "false"); break; + case Tv::Int: + if (v.is_unsigned) emit_uint(out, v.u); + else emit_int(out, v.i); + break; + case Tv::Float: emit_float(out, v.f, v.f32); break; + case Tv::Str: emit_esc(out, v.s); break; + case Tv::WStr: emit_esc_w(out, v.w.data(), v.w.size()); break; + case Tv::Ptr: out.printf("\"0x%08llx\"", static_cast(v.p)); break; + case Tv::Enum: out.printf("%lld", v.i); break; + case Tv::Null: out.puts("null"); break; + case Tv::List: + case Tv::Set: + out.put('['); + for (std::size_t i = 0; i < v.items.size(); ++i) { + if (i) out.put(','); + emit_tv(out, v.items[i]); + } + out.put(']'); + break; + case Tv::Struct: + out.put('{'); + for (std::size_t i = 0; i < v.items.size(); ++i) { + if (i) out.put(','); + emit_esc(out, v.keys[i]); + out.put(':'); + emit_tv(out, v.items[i]); + } + out.put('}'); + break; + case Tv::Json: out.puts(v.s.c_str()); break; + case Tv::Bytes: break; // handled above + } + if (v.kind == Tv::Enum && !v.s.empty()) { + out.puts(",\"name\":"); + emit_esc(out, v.s); + } + if (!v.name.empty()) { + out.puts(",\"n\":"); + emit_esc(out, v.name); + } + out.put('}'); +} + +std::string canonical(const Tv& v) { + Buf b; + emit_tv(b, v); + return b.str(); +} + +// ---- records ---------------------------------------------------------------------------------------- + +namespace { + +void emit_side(Buf& out, const std::vector& side) { + out.put('{'); + for (std::size_t i = 0; i < side.size(); ++i) { + const SideEntry& e = side[i]; + if (i) out.put(','); + emit_esc(out, e.name); + out.puts(":{"); + if (e.before_kind == SideEntry::IsNull) { + out.puts("\"before\":null,"); + } else if (e.before_kind == SideEntry::Value) { + out.puts("\"before\":"); + emit_tv(out, e.before); + out.put(','); + } + out.puts("\"after\":"); + emit_tv(out, e.after); + out.put('}'); + } + out.put('}'); +} + +void emit_ours_side(Buf& out, const std::vector& side) { + out.put('{'); + for (std::size_t i = 0; i < side.size(); ++i) { + if (i) out.put(','); + emit_esc(out, side[i].name); + out.puts(":{\"after\":"); + emit_tv(out, side[i].after); + out.put('}'); + } + out.put('}'); +} + +void emit_opt_tv(Buf& out, const std::optional& v) { + if (v) emit_tv(out, *v); + else out.puts("null"); +} + +void emit_diff(Buf& out, const std::vector& diff) { + out.put('['); + for (std::size_t i = 0; i < diff.size(); ++i) { + const DiffEntry& d = diff[i]; + if (i) out.put(','); + out.puts("{\"path\":"); + emit_esc(out, d.path); + out.puts(",\"why\":"); + emit_esc(out, d.why, std::strlen(d.why)); + out.puts(",\"orig\":"); + if (!d.orig_raw.empty()) out.puts(d.orig_raw.c_str()); + else emit_opt_tv(out, d.orig); + out.puts(",\"ours\":"); + if (!d.ours_raw.empty()) out.puts(d.ours_raw.c_str()); + else emit_opt_tv(out, d.ours); + if (d.first_diff_offset >= 0) out.printf(",\"first_diff_offset\":%lld", d.first_diff_offset); + if (!d.note.empty()) { + out.puts(",\"note\":"); + emit_esc(out, d.note); + } + out.put('}'); + } + out.put(']'); +} + +} // namespace + +// mkfixture.emit_record(): fixed KEY_ORDER, optional keys omitted. +void emit_record(Buf& out, const Record& r) { + out.printf("{\"ts\":%lld,\"hook\":", static_cast(r.ts)); + emit_esc(out, r.hook); + out.printf(",\"mode\":\"%s\",\"call_id\":%llu,\"thread\":%llu", mode_name(r.mode), + static_cast(r.call_id), static_cast(r.thread)); + if (r.depth >= 0) out.printf(",\"depth\":%d", r.depth); + out.puts(",\"args\":["); + for (std::size_t i = 0; i < r.args.size(); ++i) { + if (i) out.put(','); + emit_tv(out, r.args[i]); + } + out.puts("],\"ret\":"); + emit_opt_tv(out, r.ret); + out.puts(",\"side\":"); + emit_side(out, r.side); + if (r.has_ours) { + out.puts(",\"ours\":{\"ret\":"); + emit_opt_tv(out, r.ours_ret); + out.puts(",\"side\":"); + emit_ours_side(out, r.ours_side); + out.put('}'); + } + if (r.diverged >= 0) out.puts(r.diverged ? ",\"diverged\":true" : ",\"diverged\":false"); + if (r.has_diff) { + out.puts(",\"diff\":"); + emit_diff(out, r.diff); + } + if (r.err) { + out.puts(",\"err\":"); + emit_esc(out, *r.err); + } + if (r.note) { + out.puts(",\"note\":"); + emit_esc(out, *r.note); + } + out.puts("}\n"); +} + +// mkfixture.emit_meta() over the dict shape mkfixture.meta() builds. +void emit_meta(Buf& out, const Meta& m) { + out.puts("{\"meta\":{\"format\":1,\"build\":"); + emit_esc(out, m.build); + out.puts(",\"exe_sha256\":"); + emit_esc(out, m.exe_sha256); + out.puts(",\"started\":"); + emit_esc(out, m.started); + out.printf(",\"inline_max\":%u,\"hooks\":{", m.inline_max); + for (std::size_t i = 0; i < m.hooks.size(); ++i) { + const HookPolicy& p = m.hooks[i].second; + if (i) out.put(','); + emit_esc(out, m.hooks[i].first); + out.puts(":{\"ftol\":"); + emit_float(out, p.ftol, false); + out.printf(",\"ftol_kind\":\"%s\",\"ptr\":\"%s\"", p.ftol_kind, p.ptr_exact ? "exact" : "ignore"); + if (!p.unordered.empty()) { + out.puts(",\"unordered\":["); + for (std::size_t j = 0; j < p.unordered.size(); ++j) { + if (j) out.put(','); + emit_esc(out, p.unordered[j]); + } + out.put(']'); + } + out.put('}'); + } + out.puts("}}}\n"); +} + +} // namespace shim::trace diff --git a/src/shim/trace/emitter.h b/src/shim/trace/emitter.h new file mode 100644 index 0000000..d5d2f7b --- /dev/null +++ b/src/shim/trace/emitter.h @@ -0,0 +1,196 @@ +// JSONL emitter for the trace/compare log (sots-re/verify/harness/compare/TRACE_FORMAT.md, v1). +// +// A faithful port of the reference emitter in mkfixture.py (esc / fmt_num / emit_tv / +// emit_record / emit_meta): fixed key order, ASCII-only escaping, %.9g / %.17g floats, +// big integers as decimal strings, bytes as n + sha256 (+ hex inline or a 32-byte head). +// +// Typed values (`Tv`) are a small tree; building one allocates (std::string / std::vector), +// so callers on a hook boundary wrap construction in try/catch (hook.h does). The output +// buffer (`Buf`) is malloc-based and never throws: on allocation failure it flags overflow +// and drops bytes, and the tracer refuses to write a record whose buffer is not ok(). +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace shim::trace { + +// ---- modes ---------------------------------------------------------------------------------- + +enum class Mode : std::uint8_t { Off, Trace, Compare, Replace }; + +const char* mode_name(Mode m); // "off" | "trace" | "compare" | "replace" +bool parse_mode(const char* text, Mode& out); // accepts the same four words + +// ---- typed values ----------------------------------------------------------------------------- + +struct Tv { + enum Kind : std::uint8_t { Bool, Int, Float, Str, WStr, Ptr, Enum, Null, Bytes, List, Set, Struct, Json }; + + Kind kind = Null; + const char* itype = "i32"; // Int: "i8".."u64" + bool is_unsigned = false; // Int: which of i/u is live + bool b = false; // Bool + long long i = 0; // Int (signed), Enum + unsigned long long u = 0; // Int (unsigned) + double f = 0.0; // Float (already rounded to float32 when f32) + bool f32 = false; // Float width + std::uintptr_t p = 0; // Ptr + std::string s; // Str raw bytes | Enum symbolic name ("" = none) | Json canonical text + std::vector w; // WStr UTF-16 units + // Bytes: length, sha256 hex, and either the full data (inline_full) or the first 32 bytes. + std::size_t blen = 0; + char sha[65] = {}; + bool inline_full = false; + std::vector data; + // List / Set: items. Struct: keys[i] names items[i] (insertion order is emission order). + std::vector items; + std::vector keys; + std::string name; // optional "n" (never emitted for Bytes) + + const char* type_name() const; + Tv&& named(const char* n) && { name = n; return std::move(*this); } + Tv& named(const char* n) & { name = n; return *this; } + // Struct helper. + Tv& add(const char* key, Tv v); +}; + +namespace tv { +Tv boolean(bool v); +Tv i8(std::int8_t v); +Tv i16(std::int16_t v); +Tv i32(std::int32_t v); +Tv i64(std::int64_t v); +Tv u8(std::uint8_t v); +Tv u16(std::uint16_t v); +Tv u32(std::uint32_t v); +Tv u64(std::uint64_t v); +Tv f32(float v); +Tv f64(double v); +Tv str(const char* s); // NUL-terminated raw bytes (nullptr -> ptr 0x0, see below) +Tv str(const void* bytes, std::size_t n); // raw bytes, embedded NULs preserved +Tv wstr(const std::uint16_t* units, std::size_t n); +Tv wstr(const wchar_t* s); // NUL-terminated; units truncated to 16 bits +Tv ptr(const void* p); +Tv ptr(std::uintptr_t p); +Tv enum_(long long v, const char* symbolic = nullptr); +Tv null(); +Tv bytes(const void* data, std::size_t n, std::size_t inline_max); +Tv list(std::vector items); +Tv set(std::vector items); +Tv struct_(); +Tv json(const char* canonical_text); // caller guarantees section-7 canonical JSON +} // namespace tv + +// ---- records ------------------------------------------------------------------------------------ + +struct SideEntry { + std::string name; + enum Before : std::uint8_t { Absent, IsNull, Value } before_kind = Absent; + Tv before; + Tv after; +}; + +struct OursSide { + std::string name; + Tv after; +}; + +struct DiffEntry { + std::string path; + const char* why = "exact"; // exact | ftol | type | len | missing | extra | hash | err + std::optional orig; + std::optional ours; + std::string orig_raw; // when non-empty: raw JSON text emitted instead of `orig` (e.g. a length) + std::string ours_raw; // same for `ours` (e.g. the quoted err text) + long long first_diff_offset = -1; // bytes only, when both inline + std::string note; // free text, "" = absent +}; + +struct Record { + std::int64_t ts = 0; + std::string hook; + Mode mode = Mode::Trace; + std::uint32_t call_id = 0; + std::uint32_t thread = 0; + int depth = -1; // <0 = omit + std::vector args; + std::optional ret; // nullopt = void -> null + std::vector side; + bool has_ours = false; + std::optional ours_ret; + std::vector ours_side; + int diverged = -1; // <0 omit, 0 false, 1 true + bool has_diff = false; + std::vector diff; + std::optional err; + std::optional note; +}; + +// Per-hook policy, recorded in meta.hooks (the keys tracecmp.py --tolerance accepts). +struct HookPolicy { + double ftol = 0.0; + const char* ftol_kind = "abs"; // abs | rel | ulp + bool ptr_exact = false; + std::vector unordered; // list paths to treat as sets +}; + +struct Meta { + std::string build; + std::string exe_sha256; + std::string started; + unsigned inline_max = 256; + std::vector> hooks; +}; + +// ---- output buffer -------------------------------------------------------------------------------- + +class Buf { +public: + Buf(); + ~Buf(); + Buf(const Buf&) = delete; + Buf& operator=(const Buf&) = delete; + + void put(char c); + void put(const void* bytes, std::size_t n); + void puts(const char* s); +#if defined(__MINGW32__) + void printf(const char* fmt, ...) __attribute__((format(gnu_printf, 2, 3))); +#else + void printf(const char* fmt, ...) __attribute__((format(printf, 2, 3))); +#endif + void clear(); + + const char* data() const { return data_ ? data_ : ""; } + std::size_t size() const { return len_; } + bool ok() const { return !overflow_; } + std::string str() const { return std::string(data(), len_); } + +private: + bool grow(std::size_t need); + char* data_ = nullptr; + std::size_t len_ = 0, cap_ = 0; + bool overflow_ = false; +}; + +// ---- emit --------------------------------------------------------------------------------------------- + +void emit_esc(Buf& out, const char* bytes, std::size_t n); // section 4 rule 2 (str) +void emit_esc(Buf& out, const std::string& s); +void emit_esc_w(Buf& out, const std::uint16_t* units, std::size_t n); // section 4 rule 2 (wstr) +void emit_int(Buf& out, long long v); +void emit_uint(Buf& out, unsigned long long v); +void emit_float(Buf& out, double v, bool f32); +void emit_tv(Buf& out, const Tv& v); +void emit_record(Buf& out, const Record& r); // one line, ends in "}\n" +void emit_meta(Buf& out, const Meta& m); // one line, ends in "}\n" + +// Canonical text of a typed value: emit_tv() output. Used as the sort key for sets. +std::string canonical(const Tv& v); + +} // namespace shim::trace diff --git a/src/shim/trace/hook.h b/src/shim/trace/hook.h new file mode 100644 index 0000000..a47a9d1 --- /dev/null +++ b/src/shim/trace/hook.h @@ -0,0 +1,222 @@ +// Hook template: mode dispatch (off / trace / compare / replace) around a hooked C function. +// +// A hook is a descriptor struct D (see docs/shim-trace.md, "Declaring a hook"): +// +// struct MyHook { +// static constexpr const char* name = "Game::Foo"; // record `hook` name +// static constexpr CallConv conv = CallConv::Cdecl; // Cdecl | Stdcall +// using Ret = int; // void allowed +// using Args = std::tuple; // parameter list +// static void describe_args(std::vector& out, Table* t, const char* k, int v); +// static Tv describe_ret(int r); // omit when Ret is void +// static void regions(std::vector& out, Table* t, const char* k, int v); +// static Args rebind(Scratch& s, Table* t, const char* k, int v); // args for `ours` +// static int ours(Table* t, const char* k, int v); // the reimplementation +// static HookPolicy policy(); // meta.hooks entry +// }; +// +// Hook::detour() is the function to install (MinHook target -> detour) and +// Hook::original the trampoline (or the real function in host tests). +// +// Mode semantics: +// Off call original. +// Trace snapshot regions, call original, snapshot again, emit {args, ret, side}. +// Compare snapshot, call original, snapshot; copy the *before* snapshots into scratch +// memory, rebind the args onto the copies, call ours, snapshot the copies, diff, +// emit {.., ours, diverged, diff}. The original's result is what the caller gets. +// Replace call ours only; nothing is emitted (nothing to diff against). +// +// Nothing escapes the hook boundary: every capture step is wrapped, a throw from ours or +// from a describe/regions/rebind callback becomes `err` in the record, and the original is +// always invoked in Off/Trace/Compare regardless of capture failures. +// +// thiscall (and any hook whose prototype is [unverified] in sots_addresses.h) must NOT use +// this template: see docs/M0.md gotcha 1. Those stay register-transparent asm stubs that +// only log an entry and tail-jump to the trampoline (trace-only, no return capture). +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "shim/trace/emitter.h" +#include "shim/trace/platform.h" +#include "shim/trace/tracer.h" + +namespace shim::trace { + +enum class CallConv { Cdecl, Stdcall }; + +template +struct FnPtr; +template +struct FnPtr { + using type = R(SHIM_CDECL*)(A...); +}; +template +struct FnPtr { + using type = R(SHIM_STDCALL*)(A...); +}; + +namespace detail { + +// Holds a return value, or nothing for void, so one code path serves both. +template +struct RetSlot { + R v{}; + template + void invoke(F f, X... x) { v = f(x...); } + template + void apply(F f, T& t) { v = std::apply(f, t); } + R get() const { return v; } + template + std::optional describe() const { return D::describe_ret(v); } +}; + +template <> +struct RetSlot { + template + void invoke(F f, X... x) { f(x...); } + template + void apply(F f, T& t) { std::apply(f, t); } + void get() const {} + template + std::optional describe() const { return std::nullopt; } +}; + +inline std::string what(const char* stage) { + std::string s = stage; + try { + throw; + } catch (const std::exception& e) { + s += ": "; + s += e.what(); + } catch (...) { + s += ": non-std exception"; + } + return s; +} + +} // namespace detail + +template +class Hook; + +template +class Hook> { +public: + using Ret = typename D::Ret; + using Fn = typename FnPtr::type; + + // Trampoline to the original (MinHook fills it) — or, in host tests, the real function. + inline static Fn original = nullptr; + inline static Mode mode = Mode::Off; + + // Reads the mode for D::name from the tracer (Off when the tracer is not open). + static void configure(Tracer& t) { mode = t.mode_for(D::name); } + static void register_policy(Tracer& t) { t.register_hook(D::name, D::policy()); } + + static Fn detour() { + if constexpr (D::conv == CallConv::Cdecl) return &detour_cdecl; + else return &detour_stdcall; + } + + static Ret dispatch(A... a) { + switch (mode) { + case Mode::Off: return original(a...); + case Mode::Replace: return D::ours(a...); + case Mode::Trace: return run(a...); + case Mode::Compare: return run(a...); + } + return original(a...); + } + +private: + static Ret SHIM_CDECL detour_cdecl(A... a) { return dispatch(a...); } + static Ret SHIM_STDCALL detour_stdcall(A... a) { return dispatch(a...); } + + template + static Ret run(A... a) { + Tracer& tr = Tracer::instance(); + const unsigned inline_max = tr.inline_max(); + Record rec; + std::vector regions; + std::vector before; + bool captured = false; + rec.mode = kCompare ? Mode::Compare : Mode::Trace; + rec.call_id = tr.next_call_id(); // before the original: nested hooks get later ids + rec.depth = tr.enter(); + try { + rec.hook = D::name; + D::describe_args(rec.args, a...); + D::regions(regions, a...); + for (const Region& r : regions) before.push_back(Snapshot::capture(r)); + captured = true; + } catch (...) { + rec.err = detail::what("capture before original"); + } + + detail::RetSlot orig_ret; + orig_ret.invoke(original, a...); // always; the game must see the original's behavior + + try { + if (captured) { + rec.ret = orig_ret.template describe(); + for (std::size_t i = 0; i < regions.size(); ++i) { + SideEntry e; + e.name = before[i].name; + e.before_kind = SideEntry::Value; + e.before = before[i].to_tv(inline_max); + e.after = before[i].recapture(regions[i]).to_tv(inline_max); + rec.side.push_back(std::move(e)); + } + if (kCompare) compare(rec, before, inline_max, a...); + } + } catch (...) { + rec.err = detail::what("capture after original"); + } + try { + tr.write(rec); + } catch (...) { + } + tr.leave(); + return orig_ret.get(); + } + + static void compare(Record& rec, const std::vector& before, unsigned inline_max, A... a) { + Scratch scratch(before); + detail::RetSlot ours_ret; + try { + std::tuple args2 = D::rebind(scratch, a...); + ours_ret.apply(&D::ours, args2); + } catch (...) { + rec.err = detail::what("ours"); + rec.diverged = 1; + rec.has_diff = true; + DiffEntry d; + d.path = "call"; + d.why = "err"; + Buf b; + emit_esc(b, *rec.err); + d.ours_raw = b.str(); + rec.diff.push_back(std::move(d)); + return; + } + rec.has_ours = true; + rec.ours_ret = ours_ret.template describe(); + for (std::size_t i = 0; i < scratch.count(); ++i) { + OursSide s; + s.name = scratch.current(i).name; + s.after = scratch.current(i).to_tv(inline_max); + rec.ours_side.push_back(std::move(s)); + } + rec.has_diff = true; + rec.diverged = diff_outputs(rec.ret, rec.side, rec.ours_ret, rec.ours_side, D::policy(), rec.diff) ? 1 : 0; + } +}; + +} // namespace shim::trace diff --git a/src/shim/trace/platform.h b/src/shim/trace/platform.h new file mode 100644 index 0000000..db04891 --- /dev/null +++ b/src/shim/trace/platform.h @@ -0,0 +1,122 @@ +// Tiny platform layer so the trace/compare infrastructure compiles both inside the shim +// (i686 MinGW, Win32 API) and on the host (Linux, pthread) for ctest. +// +// Nothing here throws. Nothing here allocates. +#pragma once + +#include +#include + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#include +#include +#include +#endif + +#if defined(_WIN32) && (defined(__i386__) || defined(_M_IX86)) +#define SHIM_X86_WIN 1 +#define SHIM_CDECL __attribute__((cdecl)) +#define SHIM_STDCALL __attribute__((stdcall)) +#else +#define SHIM_X86_WIN 0 +#define SHIM_CDECL +#define SHIM_STDCALL +#endif + +namespace shim { + +// Non-recursive mutex. CRITICAL_SECTION on Windows (recursive there, which is fine: a hook +// that re-enters the tracer from inside `write` would be a bug either way). +class Mutex { +public: + Mutex(); + ~Mutex(); + Mutex(const Mutex&) = delete; + Mutex& operator=(const Mutex&) = delete; + void lock(); + void unlock(); + +private: +#if defined(_WIN32) + CRITICAL_SECTION cs_; +#else + pthread_mutex_t m_; +#endif +}; + +class LockGuard { +public: + explicit LockGuard(Mutex& m) : m_(m) { m_.lock(); } + ~LockGuard() { m_.unlock(); } + LockGuard(const LockGuard&) = delete; + LockGuard& operator=(const LockGuard&) = delete; + +private: + Mutex& m_; +}; + +// Monotonic microseconds (QPC on Windows, CLOCK_MONOTONIC elsewhere). Absolute origin is +// arbitrary; the tracer subtracts its own start time. +std::int64_t monotonic_us(); + +// OS thread id (GetCurrentThreadId / gettid). +std::uint32_t thread_id(); + +// "YYYY-MM-DDTHH:MM:SSZ" of now, UTC. `out` must hold at least 21 bytes. +void utc_timestamp(char* out, unsigned cap); + +// ---- inline definitions --------------------------------------------------------------------- + +#if defined(_WIN32) +inline Mutex::Mutex() { InitializeCriticalSection(&cs_); } +inline Mutex::~Mutex() { DeleteCriticalSection(&cs_); } +inline void Mutex::lock() { EnterCriticalSection(&cs_); } +inline void Mutex::unlock() { LeaveCriticalSection(&cs_); } + +inline std::int64_t monotonic_us() { + LARGE_INTEGER f, c; + QueryPerformanceFrequency(&f); + QueryPerformanceCounter(&c); + // split to avoid overflowing 64 bits for long uptimes + const long long secs = c.QuadPart / f.QuadPart; + const long long rem = c.QuadPart % f.QuadPart; + return static_cast(secs * 1000000LL + rem * 1000000LL / f.QuadPart); +} + +inline std::uint32_t thread_id() { return static_cast(GetCurrentThreadId()); } + +inline void utc_timestamp(char* out, unsigned cap) { + SYSTEMTIME st; + GetSystemTime(&st); + if (cap < 21) { if (cap) out[0] = '\0'; return; } + std::snprintf(out, cap, "%04u-%02u-%02uT%02u:%02u:%02uZ", unsigned(st.wYear), unsigned(st.wMonth), + unsigned(st.wDay), unsigned(st.wHour), unsigned(st.wMinute), unsigned(st.wSecond)); +} +#else +inline Mutex::Mutex() { pthread_mutex_init(&m_, nullptr); } +inline Mutex::~Mutex() { pthread_mutex_destroy(&m_); } +inline void Mutex::lock() { pthread_mutex_lock(&m_); } +inline void Mutex::unlock() { pthread_mutex_unlock(&m_); } + +inline std::int64_t monotonic_us() { + timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return static_cast(ts.tv_sec) * 1000000LL + ts.tv_nsec / 1000; +} + +inline std::uint32_t thread_id() { return static_cast(syscall(SYS_gettid)); } + +inline void utc_timestamp(char* out, unsigned cap) { + time_t now = time(nullptr); + tm t; + gmtime_r(&now, &t); + if (cap < 21) { if (cap) out[0] = '\0'; return; } + strftime(out, cap, "%Y-%m-%dT%H:%M:%SZ", &t); +} +#endif + +} // namespace shim diff --git a/src/shim/trace/selftest.cpp b/src/shim/trace/selftest.cpp new file mode 100644 index 0000000..7e0f033 --- /dev/null +++ b/src/shim/trace/selftest.cpp @@ -0,0 +1,69 @@ +#include "shim/trace/selftest.h" + +#include + +namespace shim::selftest { + +using trace::Tv; + +std::uint32_t SHIM_CDECL Fill(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed) { + std::uint32_t x = seed, sum = 0; + for (std::uint32_t i = 0; i < n; ++i) { + x = x * 1664525u + 1013904223u; + buf[i] = static_cast(x >> 24); + sum = sum * 31u + buf[i]; + } + return sum; +} + +// Same contract, written differently (rolling checksum accumulated after the fill). +std::uint32_t SHIM_CDECL FillOurs(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed) { + std::uint32_t x = seed; + for (std::uint32_t i = 0; i < n; ++i) { + x = 1664525u * x + 1013904223u; + buf[i] = static_cast((x >> 24) & 0xffu); + } + std::uint32_t sum = 0; + for (std::uint32_t i = 0; i < n; ++i) sum = sum * 31u + buf[i]; + return sum; +} + +std::uint32_t SHIM_CDECL FillWrong(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed) { + const std::uint32_t sum = FillOurs(buf, n, seed); + if (n) buf[n - 1] = static_cast(buf[n - 1] + 1); + return sum; // checksum still reports the original's answer: only the region diverges +} + +std::uint32_t SHIM_CDECL FillThrows(std::uint8_t*, std::uint32_t, std::uint32_t) { + throw std::runtime_error("selftest: deliberate throw"); +} + +void FillHook::describe_args(std::vector& out, std::uint8_t* buf, std::uint32_t n, std::uint32_t seed) { + out.push_back(trace::tv::ptr(buf).named("buf")); + out.push_back(trace::tv::u32(n).named("n")); + out.push_back(trace::tv::u32(seed).named("seed")); +} + +Tv FillHook::describe_ret(std::uint32_t r) { return trace::tv::u32(r); } + +void FillHook::regions(std::vector& out, std::uint8_t* buf, std::uint32_t n, std::uint32_t) { + trace::Region r; + r.name = "buf"; + r.ptr = buf; + r.size = n; + out.push_back(r); +} + +FillHook::Args FillHook::rebind(trace::Scratch& s, std::uint8_t*, std::uint32_t n, std::uint32_t seed) { + return Args(s.as(0), n, seed); +} + +std::uint32_t run_once(trace::Mode mode) { + using H = trace::Hook; + H::original = &Fill; + H::mode = mode; + std::uint8_t buf[64] = {}; + return H::detour()(buf, sizeof buf, 0x5eed); +} + +} // namespace shim::selftest diff --git a/src/shim/trace/selftest.h b/src/shim/trace/selftest.h new file mode 100644 index 0000000..8aac6f6 --- /dev/null +++ b/src/shim/trace/selftest.h @@ -0,0 +1,53 @@ +// Worked example hook that needs no game knowledge: the shim's own `Fill` function. +// +// Fill(buf, n, seed): writes n bytes of an LCG stream into buf, returns a 32-bit checksum. +// +// FillHook wraps it with the hook template (args: ptr/u32/u32, ret: u32, one side region +// "buf"), with `ours` = an independent reimplementation. FillWrongHook is the same hook with +// a deliberately wrong `ours` (last byte off by one) so the compare path's divergence +// reporting can be exercised. The shim runs FillHook once at startup (see main.cpp) as a +// smoke record; the host test drives all modes and validates the log with tracecmp.py. +#pragma once + +#include +#include +#include + +#include "shim/trace/hook.h" + +namespace shim::selftest { + +std::uint32_t SHIM_CDECL Fill(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed); +std::uint32_t SHIM_CDECL FillOurs(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed); +std::uint32_t SHIM_CDECL FillWrong(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed); +std::uint32_t SHIM_CDECL FillThrows(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed); + +struct FillHook { + static constexpr const char* name = "Shim::SelfTest::Fill"; + static constexpr trace::CallConv conv = trace::CallConv::Cdecl; + using Ret = std::uint32_t; + using Args = std::tuple; + + static void describe_args(std::vector& out, std::uint8_t* buf, std::uint32_t n, std::uint32_t seed); + static trace::Tv describe_ret(std::uint32_t r); + static void regions(std::vector& out, std::uint8_t* buf, std::uint32_t n, std::uint32_t seed); + static Args rebind(trace::Scratch& s, std::uint8_t* buf, std::uint32_t n, std::uint32_t seed); + static std::uint32_t ours(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed) { return FillOurs(buf, n, seed); } + static trace::HookPolicy policy() { return trace::HookPolicy{}; } +}; + +struct FillWrongHook : FillHook { + static constexpr const char* name = "Shim::SelfTest::FillWrong"; + static std::uint32_t ours(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed) { return FillWrong(buf, n, seed); } +}; + +struct FillThrowsHook : FillHook { + static constexpr const char* name = "Shim::SelfTest::FillThrows"; + static std::uint32_t ours(std::uint8_t* buf, std::uint32_t n, std::uint32_t seed) { return FillThrows(buf, n, seed); } +}; + +// Runs FillHook once through its detour (original = Fill) with a 64-byte buffer. +// Returns the checksum. `mode` is applied to the hook first. +std::uint32_t run_once(trace::Mode mode); + +} // namespace shim::selftest diff --git a/src/shim/trace/sha256.cpp b/src/shim/trace/sha256.cpp new file mode 100644 index 0000000..07616de --- /dev/null +++ b/src/shim/trace/sha256.cpp @@ -0,0 +1,116 @@ +#include "shim/trace/sha256.h" + +#include + +namespace shim { + +namespace { + +const std::uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +inline std::uint32_t rotr(std::uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); } + +} // namespace + +Sha256::Sha256() : buflen_(0), total_(0) { + static const std::uint32_t init[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + std::memcpy(h_, init, sizeof h_); +} + +void Sha256::block(const std::uint8_t* p) { + std::uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (std::uint32_t(p[4 * i]) << 24) | (std::uint32_t(p[4 * i + 1]) << 16) | + (std::uint32_t(p[4 * i + 2]) << 8) | std::uint32_t(p[4 * i + 3]); + } + for (int i = 16; i < 64; ++i) { + const std::uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + const std::uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + std::uint32_t a = h_[0], b = h_[1], c = h_[2], d = h_[3], e = h_[4], f = h_[5], g = h_[6], h = h_[7]; + for (int i = 0; i < 64; ++i) { + const std::uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + const std::uint32_t ch = (e & f) ^ (~e & g); + const std::uint32_t t1 = h + S1 + ch + K[i] + w[i]; + const std::uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + const std::uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + const std::uint32_t t2 = S0 + maj; + h = g; g = f; f = e; e = d + t1; + d = c; c = b; b = a; a = t1 + t2; + } + h_[0] += a; h_[1] += b; h_[2] += c; h_[3] += d; + h_[4] += e; h_[5] += f; h_[6] += g; h_[7] += h; +} + +void Sha256::update(const void* data, std::size_t len) { + const std::uint8_t* p = static_cast(data); + total_ += len; + if (buflen_) { + const std::size_t take = (64 - buflen_ < len) ? 64 - buflen_ : len; + std::memcpy(buf_ + buflen_, p, take); + buflen_ += take; + p += take; + len -= take; + if (buflen_ < 64) return; + block(buf_); + buflen_ = 0; + } + while (len >= 64) { + block(p); + p += 64; + len -= 64; + } + if (len) { + std::memcpy(buf_, p, len); + buflen_ = len; + } +} + +void Sha256::finish(std::uint8_t out[32]) { + const std::uint64_t bits = total_ * 8; + std::uint8_t pad[72]; + std::size_t padlen = (buflen_ < 56) ? 56 - buflen_ : 120 - buflen_; + std::memset(pad, 0, sizeof pad); + pad[0] = 0x80; + for (int i = 0; i < 8; ++i) pad[padlen + i] = static_cast(bits >> (56 - 8 * i)); + update(pad, padlen + 8); + for (int i = 0; i < 8; ++i) { + out[4 * i] = static_cast(h_[i] >> 24); + out[4 * i + 1] = static_cast(h_[i] >> 16); + out[4 * i + 2] = static_cast(h_[i] >> 8); + out[4 * i + 3] = static_cast(h_[i]); + } +} + +void Sha256::digest(const void* data, std::size_t len, std::uint8_t out[32]) { + Sha256 s; + s.update(data, len); + s.finish(out); +} + +void Sha256::to_hex(const std::uint8_t d[32], char hex[65]) { + static const char* digits = "0123456789abcdef"; + for (int i = 0; i < 32; ++i) { + hex[2 * i] = digits[d[i] >> 4]; + hex[2 * i + 1] = digits[d[i] & 15]; + } + hex[64] = '\0'; +} + +void Sha256::digest_hex(const void* data, std::size_t len, char hex[65]) { + std::uint8_t d[32]; + digest(data, len, d); + to_hex(d, hex); +} + +} // namespace shim diff --git a/src/shim/trace/sha256.h b/src/shim/trace/sha256.h new file mode 100644 index 0000000..2a27e53 --- /dev/null +++ b/src/shim/trace/sha256.h @@ -0,0 +1,30 @@ +// Minimal SHA-256 (FIPS 180-4), written from the standard. No allocation, no exceptions. +#pragma once + +#include +#include + +namespace shim { + +class Sha256 { +public: + Sha256(); + void update(const void* data, std::size_t len); + // Finalizes into `out[32]`. The object must not be updated afterwards. + void finish(std::uint8_t out[32]); + + // One-shot helpers. + static void digest(const void* data, std::size_t len, std::uint8_t out[32]); + // Lowercase hex of the digest into `hex[65]` (NUL-terminated). + static void digest_hex(const void* data, std::size_t len, char hex[65]); + static void to_hex(const std::uint8_t d[32], char hex[65]); + +private: + void block(const std::uint8_t* p); + std::uint32_t h_[8]; + std::uint8_t buf_[64]; + std::size_t buflen_; + std::uint64_t total_; +}; + +} // namespace shim diff --git a/src/shim/trace/tracer.cpp b/src/shim/trace/tracer.cpp new file mode 100644 index 0000000..5038f19 --- /dev/null +++ b/src/shim/trace/tracer.cpp @@ -0,0 +1,401 @@ +#include "shim/trace/tracer.h" + +#include +#include +#include +#include +#include + +#include "shim/trace/sha256.h" + +namespace shim::trace { + +// ---- configuration -------------------------------------------------------------------------- + +bool Config::apply(const char* key, const char* value, std::string* err) { + auto fail = [&](const char* msg) { + if (err) *err = msg; + return true; + }; + if (std::strcmp(key, "hooks") == 0) { + Mode m; + if (!parse_mode(value, m)) return fail("hooks: expected off|trace|compare|replace"); + default_mode = m; + return true; + } + if (std::strncmp(key, "hook.", 5) == 0) { + const char* name = key + 5; + if (!*name) return fail("hook.: empty name"); + Mode m; + if (!parse_mode(value, m)) return fail("hook.: expected off|trace|compare|replace"); + for (auto& hm : hook_modes) { + if (hm.first == name) { + hm.second = m; + return true; + } + } + hook_modes.emplace_back(name, m); + return true; + } + if (std::strcmp(key, "trace.path") == 0) { + if (!*value) return fail("trace.path: empty"); + path = value; + return true; + } + if (std::strcmp(key, "trace.inline_max") == 0) { + char* end = nullptr; + const unsigned long n = std::strtoul(value, &end, 10); + if (!end || *end || end == value) return fail("trace.inline_max: expected a number"); + inline_max = static_cast(n); + return true; + } + if (std::strcmp(key, "trace.flush") == 0) { + if (std::strcmp(value, "always") == 0) flush_always = true; + else if (std::strcmp(value, "lazy") == 0) flush_always = false; + else return fail("trace.flush: expected always|lazy"); + return true; + } + return false; +} + +Mode Config::mode_for(const char* hook) const { + for (const auto& hm : hook_modes) + if (hm.first == hook) return hm.second; + return default_mode; +} + +// ---- snapshots --------------------------------------------------------------------------------- + +Snapshot Snapshot::capture(const Region& r) { + Snapshot s; + s.name = r.name; + s.describe = r.describe; + const std::uint8_t* p = static_cast(r.ptr); + if (p && r.size) s.data.assign(p, p + r.size); + return s; +} + +Snapshot Snapshot::recapture(const Region& r) const { + Snapshot s = capture(r); + s.name = name; + s.describe = describe; + return s; +} + +Tv Snapshot::to_tv(unsigned inline_max) const { + if (describe) return describe(data.data(), data.size(), inline_max); + return tv::bytes(data.data(), data.size(), inline_max); +} + +Scratch::Scratch(const std::vector& before) : copies_(before) {} + +// ---- diff ---------------------------------------------------------------------------------------------- + +namespace { + +DiffEntry entry(const std::string& path, const char* why, const Tv* a, const Tv* b) { + DiffEntry d; + d.path = path; + d.why = why; + if (a) d.orig = *a; + if (b) d.ours = *b; + return d; +} + +float to_f32(double x) { return static_cast(x); } + +long long ulp_int(double x, bool f32) { + if (f32) { + const float f = to_f32(x); + std::int32_t i; + std::memcpy(&i, &f, sizeof i); + return i >= 0 ? i : -static_cast(i & 0x7fffffff); + } + std::int64_t i; + std::memcpy(&i, &x, sizeof i); + return i >= 0 ? i : -static_cast(i & 0x7fffffffffffffffLL); +} + +// tracecmp.floats_equal with the f32 rounding rule applied by the caller. +bool floats_equal(double a, double b, const HookPolicy& pol, bool f32) { + if (std::isnan(a) || std::isnan(b)) return std::isnan(a) && std::isnan(b); + if (std::isinf(a) || std::isinf(b)) return a == b; + if (a == b) return true; + if (pol.ftol <= 0) return false; + if (std::strcmp(pol.ftol_kind, "abs") == 0) return std::fabs(a - b) <= pol.ftol; + if (std::strcmp(pol.ftol_kind, "rel") == 0) + return std::fabs(a - b) <= pol.ftol * std::max(std::fabs(a), std::fabs(b)); + if (std::strcmp(pol.ftol_kind, "ulp") == 0) { + const long long d = ulp_int(a, f32) - ulp_int(b, f32); + return static_cast(d < 0 ? -d : d) <= pol.ftol; + } + return false; +} + +bool is_unordered(const HookPolicy& pol, const std::string& path) { + for (const auto& p : pol.unordered) + if (p == path) return true; + return false; +} + +std::vector sorted_by_canonical(const std::vector& items) { + std::vector> keyed; + keyed.reserve(items.size()); + for (const Tv& t : items) keyed.emplace_back(canonical(t), &t); + std::sort(keyed.begin(), keyed.end(), + [](const auto& x, const auto& y) { return x.first < y.first; }); + std::vector out; + out.reserve(keyed.size()); + for (const auto& k : keyed) out.push_back(k.second); + return out; +} + +const Tv* struct_field(const Tv& s, const std::string& key) { + for (std::size_t i = 0; i < s.keys.size(); ++i) + if (s.keys[i] == key) return &s.items[i]; + return nullptr; +} + +std::string len_text(std::size_t n) { return std::to_string(n); } + +} // namespace + +bool diff_tv(const Tv& a, const Tv& b, const HookPolicy& pol, const std::string& path, + std::vector& out) { + const std::size_t start = out.size(); + if (a.kind != b.kind || (a.kind == Tv::Int && std::strcmp(a.itype, b.itype) != 0) || + (a.kind == Tv::Float && a.f32 != b.f32)) { + out.push_back(entry(path, "type", &a, &b)); + return true; + } + switch (a.kind) { + case Tv::Ptr: { + const bool za = a.p == 0, zb = b.p == 0; + if (za != zb || (pol.ptr_exact && a.p != b.p)) out.push_back(entry(path, "exact", &a, &b)); + break; + } + case Tv::Null: + break; + case Tv::Bool: + if (a.b != b.b) out.push_back(entry(path, "exact", &a, &b)); + break; + case Tv::Enum: + if (a.i != b.i) out.push_back(entry(path, "exact", &a, &b)); + break; + case Tv::Str: + case Tv::Json: + if (a.s != b.s) out.push_back(entry(path, "exact", &a, &b)); + break; + case Tv::WStr: + if (a.w != b.w) out.push_back(entry(path, "exact", &a, &b)); + break; + case Tv::Int: + if (a.is_unsigned ? (a.u != b.u) : (a.i != b.i)) out.push_back(entry(path, "exact", &a, &b)); + break; + case Tv::Float: { + // %.9g round-trips the float32, not the widened double: compare at float32 width. + const double x = a.f32 ? static_cast(to_f32(a.f)) : a.f; + const double y = b.f32 ? static_cast(to_f32(b.f)) : b.f; + if (!floats_equal(x, y, pol, a.f32)) out.push_back(entry(path, pol.ftol > 0 ? "ftol" : "exact", &a, &b)); + break; + } + case Tv::Bytes: { + if (a.blen != b.blen) { + out.push_back(entry(path, "len", &a, &b)); + break; + } + if (std::strcmp(a.sha, b.sha) != 0) { + DiffEntry d = entry(path, "hash", &a, &b); + if (a.inline_full && b.inline_full) { + const std::size_t n = std::min(a.data.size(), b.data.size()); + for (std::size_t i = 0; i < n; ++i) { + if (a.data[i] != b.data[i]) { + d.first_diff_offset = static_cast(i); + break; + } + } + } + out.push_back(std::move(d)); + } else if (a.inline_full && b.inline_full && a.data != b.data) { + DiffEntry d = entry(path, "hash", &a, &b); + d.note = "same sha256, different hex: corrupt log"; + out.push_back(std::move(d)); + } + break; + } + case Tv::List: + case Tv::Set: { + std::vector va, vb; + if (a.kind == Tv::Set || is_unordered(pol, path)) { + va = sorted_by_canonical(a.items); + vb = sorted_by_canonical(b.items); + } else { + for (const Tv& t : a.items) va.push_back(&t); + for (const Tv& t : b.items) vb.push_back(&t); + } + if (va.size() != vb.size()) { + DiffEntry d = entry(path, "len", nullptr, nullptr); + d.orig_raw = len_text(va.size()); + d.ours_raw = len_text(vb.size()); + out.push_back(std::move(d)); + break; + } + for (std::size_t i = 0; i < va.size(); ++i) + diff_tv(*va[i], *vb[i], pol, path + ".v[" + std::to_string(i) + "]", out); + break; + } + case Tv::Struct: { + std::vector ka = a.keys, kb = b.keys; + std::sort(ka.begin(), ka.end()); + std::sort(kb.begin(), kb.end()); + for (const std::string& k : ka) + if (!struct_field(b, k)) out.push_back(entry(path + ".v." + k, "missing", struct_field(a, k), nullptr)); + for (const std::string& k : kb) + if (!struct_field(a, k)) out.push_back(entry(path + ".v." + k, "extra", nullptr, struct_field(b, k))); + for (const std::string& k : ka) + if (const Tv* fb = struct_field(b, k)) diff_tv(*struct_field(a, k), *fb, pol, path + ".v." + k, out); + break; + } + } + return out.size() > start; +} + +bool diff_outputs(const std::optional& ret_a, const std::vector& side_a, + const std::optional& ret_b, const std::vector& side_b, const HookPolicy& pol, + std::vector& out) { + const std::size_t start = out.size(); + if (ret_a.has_value() != ret_b.has_value()) { + out.push_back(entry("ret", ret_b ? "extra" : "missing", ret_a ? &*ret_a : nullptr, ret_b ? &*ret_b : nullptr)); + } else if (ret_a) { + diff_tv(*ret_a, *ret_b, pol, "ret", out); + } + auto find_a = [&](const std::string& n) -> const SideEntry* { + for (const auto& e : side_a) if (e.name == n) return &e; + return nullptr; + }; + auto find_b = [&](const std::string& n) -> const OursSide* { + for (const auto& e : side_b) if (e.name == n) return &e; + return nullptr; + }; + std::vector na, nb; + for (const auto& e : side_a) na.push_back(e.name); + for (const auto& e : side_b) nb.push_back(e.name); + std::sort(na.begin(), na.end()); + std::sort(nb.begin(), nb.end()); + for (const std::string& n : na) + if (!find_b(n)) out.push_back(entry("side." + n + ".after", "missing", &find_a(n)->after, nullptr)); + for (const std::string& n : nb) + if (!find_a(n)) out.push_back(entry("side." + n + ".after", "extra", nullptr, &find_b(n)->after)); + for (const std::string& n : na) + if (const OursSide* b = find_b(n)) diff_tv(find_a(n)->after, b->after, pol, "side." + n + ".after", out); + return out.size() > start; +} + +// ---- tracer ------------------------------------------------------------------------------------------------ + +namespace { +thread_local int t_depth = 0; +std::atomic g_next_call_id{0}; +} // namespace + +Tracer& Tracer::instance() { + static Tracer t; + return t; +} + +void Tracer::configure(const Config& cfg) { cfg_ = cfg; } + +void Tracer::register_hook(const char* name, const HookPolicy& policy) { + for (auto& h : hooks_) { + if (h.first == name) { + h.second = policy; + return; + } + } + hooks_.emplace_back(name, policy); +} + +bool Tracer::open(const char* build_id, const char* exe_sha256) { + if (file_) return true; + if (cfg_.path.empty()) return false; + std::FILE* f = std::fopen(cfg_.path.c_str(), "wb"); + if (!f) return false; + Meta m; + m.build = build_id ? build_id : ""; + m.exe_sha256 = exe_sha256 ? exe_sha256 : ""; + char ts[32]; + utc_timestamp(ts, sizeof ts); + m.started = ts; + m.inline_max = cfg_.inline_max; + m.hooks = hooks_; + Buf b; + emit_meta(b, m); + if (!b.ok()) { + std::fclose(f); + return false; + } + std::fwrite(b.data(), 1, b.size(), f); + std::fflush(f); + LockGuard g(mu_); + file_ = f; + t0_ = monotonic_us(); + written_ = 0; + return true; +} + +void Tracer::close() { + LockGuard g(mu_); + if (!file_) return; + std::fflush(file_); + std::fclose(file_); + file_ = nullptr; +} + +Mode Tracer::mode_for(const char* hook) const { + if (!file_) return Mode::Off; + return cfg_.mode_for(hook); +} + +std::uint32_t Tracer::next_call_id() { return g_next_call_id.fetch_add(1, std::memory_order_relaxed); } + +int Tracer::enter() { return t_depth++; } +void Tracer::leave() { if (t_depth > 0) --t_depth; } + +std::int64_t Tracer::now_us() const { return monotonic_us() - t0_; } + +bool Tracer::write(Record& rec) { + if (!file_) return false; + rec.ts = now_us(); + rec.thread = thread_id(); + Buf b; + emit_record(b, rec); + if (!b.ok()) return false; + const bool urgent = cfg_.flush_always || rec.diverged > 0 || rec.err.has_value(); + LockGuard g(mu_); + if (!file_) return false; + std::fwrite(b.data(), 1, b.size(), file_); + if (urgent) std::fflush(file_); + ++written_; + return true; +} + +void Tracer::flush() { + LockGuard g(mu_); + if (file_) std::fflush(file_); +} + +bool sha256_file(const char* path, char hex[65]) { + std::FILE* f = std::fopen(path, "rb"); + if (!f) return false; + Sha256 h; + std::uint8_t buf[65536]; + std::size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) h.update(buf, n); + std::fclose(f); + std::uint8_t d[32]; + h.finish(d); + Sha256::to_hex(d, hex); + return true; +} + +} // namespace shim::trace diff --git a/src/shim/trace/tracer.h b/src/shim/trace/tracer.h new file mode 100644 index 0000000..4dd8255 --- /dev/null +++ b/src/shim/trace/tracer.h @@ -0,0 +1,133 @@ +// Tracer: per-hook modes from shim.cfg, the locked JSONL writer, call ids / depth, side-effect +// snapshots + scratch copies, and the shim-side diff (TRACE_FORMAT.md section 6). +// +// Host-testable: only platform.h touches the OS. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "shim/trace/emitter.h" +#include "shim/trace/platform.h" + +namespace shim::trace { + +// ---- configuration -------------------------------------------------------------------------- +// +// shim.cfg keys handled here (main.cpp forwards every key it does not own): +// hooks=off|trace|compare|replace default mode for every hook ("off" = nothing installed) +// hook.=off|trace|compare|replace per-hook override ( is the hook's record name) +// trace.path= log file (default: \shim.trace.jsonl) +// trace.inline_max= bytes inlined as hex (default 256) +// trace.flush=always|lazy fflush after every record (default) or only on +// divergence / err / close +struct Config { + Mode default_mode = Mode::Trace; + std::vector> hook_modes; + std::string path; + unsigned inline_max = 256; + bool flush_always = true; + + // Returns true when `key` is a trace key. Sets *err (if given) and returns true when the + // key is ours but the value is unusable (the key is then left unchanged). + bool apply(const char* key, const char* value, std::string* err = nullptr); + Mode mode_for(const char* hook) const; +}; + +// ---- side-effect regions ------------------------------------------------------------------------ + +// A declared region of memory the hooked function may write. `describe` turns the captured +// bytes into a structured Tv (so diffs name a field); nullptr = plain `bytes`. +struct Region { + const char* name = ""; + const void* ptr = nullptr; + std::size_t size = 0; + Tv (*describe)(const void* data, std::size_t size, unsigned inline_max) = nullptr; +}; + +struct Snapshot { + std::string name; + std::vector data; + Tv (*describe)(const void*, std::size_t, unsigned) = nullptr; + + static Snapshot capture(const Region& r); + // Re-read the same region (for "after"); the result shares name/describe. + Snapshot recapture(const Region& r) const; + Tv to_tv(unsigned inline_max) const; +}; + +// Writable copies of a set of snapshots: the state `ours` runs on in compare mode. +class Scratch { +public: + explicit Scratch(const std::vector& before); + std::size_t count() const { return copies_.size(); } + void* ptr(std::size_t i) { return copies_[i].data.data(); } + template + T* as(std::size_t i) { return static_cast(ptr(i)); } + std::size_t size(std::size_t i) const { return copies_[i].data.size(); } + const Snapshot& current(std::size_t i) const { return copies_[i]; } // reflects writes by ours + +private: + std::vector copies_; +}; + +// ---- diff ------------------------------------------------------------------------------------------ + +// Appends diff entries for `a` (original) vs `b` (ours) at `path`; returns true if any. +bool diff_tv(const Tv& a, const Tv& b, const HookPolicy& pol, const std::string& path, + std::vector& out); + +// Section 6 over a whole record: ret, then side names (missing/extra), then side..after. +bool diff_outputs(const std::optional& ret_a, const std::vector& side_a, + const std::optional& ret_b, const std::vector& side_b, const HookPolicy& pol, + std::vector& out); + +// ---- tracer ------------------------------------------------------------------------------------------ + +class Tracer { +public: + static Tracer& instance(); + + // All three before open(): policy goes into the meta record. + void configure(const Config& cfg); + void register_hook(const char* name, const HookPolicy& policy); + const Config& config() const { return cfg_; } + + // Opens cfg.path ("w"), writes the meta line. false = could not open (tracing stays off). + bool open(const char* build_id, const char* exe_sha256); + void close(); + bool is_open() const { return file_ != nullptr; } + + Mode mode_for(const char* hook) const; // Off when not open + unsigned inline_max() const { return cfg_.inline_max; } + + std::uint32_t next_call_id(); // atomic; take it BEFORE calling the original + int enter(); // ++ thread-local depth; returns the depth of this call + void leave(); + std::int64_t now_us() const; // microseconds since open() + + // Fills ts/thread, serializes, appends under the lock. false = not open / buffer overflow. + bool write(Record& rec); + void flush(); + std::uint32_t records_written() const { return written_; } + +private: + Tracer() = default; + Config cfg_; + std::vector> hooks_; + std::FILE* file_ = nullptr; + std::int64_t t0_ = 0; + Mutex mu_; + std::uint32_t next_id_ = 0; + std::uint32_t written_ = 0; +}; + +// SHA-256 of a whole file, lowercase hex. false if unreadable. +bool sha256_file(const char* path, char hex[65]); + +} // namespace shim::trace diff --git a/tests/shim_trace/CMakeLists.txt b/tests/shim_trace/CMakeLists.txt new file mode 100644 index 0000000..519c02a --- /dev/null +++ b/tests/shim_trace/CMakeLists.txt @@ -0,0 +1,22 @@ +# Host tests for the shim's trace/compare infrastructure (src/shim/trace). The Python +# oracle (sots-re/verify/harness/compare) is used when SOTS_TRACECMP_DIR points at it; +# the tests skip those steps cleanly otherwise. +set(SOTS_TRACECMP_DIR "$ENV{HOME}/sots-re/verify/harness/compare" CACHE PATH + "Directory holding tracecmp.py + mkfixture.py (python oracle for the trace tests)") + +set(_out ${CMAKE_CURRENT_BINARY_DIR}/out) +file(MAKE_DIRECTORY ${_out}) + +foreach(_t sha256 emitter diff hook) + add_executable(shim_trace_test_${_t} test_${_t}.cpp) + target_link_libraries(shim_trace_test_${_t} PRIVATE shim_trace) + target_compile_options(shim_trace_test_${_t} PRIVATE -Wall -Wextra -Werror) + target_compile_definitions(shim_trace_test_${_t} PRIVATE SOTS_TRACECMP_DIR="${SOTS_TRACECMP_DIR}") +endforeach() +target_link_libraries(shim_trace_test_hook PRIVATE pthread) + +add_test(NAME shim_trace_sha256 COMMAND shim_trace_test_sha256) +add_test(NAME shim_trace_emitter COMMAND shim_trace_test_emitter ${_out}/emitter_oracle.jsonl + ${CMAKE_CURRENT_SOURCE_DIR}/oracle_emit.py) +add_test(NAME shim_trace_diff COMMAND shim_trace_test_diff) +add_test(NAME shim_trace_hook COMMAND shim_trace_test_hook ${_out}) diff --git a/tests/shim_trace/check.h b/tests/shim_trace/check.h new file mode 100644 index 0000000..e8b9f6c --- /dev/null +++ b/tests/shim_trace/check.h @@ -0,0 +1,37 @@ +// Minimal test helpers for the shim trace tests (no dependency on other modules). +#pragma once + +#include +#include + +namespace tracetest { + +inline int& failures() { static int n = 0; return n; } +inline int& checks() { static int n = 0; return n; } + +inline void report(bool ok, const char* expr, const char* file, int line, const std::string& detail) { + ++checks(); + if (ok) return; + ++failures(); + std::fprintf(stderr, "FAIL %s:%d %s%s%s\n", file, line, expr, detail.empty() ? "" : " -- ", detail.c_str()); +} + +inline void check_str(const std::string& got, const std::string& want, const char* expr, const char* file, int line) { + report(got == want, expr, file, line, got == want ? "" : "\n got: " + got + "\n want: " + want); +} + +template +inline void check_eq(const A& a, const B& b, const char* expr, const char* file, int line) { + report(a == b, expr, file, line, a == b ? "" : "got " + std::to_string(a) + ", expected " + std::to_string(b)); +} + +inline int finish(const char* name) { + std::printf("%s: %d checks, %d failures\n", name, checks(), failures()); + return failures() == 0 ? 0 : 1; +} + +} // namespace tracetest + +#define CHECK(expr) ::tracetest::report((expr), #expr, __FILE__, __LINE__, "") +#define CHECK_EQ(a, b) ::tracetest::check_eq((a), (b), #a " == " #b, __FILE__, __LINE__) +#define CHECK_STR(got, want) ::tracetest::check_str((got), (want), #got, __FILE__, __LINE__) diff --git a/tests/shim_trace/harness.h b/tests/shim_trace/harness.h new file mode 100644 index 0000000..311e43f --- /dev/null +++ b/tests/shim_trace/harness.h @@ -0,0 +1,62 @@ +// Running the Python oracle (sots-re/verify/harness/compare) from a host test. +// +// The harness directory comes from -DSOTS_TRACECMP_DIR (CMake) or the SOTS_TRACECMP_DIR +// environment variable; when neither points at a directory holding tracecmp.py the +// python steps are skipped (returning kSkipped) and the test still passes. +#pragma once + +#include +#include +#include + +namespace tracetest { + +constexpr int kSkipped = -1; + +inline std::string harness_dir() { + if (const char* e = std::getenv("SOTS_TRACECMP_DIR"); e && *e) return e; +#ifdef SOTS_TRACECMP_DIR + return SOTS_TRACECMP_DIR; +#else + return ""; +#endif +} + +inline bool file_exists(const std::string& p) { + if (std::FILE* f = std::fopen(p.c_str(), "rb")) { + std::fclose(f); + return true; + } + return false; +} + +inline bool harness_present() { return file_exists(harness_dir() + "/tracecmp.py"); } + +// Exit status of `python3