diff --git a/CMakeLists.txt b/CMakeLists.txt index 7249c71..d37b824 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,8 @@ add_subdirectory(src/mars/parse) # brace-block + .effect readers (lib mars_p add_subdirectory(src/mars/text) # flat-kv, id-manifest, csv (lib mars_text) add_subdirectory(src/game/sim) # strategic formulas, pure (lib sots_game_sim) add_subdirectory(src/mars/vfs) # .gob ZIP reader + native override (lib mars_vfs, miniz) +add_subdirectory(src/mars/stream) # Streamable save format + gzip (lib mars_stream) +add_subdirectory(src/mars/rng) # MT19937 (lib mars_rng) if(WIN32) # ---- shim: proxy binkw32.dll that the original game loads (Phase 2 frontend) ---- @@ -45,7 +47,7 @@ else() add_executable(addr_smoke tests/addr_smoke.cpp) target_link_libraries(addr_smoke PRIVATE sots_addresses) add_test(NAME addr_smoke COMMAND addr_smoke) - foreach(_t mars_parse mars_text game_sim mars_vfs) + foreach(_t mars_parse mars_text game_sim mars_vfs mars_stream) if(EXISTS ${CMAKE_SOURCE_DIR}/tests/${_t}/CMakeLists.txt) add_subdirectory(tests/${_t}) endif() diff --git a/docs/mars-rng.md b/docs/mars-rng.md new file mode 100644 index 0000000..8372858 --- /dev/null +++ b/docs/mars-rng.md @@ -0,0 +1,60 @@ +# mars::rng — the engine PRNG + +`src/mars/rng/mt19937.h` — a textbook 32-bit Mersenne Twister (MT19937), the generator the +strategy simulation draws every roll from (map generation, research, encounters, raids, …). +Because all lockstep peers share one seeded stream, the reimplementation has to be bit-exact +and consume words in the same order; the save file carries the generator state verbatim. + +## API + +```cpp +mars::rng::MT19937 r(seed); // seed(): Knuth initializer, then one twist (left == 624) +uint32_t y = r.next_u32(); // tempered output +float f = r.next_float(); // (float)(y * 2^-32), see below +uint32_t k = r.next_int(n); // uniform [0, n): power-of-two mask + rejection +r.load_state(mt, left); // or load_state(blob, 0x9c4) from a save's "RNG" item +r.save_state(out); // mt[624] + left, 0x9c4 bytes little-endian +r.left(); r.index(); r.state(); +``` + +## State model + +| member | meaning | +|---|---| +| `mt[624]` | the untempered state block | +| `left` | words still unread in the current block; next output is `mt[624 - left]` | + +`next_u32()` twists when `left` is 0, hands out `mt[624 - left]`, decrements `left`, and +tempers. A freshly seeded generator has already twisted once, so `left == 624` and the +first draw is `mt[0]`. + +## Serialized form (the save's `RNG` frame) + +`Sim → RNG { "." raw[2503] }`: 624 × uint32 (`mt`) followed by one int32 (`left`) = 0x9c4 = +2500 bytes, plus the 3 joint-padding bytes of the item. `MT19937::load_state(blob, n)` parses +it and rejects `left` outside 0..624; `save_state` writes the same layout. + +**Verified on the real saves** (`tests/mars_stream/test_save.cpp`): the 624-word block in all +three saves equals `seed(CreateParams.RSeed)` followed by exactly two whole twists, and `left` +decreases turn over turn (454 → 432 → 413, i.e. ~20 draws per turn). That confirms the +initializer, the twist, the seed source (`RSeed`) and the blob layout. It does not exercise +the tempering or the float mapping (those never touch the saved state). + +## Reference vectors (`tests/mars_stream/test_rng.cpp`) + +* seed 5489 → 3499211612, 581869302, 3890346734, … ; the 10000th output is 4123659995. +* `save_state`/`load_state` round trip, `left` positioning, malformed-blob rejection. + +## Choices that still need binary confirmation + +1. **Float mapping.** `next_float()` returns `(float)((double)y * 2^-32)`. This is the mapping + recorded in the RE notes for the engine's float roll (product in double, then narrowed). + Note the narrowing rounds `y >= 0xFFFFFF80` up to exactly `1.0f`, so the range is `[0, 1]` + in practice. Confirm against a captured sequence before relying on the exact bits. +2. **Twist timing at the block boundary.** We twist lazily when `left` reaches 0 (so a saved + state may carry `left == 0`). If the original twists eagerly after the last word of a block + (`left` then never saved as 0, block already advanced), the output sequence is identical but + the saved blob at that one boundary differs. The three saves (`left` = 454/432/413) do not + distinguish the two. +3. **`next_int(n)`** — mask = smallest `2^k - 1 >= n - 1`, reject while `r >= n`. The rejection + scheme matches the RE description; the exact mask computation is unconfirmed. diff --git a/docs/mars-stream.md b/docs/mars-stream.md new file mode 100644 index 0000000..e2e4fa6 --- /dev/null +++ b/docs/mars-stream.md @@ -0,0 +1,122 @@ +# mars::stream — Streamable serialization (save files) + +`src/mars/stream/` reimplements the engine's self-describing "Streamable" stream as used by +`.sav` files: a generic walker that recovers the item tree from any stream, a writer that +emits the same framing, and typed shapes for the confirmed top-level structures. The format +facts come from the RE repo's confirmed description (`verify/save-reader/SAVE_FORMAT.md`); +the reference Python reader is the oracle the C++ is tested against. + +## The format in one screen + +``` +container one gzip member; everything below is the inflated stream, little-endian +item [int32 len][name bytes][value][NUL pad] pad brings the WHOLE item to 4 bytes + ("joint" padding; a NULL name is written as ".") +scalars int32 | float32 | bool (1 byte) | int64 | string = [int32 len][cp1252 bytes] + (an empty string is 4 zero bytes — byte-identical to int 0; no type bytes anywhere) +frame [len][name][pad] BE EF BE EF ...items... 10 41 10 41 (0x41104110 = ~0xBEEFBEEF) +arrays VectorHelper: a frame holding "." count + n × "." elements (frames or scalars) + inline arrays: a named count followed by n × element in the same frame +Vector3 a frame of 3 × "." float +root Summary → CreateParams → Sim → CDT → n × CD +RNG Sim.RNG { "." raw[2503] }: MT19937 mt[624] + left (0x9c4) + 3 pad bytes (see mars-rng.md) +``` + +## Modules + +| file | contents | +|---|---| +| `bytes.h` | LE read/write helpers, `pad4`, the two markers | +| `node.h` | `Node{name, tagged, kind, raw, hinted, offset, size, children}`, `Issue`, `Stats` | +| `gzip.h/.cpp` | `gunzip`, `inflate_container` (passthrough for already-inflated data), `gzip` — via vendored miniz | +| `reader.h/.cpp` | `Walker` — the generic tokenizer; `read_tree()` | +| `writer.h/.cpp` | `Writer` — scalar/frame/raw emitters with joint padding; `Writer::node()` re-emits a tree | +| `dump.h/.cpp` | `dump_tree()` text form (identical to the reference `--dump`), Python float repr, JSON quoting | +| `schema.h` | `Desc`/`Hint`/`Registry` — the type hints the walker consults | +| `archive.h` | `ReadArchive`, `WriteArchive`, `SchemaBuilder` — drive a shape's `io()` field list | +| `shapes.h` | the typed shapes (Summary, CreateParams/MapP, Sim, Player, Sys, Fleet, Ship, …) | +| `save.h/.cpp` | `read_save_file/bytes()` → `SaveDocument{inflated, tree, game, issues, stats}`, `write_save()`, `write_tree()`, `save_registry()` | +| `savedump_main.cpp` | `sots_savedump SAVE [--dump] [--strict] [--roundtrip] [--rewrite OUT]` | + +## Walker (generic reader) + +The walker needs no schema. Frames are hard synchronisation points; between markers each +item's scalar layout is chosen by + +1. a **hint** — positional (`Desc::prefix`, from the shape describing the frame) or by tag + (`Desc::by_name`, then the global tag → kind catalog), or +2. **guessing**: try `[word, bool, string, int64]` and keep the first layout after which + another plausible item, a marker or EOF follows (two items of lookahead; string values + only text-checked while guessing, since a known string may hold any cp1252 byte); a bare + 4-byte word is int if `|i| <= 100000`, float if finite with `1e-6 <= |f| < 1e12`, else int. + +Nothing readable is lost: when no layout fits, the bytes up to the next marker or plausible +tag become a `raw` node (a warning), a small unnamed payload before an END is a `raw` node +(info), and the walk resumes. Every node keeps its exact value bytes and offset, so +`write_tree(read_tree(x)) == x` for any input the walker accepts, damaged or not. These rules +are the reference reader's; the test suite checks the two implementations tokenise the three +real saves identically (every line of the dump, including the guessed/hinted marks). + +Hints are not hand-maintained: `save_registry()` runs every shape's `io()` under +`SchemaBuilder`, which records prefixes, by-name tables and the global catalog (a tag +claimed with two different kinds is dropped from the catalog; optional legacy tags register +at lowest priority; `"."` is never a hint key because it carries ints, floats and frames alike). + +## Shapes and archives + +A shape is a struct with `kStreamName` and a `template void io(Ar&)` listing its +fields in disk order: + +```cpp +struct Summary { + static constexpr const char* kStreamName = "Summary"; + std::string gameName; int32_t turn = 0; ... + template void io(Ar& ar) { + ar.str(A("GameName"), gameName); // A(): confirmed on-disk tag, matched by name + ar.i32(A("Turn"), turn); + ar.carr(A("Players"), players); // VectorHelper: "." count + "." frames + ... + ar.rest(extra); // anything the shape does not describe, kept generic + } +}; +``` + +`R("idx")` fields are positional (the game wrote a NULL name — `"."` on disk — or the +reference name differs from the disk spelling, e.g. `faiDes` → `FAIDes`, `ontF` → `otnF`, +`nextId` → `nextid`); the second argument of `R` is the tag to write. Conditionals are plain +`ar.when(vnh, ...)`, optional legacy tags `ar.opt_i32(A("ARes"), aRes)`, uncounted lists +`ar.repeat("stats", ...)`, inline arrays `ar.narr(A("NumSys"), systems, elem)`. Bodies the +format keeps opaque (`TechTree`, `Events`, `ShipRecs`, `spy2`, `civr`, `comms`, `Ojvs`, +`Attrib`, `sprjs`, `SvSctOb`, `trdmgr`, `spymgr`, `CD`, the RNG blob) are `Node` members and +re-emit verbatim. + +`ReadArchive` mirrors the reference applier: a confirmed tag that is missing is an error, +unexpected items before it a warning, a positional item read under another tag an info; +kinds are coerced with the same width/zero rules (4 zero bytes are the empty string). The +same `io()` drives `WriteArchive`, so a shape loaded from a real save writes back +**byte-identically** — the whole file, typed shapes plus retained generic bodies. + +## Tests (`tests/mars_stream/build_and_run.sh`, plain g++) + +* `test_stream.cpp` — hand-built byte fixtures: primitive encodings and joint padding (bytes + written out by hand and compared with the `Writer`), framing/nesting/tagless/empty frames, + `"."` arrays, resync (two fixtures cross-checked with the reference reader), cp1252 values, + hints from the registry, the raw RNG frame, typed Summary/Ship/Fleet write → read → write, + gzip, Python float repr and JSON quoting. +* `test_rng.cpp` — MT19937 reference vectors and state (de)serialization. +* `test_save.cpp` — **real saves** when `SOTS_SAVES_DIR` is set (skips otherwise, saves never + enter the repo): 0 errors/warnings/resyncs/hint failures, only the 2503-byte RNG blob raw, + typed values consistent (Summary ↔ Sim ↔ CreateParams counts, names, 7 species), RNG state + reachable from `seed(RSeed)`, tree and typed round trips byte-identical; with + `SOTS_SAVE_READER` set, `oracle/compare.py` diffs the C++ dump against the reference + `--dump` (exact and canonical agreement, summary values). + +Result on the owner's three saves (SotS 1.8, turns 1–3): 100% exact line agreement, both +round trips identical, summaries agree. + +## Not done / open + +* Only joint padding is implemented (the split convention never matched a real file). +* Shapes for the opaque bodies listed above (kept generic on purpose, as in the reference). +* The float-vs-int guess for unhinted words inside opaque bodies is a heuristic; the typed + layer never depends on it (it coerces from the raw bytes). diff --git a/src/mars/rng/CMakeLists.txt b/src/mars/rng/CMakeLists.txt new file mode 100644 index 0000000..e72d449 --- /dev/null +++ b/src/mars/rng/CMakeLists.txt @@ -0,0 +1,6 @@ +# mars::rng — the engine PRNG (MT19937). +# Included from the root CMakeLists.txt via add_subdirectory(src/mars/rng). + +add_library(mars_rng STATIC mt19937.cpp) +target_include_directories(mars_rng PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../..) +target_compile_features(mars_rng PUBLIC cxx_std_17) diff --git a/src/mars/rng/mt19937.cpp b/src/mars/rng/mt19937.cpp new file mode 100644 index 0000000..1b22498 --- /dev/null +++ b/src/mars/rng/mt19937.cpp @@ -0,0 +1,102 @@ +#include "mt19937.h" + +namespace mars::rng { + +namespace { +constexpr uint32_t kMatrixA = 0x9908b0dfu; +constexpr uint32_t kUpper = 0x80000000u; +constexpr uint32_t kLower = 0x7fffffffu; + +inline uint32_t temper(uint32_t y) { + y ^= y >> 11; + y ^= (y << 7) & 0x9d2c5680u; + y ^= (y << 15) & 0xefc60000u; + y ^= y >> 18; + return y; +} +} // namespace + +void MT19937::seed(uint32_t s) { + mt_[0] = s; + for (int i = 1; i < N; ++i) + mt_[i] = 1812433253u * (mt_[i - 1] ^ (mt_[i - 1] >> 30)) + uint32_t(i); + twist(); +} + +void MT19937::twist() { + int kk = 0; + for (; kk < N - M; ++kk) { + uint32_t y = (mt_[kk] & kUpper) | (mt_[kk + 1] & kLower); + mt_[kk] = mt_[kk + M] ^ (y >> 1) ^ ((y & 1u) ? kMatrixA : 0u); + } + for (; kk < N - 1; ++kk) { + uint32_t y = (mt_[kk] & kUpper) | (mt_[kk + 1] & kLower); + mt_[kk] = mt_[kk + (M - N)] ^ (y >> 1) ^ ((y & 1u) ? kMatrixA : 0u); + } + uint32_t y = (mt_[N - 1] & kUpper) | (mt_[0] & kLower); + mt_[N - 1] = mt_[M - 1] ^ (y >> 1) ^ ((y & 1u) ? kMatrixA : 0u); + left_ = N; +} + +uint32_t MT19937::next_u32() { + if (left_ <= 0) twist(); + uint32_t y = mt_[N - left_]; + --left_; + return temper(y); +} + +float MT19937::next_float() { + return static_cast(static_cast(next_u32()) * (1.0 / 4294967296.0)); +} + +uint32_t MT19937::next_int(uint32_t n) { + if (n <= 1) return 0; + uint32_t mask = n - 1; + mask |= mask >> 1; + mask |= mask >> 2; + mask |= mask >> 4; + mask |= mask >> 8; + mask |= mask >> 16; + for (;;) { + uint32_t r = next_u32() & mask; + if (r < n) return r; + } +} + +void MT19937::load_state(const uint32_t mt[N], int left) { + for (int i = 0; i < N; ++i) mt_[i] = mt[i]; + left_ = left; +} + +bool MT19937::load_state(const uint8_t* blob, size_t n) { + if (n < kStateBytes) return false; + uint32_t tmp[N]; + for (int i = 0; i < N; ++i) { + const uint8_t* p = blob + size_t(i) * 4; + tmp[i] = uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24); + } + const uint8_t* p = blob + size_t(N) * 4; + int32_t left = int32_t(uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24)); + if (left < 0 || left > N) return false; + load_state(tmp, left); + return true; +} + +void MT19937::save_state(uint8_t out[kStateBytes]) const { + for (int i = 0; i < N; ++i) { + uint32_t v = mt_[i]; + uint8_t* p = out + size_t(i) * 4; + p[0] = uint8_t(v); + p[1] = uint8_t(v >> 8); + p[2] = uint8_t(v >> 16); + p[3] = uint8_t(v >> 24); + } + uint32_t v = uint32_t(left_); + uint8_t* p = out + size_t(N) * 4; + p[0] = uint8_t(v); + p[1] = uint8_t(v >> 8); + p[2] = uint8_t(v >> 16); + p[3] = uint8_t(v >> 24); +} + +} // namespace mars::rng diff --git a/src/mars/rng/mt19937.h b/src/mars/rng/mt19937.h new file mode 100644 index 0000000..61bcb0a --- /dev/null +++ b/src/mars/rng/mt19937.h @@ -0,0 +1,64 @@ +// mars::rng — the engine's PRNG: a textbook 32-bit Mersenne Twister (MT19937). +// +// State model (matches the save-file blob, see docs/mars-rng.md): +// uint32_t mt[624] the untempered state block +// int left words still unread in the current block; the next +// word to hand out is mt[624 - left] +// The serialized form is exactly mt[624] followed by left as an int32 +// (0x9c4 = 2500 bytes). A fresh generator seeds the block with the standard +// Knuth-style initializer and twists once immediately, so left == 624 right +// after seeding. +#pragma once + +#include +#include + +namespace mars::rng { + +class MT19937 { +public: + static constexpr int N = 624; + static constexpr int M = 397; + static constexpr size_t kStateBytes = size_t(N) * 4 + 4; // 0x9c4 + + explicit MT19937(uint32_t seed = 5489u) { this->seed(seed); } + + // mt[0] = seed; mt[i] = 1812433253 * (mt[i-1] ^ (mt[i-1] >> 30)) + i; then twist. + void seed(uint32_t s); + + // Next tempered 32-bit output. + uint32_t next_u32(); + + // Uniform float in [0, 1): (float)(next_u32() * 2^-32), the product formed + // in double precision then narrowed. NOTE: the narrowing can round the + // largest outputs (y >= 0xFFFFFF80) up to exactly 1.0f. Mapping recorded + // from the RE notes; still needs binary confirmation against a captured + // sequence (see docs/mars-rng.md). + float next_float(); + + // Uniform integer in [0, n) by rejection sampling with the smallest + // power-of-two mask covering n-1 (n == 0 returns 0). Mask/rejection + // details need binary confirmation. + uint32_t next_int(uint32_t n); + + // --- state access / serialization --------------------------------------- + const uint32_t* state() const { return mt_; } + int left() const { return left_; } + int index() const { return N - left_; } // words consumed in the current block + + // Load mt[624] + left from the save's RNG blob layout. + void load_state(const uint32_t mt[N], int left); + // Parse the 0x9c4-byte blob (little-endian). Returns false if n < kStateBytes + // or left is out of range. + bool load_state(const uint8_t* blob, size_t n); + // Serialize to the same 0x9c4-byte layout. + void save_state(uint8_t out[kStateBytes]) const; + +private: + uint32_t mt_[N]; + int left_ = 0; + + void twist(); +}; + +} // namespace mars::rng diff --git a/src/mars/stream/CMakeLists.txt b/src/mars/stream/CMakeLists.txt new file mode 100644 index 0000000..d81b24f --- /dev/null +++ b/src/mars/stream/CMakeLists.txt @@ -0,0 +1,22 @@ +# mars::stream — Streamable serialization (save-file format) + gzip container. +# Included from the root CMakeLists.txt via add_subdirectory(src/mars/stream). + +if(NOT TARGET miniz) # mars/vfs defines the same vendored target when added first + add_library(miniz STATIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party/miniz/miniz.c) + target_include_directories(miniz PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party/miniz) + target_compile_definitions(miniz PUBLIC MINIZ_NO_STDIO MINIZ_NO_ARCHIVE_APIS MINIZ_NO_TIME) + set_target_properties(miniz PROPERTIES LINKER_LANGUAGE C) +endif() + +add_library(mars_stream STATIC + gzip.cpp + reader.cpp + writer.cpp + dump.cpp + save.cpp) +target_include_directories(mars_stream PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../..) +target_link_libraries(mars_stream PUBLIC miniz) +target_compile_features(mars_stream PUBLIC cxx_std_17) + +add_executable(sots_savedump savedump_main.cpp) +target_link_libraries(sots_savedump PRIVATE mars_stream) diff --git a/src/mars/stream/archive.h b/src/mars/stream/archive.h new file mode 100644 index 0000000..3839510 --- /dev/null +++ b/src/mars/stream/archive.h @@ -0,0 +1,684 @@ +// mars::stream — archives that drive a typed shape's io() field list. +// +// A shape is a plain struct with +// static constexpr const char* kStreamName = "Sys"; // "" when the frame is unnamed ("." elements) +// template void io(Ar& ar) { ar.i32(A("Idx"), idx); ... } +// listing its fields in on-disk order. The same io() is driven by three +// archives: +// ReadArchive fills the struct from a generic Node tree (walker output), +// matching tags by name (A) or position (R), coercing kinds +// and reporting deviations as Issues; +// WriteArchive emits the struct through a Writer in the same order; +// SchemaBuilder records the field list as a Desc so the walker can type the +// frame's children before any shape is applied. +// +// Tag helpers: A("Turn") = on-disk tag confirmed, matched by name. +// R("treasury") = reference/positional field, written as "." (or +// the given disk tag); matched by position. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "node.h" +#include "schema.h" +#include "writer.h" + +namespace mars::stream { + +struct Tag { + const char* name; // schema name + const char* disk; // tag emitted when writing + bool positional; +}; +constexpr Tag A(const char* n) { return Tag{n, n, false}; } +constexpr Tag R(const char* n, const char* disk = ".") { return Tag{n, disk, true}; } + +struct Vec3 { + float x = 0, y = 0, z = 0; + bool operator==(const Vec3& o) const { return x == o.x && y == o.y && z == o.z; } +}; + +template +struct has_io : std::false_type {}; +template +struct has_io> : std::true_type {}; + +// --------------------------------------------------------------------------- +// ReadArchive +// --------------------------------------------------------------------------- +class ReadArchive { +public: + static constexpr bool reading = true, writing = false, building = false; + + ReadArchive(const std::vector& nodes, std::vector& issues, std::string path) + : nodes_(&nodes), issues_(&issues), path_(std::move(path)) {} + + // --- scalars ------------------------------------------------------------ + void i32(Tag t, int32_t& v) { + if (const Node* n = take_named(t)) v = coerce_int(*n, fpath(t)); + } + void f32(Tag t, float& v) { + if (const Node* n = take_named(t)) v = coerce_float(*n, fpath(t)); + } + void b(Tag t, bool& v) { + if (const Node* n = take_named(t)) v = coerce_bool(*n, fpath(t)); + } + void i64(Tag t, int64_t& v) { + if (const Node* n = take_named(t)) v = coerce_int64(*n, fpath(t)); + } + void str(Tag t, std::string& v) { + if (const Node* n = take_named(t)) v = coerce_string(*n, fpath(t)); + } + void vec3(Tag t, Vec3& v) { + if (const Node* n = take_named(t)) v = coerce_vec3(*n, fpath(t)); + } + // generic item kept as a node copy ("any") + void any(Tag t, Node& v) { + if (const Node* n = take_named(t)) v = *n; + } + void raw_frame(Tag t, Node& v) { any(t, v); } + + // --- optional named items (consumed only when the next tag matches) ------ + void opt_i32(Tag t, std::optional& v) { + if (next_is(t)) v = coerce_int(*take(), fpath(t)); + } + void opt_f32(Tag t, std::optional& v) { + if (next_is(t)) v = coerce_float(*take(), fpath(t)); + } + void opt_b(Tag t, std::optional& v) { + if (next_is(t)) v = coerce_bool(*take(), fpath(t)); + } + void opt_any(Tag t, std::optional& v) { + if (next_is(t)) v = *take(); + } + template + void opt_obj(Tag t, std::optional& v) { + if (next_is(t)) { + v.emplace(); + obj(t, *v); + } + } + + // --- framed struct --------------------------------------------------------- + template + void obj(Tag t, T& v) { + const Node* n = take_named(t); + if (!n) return; + read_frame(*n, v, fpath(t)); + } + // struct that may also appear inline (unframed); `framed` records which + template + void obj_flex(Tag t, T& v, bool& framed) { + const Node* n = peek(); + if (n && !n->is_complex()) { + framed = false; + v.io(*this); + return; + } + framed = true; + obj(t, v); + } + + // --- framed array: "." count + n x "." element ----------------------------- + template + void carr(Tag t, std::vector& v) { + const Node* n = take_named(t); + if (!n) return; + std::string p = fpath(t); + if (!n->is_complex()) { + issue(Issue::Error, p, n, "expected framed array, found " + std::string(kind_name(n->kind)) + " '" + + n->name + "'"); + return; + } + ReadArchive sub(n->children, *issues_, p); + v.clear(); + if (sub.done()) return; + int32_t count = coerce_int(*sub.take(), p); + size_t remaining = sub.remaining(); + if (count < 0 || size_t(count) > remaining) { + issue(Issue::Error, p, n, + "framed-array count " + std::to_string(count) + " exceeds " + std::to_string(remaining) + " item(s)"); + count = int32_t(std::min(size_t(std::max(count, 0)), remaining)); + } + for (int32_t i = 0; i < count; ++i) { + if (sub.done()) { + issue(Issue::Error, p, n, "framed array truncated: " + std::to_string(i) + " of " + std::to_string(count)); + break; + } + v.emplace_back(); + sub.read_elem(v.back(), p + "[" + std::to_string(i) + "]"); + } + if (!sub.done()) + issue(Issue::Warn, p, sub.peek(), + std::to_string(sub.remaining()) + " item(s) after framed array elements"); + } + template + void carr_flex(Tag t, std::vector& v, bool& framed) { + const Node* n = peek(); + if (n && !n->is_complex()) { + framed = false; + narr(t, v, [](ReadArchive& a, T& e) { a.read_elem(e, a.path_); }); + return; + } + framed = true; + carr(t, v); + } + + // --- inline array: named count + n x element ------------------------------- + template + void narr(Tag t, std::vector& v, F elem) { + const Node* cn = take_named(t); + if (!cn) return; + std::string p = fpath(t); + int32_t count = coerce_int(*cn, p); + size_t rem = remaining(); + v.clear(); + if (count < 0 || size_t(count) > rem) { + issue(Issue::Error, p, cn, + "array count " + std::to_string(count) + " exceeds the " + std::to_string(rem) + + " item(s) left in the frame"); + count = int32_t(std::min(size_t(std::max(count, 0)), rem)); + } + for (int32_t i = 0; i < count; ++i) { + if (done()) { + issue(Issue::Error, p, nullptr, "array truncated: " + std::to_string(i) + " of " + std::to_string(count) + + " elements present"); + break; + } + size_t before = i_; + v.emplace_back(); + std::string save = path_; + path_ = p + "[" + std::to_string(i) + "]"; + elem(*this, v.back()); + path_ = save; + if (i_ == before) { + issue(Issue::Error, p, peek(), "array element " + std::to_string(i) + " consumed nothing; stopping"); + v.pop_back(); + break; + } + } + } + + // --- conditional group ----------------------------------------------------- + template + void when(bool cond, F body) { + if (cond) body(*this); + } + + // --- uncounted repetition while the next tag is `lead` --------------------- + template + void repeat(const char* lead, std::vector& v, F elem) { + v.clear(); + while (!done() && peek()->tagged && peek()->name == lead) { + size_t before = i_; + v.emplace_back(); + std::string save = path_; + path_ = path_ + "/" + lead + "[" + std::to_string(v.size() - 1) + "]"; + elem(*this, v.back()); + path_ = save; + if (i_ == before) { + v.pop_back(); + break; + } + } + } + + // --- everything left in the frame, kept generic ---------------------------- + void rest(std::vector& v) { + v.assign(nodes_->begin() + long(i_), nodes_->end()); + i_ = nodes_->size(); + absorbed_ = true; + } + + // --- cursor ---------------------------------------------------------------- + bool done() const { return i_ >= nodes_->size(); } + size_t remaining() const { return nodes_->size() - i_; } + const Node* peek() const { return done() ? nullptr : &(*nodes_)[i_]; } + const Node* take() { return &(*nodes_)[i_++]; } + + // After io(): leftover items are unexpected unless rest() absorbed them. + void finish(const Node* frame) { + if (!done() && !absorbed_) { + issue(Issue::Warn, path_, peek(), std::to_string(remaining()) + " unexpected item(s) at end of frame"); + (void)frame; + i_ = nodes_->size(); + } + } + + template + void read_frame(const Node& n, T& v, const std::string& p) { + if (!n.is_complex()) { + issue(Issue::Error, p, &n, "expected frame, found " + std::string(kind_name(n.kind)) + " '" + n.name + "'"); + return; + } + ReadArchive sub(n.children, *issues_, p); + v.io(sub); + sub.finish(&n); + } + + // one array element: struct -> "." frame, int32 -> "." int, Node -> any + template + void read_elem(T& e, const std::string& p) { + const Node* n = take(); + if constexpr (std::is_same_v) e = coerce_int(*n, p); + else if constexpr (std::is_same_v) e = *n; + else read_frame(*n, e, p); + } + +private: + const std::vector* nodes_; + size_t i_ = 0; + std::vector* issues_; + std::string path_; + bool absorbed_ = false; + + std::string fpath(Tag t) const { return path_ + "/" + t.name; } + + void issue(Issue::Level l, const std::string& p, const Node* n, std::string msg) { + issues_->push_back(Issue{l, p, n ? n->offset : 0u, std::move(msg)}); + } + static std::string lower(std::string s) { + for (char& c : s) + if (c >= 'A' && c <= 'Z') c = char(c - 'A' + 'a'); + return s; + } + bool next_is(Tag t) const { + const Node* n = peek(); + return n && n->tagged && lower(n->name) == lower(t.name); + } + + const Node* take_named(Tag t) { + std::string p = fpath(t); + const Node* n = peek(); + if (!n) { + issue(t.positional ? Issue::Warn : Issue::Error, p, nullptr, "missing field '" + std::string(t.name) + "' (frame ended)"); + return nullptr; + } + if (!t.positional) { + if (!n->tagged || n->name != t.name) { + size_t j = i_; + for (; j < nodes_->size(); ++j) + if ((*nodes_)[j].tagged && (*nodes_)[j].name == t.name) break; + if (j == nodes_->size()) { + issue(Issue::Error, p, n, + "expected '" + std::string(t.name) + "', found '" + (n->tagged ? n->name : "") + + "'; field missing"); + return nullptr; + } + std::string names; + for (size_t k = i_; k < j && k < i_ + 6; ++k) names += (k > i_ ? ", '" : "'") + (*nodes_)[k].name + "'"; + issue(Issue::Warn, p, n, + std::to_string(j - i_) + " unexpected item(s) before '" + t.name + "': " + names); + i_ = j; + } + } else if (n->tagged && lower(n->name) != lower(t.name)) { + issue(Issue::Info, p, n, "tag '" + n->name + "' read positionally as '" + t.name + "'"); + } + return take(); + } + + // --- coercions (mirror the reference reader's rules) ---------------------- + int32_t coerce_int(const Node& n, const std::string& p) { + switch (n.kind) { + case Kind::Int: return n.as_int(); + case Kind::Float: return n.raw.size() == 4 ? n.as_int() : fail_int(n, p, "int"); + case Kind::Bool: return n.as_bool() ? 1 : 0; + case Kind::Int64: + issue(Issue::Warn, p, &n, "int expected, int64 read"); + return int32_t(n.as_int64()); + default: return fail_int(n, p, "int"); + } + } + float coerce_float(const Node& n, const std::string& p) { + switch (n.kind) { + case Kind::Float: return n.as_float(); + case Kind::Int: return n.raw.size() == 4 ? n.as_float() : float(fail_int(n, p, "float")); + case Kind::Bool: return n.as_bool() ? 1.f : 0.f; + default: return float(fail_int(n, p, "float")); + } + } + bool coerce_bool(const Node& n, const std::string& p) { + if (n.kind == Kind::Bool) return n.as_bool(); + if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4) { + if (n.raw[1] || n.raw[2] || n.raw[3] || n.raw[0] > 1) + issue(Issue::Warn, p, &n, "bool expected, word " + hex(n.raw.data(), 4) + " read"); + return n.raw[0] != 0; + } + return fail_int(n, p, "bool") != 0; + } + int64_t coerce_int64(const Node& n, const std::string& p) { + if (n.kind == Kind::Int64) return n.as_int64(); + if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4) { + issue(Issue::Warn, p, &n, "int64 expected, 4-byte word read (width mismatch)"); + return n.as_int(); + } + return fail_int(n, p, "int64"); + } + std::string coerce_string(const Node& n, const std::string& p) { + if (n.kind == Kind::String) return n.as_string(); + // an empty string is "len 0" = 4 zero bytes, byte-identical to int 0 + if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4 && n.as_int() == 0) return ""; + fail_int(n, p, "string"); + return ""; + } + Vec3 coerce_vec3(const Node& n, const std::string& p) { + Vec3 v; + if (n.is_complex()) { + const auto& ch = n.children; + if (ch.size() == 1 && ch[0].kind == Kind::Raw && ch[0].raw.size() == 12) { + v.x = rd_f32(ch[0].raw.data()); + v.y = rd_f32(ch[0].raw.data() + 4); + v.z = rd_f32(ch[0].raw.data() + 8); + return v; + } + if (ch.size() == 3 && !ch[0].is_complex() && !ch[1].is_complex() && !ch[2].is_complex() && + ch[0].raw.size() == 4 && ch[1].raw.size() == 4 && ch[2].raw.size() == 4) { + v.x = ch[0].as_float(); + v.y = ch[1].as_float(); + v.z = ch[2].as_float(); + return v; + } + issue(Issue::Error, p, &n, "vec3 frame has unexpected body (" + std::to_string(ch.size()) + " items)"); + return v; + } + if (n.kind == Kind::Raw && n.raw.size() == 12) { + v.x = rd_f32(n.raw.data()); + v.y = rd_f32(n.raw.data() + 4); + v.z = rd_f32(n.raw.data() + 8); + return v; + } + issue(Issue::Error, p, &n, "expected vec3, found " + std::string(kind_name(n.kind))); + return v; + } + int32_t fail_int(const Node& n, const std::string& p, const char* want) { + issue(Issue::Error, p, &n, + std::string("expected ") + want + ", read " + (n.is_complex() ? "frame '" + n.name + "'" : kind_name(n.kind))); + return n.raw.size() >= 4 ? n.as_int() : 0; + } +}; + +// --------------------------------------------------------------------------- +// WriteArchive +// --------------------------------------------------------------------------- +class WriteArchive { +public: + static constexpr bool reading = false, writing = true, building = false; + + explicit WriteArchive(Writer& w) : w_(w) {} + + void i32(Tag t, int32_t& v) { w_.int32(t.disk, v); } + void f32(Tag t, float& v) { w_.float32(t.disk, v); } + void b(Tag t, bool& v) { w_.boolean(t.disk, v); } + void i64(Tag t, int64_t& v) { w_.int64(t.disk, v); } + void str(Tag t, std::string& v) { w_.string(t.disk, v); } + void vec3(Tag t, Vec3& v) { + w_.begin(t.disk); + w_.float32(".", v.x); + w_.float32(".", v.y); + w_.float32(".", v.z); + w_.end(); + } + void any(Tag, Node& v) { w_.node(v); } + void raw_frame(Tag t, Node& v) { any(t, v); } + + void opt_i32(Tag t, std::optional& v) { + if (v) w_.int32(t.disk, *v); + } + void opt_f32(Tag t, std::optional& v) { + if (v) w_.float32(t.disk, *v); + } + void opt_b(Tag t, std::optional& v) { + if (v) w_.boolean(t.disk, *v); + } + void opt_any(Tag, std::optional& v) { + if (v) w_.node(*v); + } + template + void opt_obj(Tag t, std::optional& v) { + if (v) obj(t, *v); + } + + template + void obj(Tag t, T& v) { + w_.begin(t.disk); + v.io(*this); + w_.end(); + } + template + void obj_flex(Tag t, T& v, bool& framed) { + if (framed) obj(t, v); + else v.io(*this); + } + template + void carr(Tag t, std::vector& v) { + w_.begin(t.disk); + w_.int32(".", int32_t(v.size())); + for (T& e : v) write_elem(e); + w_.end(); + } + template + void carr_flex(Tag t, std::vector& v, bool& framed) { + if (framed) { + carr(t, v); + return; + } + w_.int32(t.disk, int32_t(v.size())); + for (T& e : v) write_elem(e); + } + template + void narr(Tag t, std::vector& v, F elem) { + w_.int32(t.disk, int32_t(v.size())); + for (T& e : v) elem(*this, e); + } + template + void when(bool cond, F body) { + if (cond) body(*this); + } + template + void repeat(const char*, std::vector& v, F elem) { + for (T& e : v) elem(*this, e); + } + void rest(std::vector& v) { + for (const Node& n : v) w_.node(n); + } + + template + void write_elem(T& e) { + if constexpr (std::is_same_v) w_.int32(".", e); + else if constexpr (std::is_same_v) w_.node(e); + else { + w_.begin("."); + e.io(*this); + w_.end(); + } + } + +private: + Writer& w_; +}; + +// --------------------------------------------------------------------------- +// SchemaBuilder — runs io() on default-constructed shapes to record hints. +// --------------------------------------------------------------------------- +class SchemaBuilder { +public: + static constexpr bool reading = false, writing = false, building = true; + + struct Context { + Registry& reg; + std::map> strong; // A/R fields: name -> kinds seen + std::map weak; // Opt fields: lowest priority + std::map memo; + std::set in_progress; + const Desc* raw_desc = nullptr; + explicit Context(Registry& r) : reg(r) {} + }; + + SchemaBuilder(Context& ctx, Desc* cur) : ctx_(ctx), cur_(cur) {} + + void i32(Tag t, int32_t&) { prim(t, Prim::Int); } + void f32(Tag t, float&) { prim(t, Prim::Float); } + void b(Tag t, bool&) { prim(t, Prim::Bool); } + void i64(Tag t, int64_t&) { prim(t, Prim::Int64); } + void str(Tag t, std::string&) { prim(t, Prim::String); } + void vec3(Tag t, Vec3&) { nohint(t); } + void any(Tag t, Node&) { nohint(t); } + void raw_frame(Tag t, Node&) { + nohint(t); + if (!ctx_.raw_desc) { + Desc d; + d.type = Desc::Raw; + d.name = t.name; + ctx_.raw_desc = ctx_.reg.add(std::move(d)); + } + ctx_.reg.shapes.emplace(t.name, ctx_.raw_desc); + } + + void opt_i32(Tag t, std::optional&) { prim(t, Prim::Int, true); } + void opt_f32(Tag t, std::optional&) { prim(t, Prim::Float, true); } + void opt_b(Tag t, std::optional&) { prim(t, Prim::Bool, true); } + void opt_any(Tag t, std::optional&) { + close_prefix(); + nohint(t); + } + template + void opt_obj(Tag t, std::optional&) { + close_prefix(); + const Desc* d = describe(); + by_name(t, Hint{Prim::None, d}); + register_shape(t, d); + } + + template + void obj(Tag t, T&) { + const Desc* d = describe(); + push_prefix(Hint{Prim::None, d}); + by_name(t, Hint{Prim::None, d}); + register_shape(t, d); + } + template + void obj_flex(Tag t, T& v, bool&) { + close_prefix(); + obj(t, v); + } + template + void carr(Tag t, std::vector&) { + Desc d; + d.type = Desc::CArr; + if constexpr (std::is_same_v) d.elem.kind = Prim::Int; + else if constexpr (std::is_same_v) d.elem = Hint{}; + else d.elem.sub = describe(); + const Desc* cd = ctx_.reg.add(std::move(d)); + push_prefix(Hint{Prim::None, cd}); + // the NULL-name tag "." is never a frame hint key: it carries ints, + // floats and frames alike (bare "." elements inside inline arrays) + if (std::string(t.name) != ".") { + by_name(t, Hint{Prim::None, cd}); + if (*t.name) ctx_.reg.shapes.emplace(t.name, cd); + } + } + template + void carr_flex(Tag t, std::vector& v, bool&) { + close_prefix(); + carr(t, v); + } + template + void narr(Tag t, std::vector&, F elem) { + push_prefix(Hint{Prim::Int, nullptr}); + close_prefix(); + by_name(t, Hint{}); + T tmp{}; + elem(*this, tmp); // element fields register by name (prefix already closed) + } + template + void when(bool, F body) { + close_prefix(); + body(*this); + } + template + void repeat(const char*, std::vector&, F elem) { + close_prefix(); + T tmp{}; + elem(*this, tmp); + } + void rest(std::vector&) { close_prefix(); } + + // Describe shape T (memoized per type) and register its named frame. + template + const Desc* describe() { + std::type_index ti(typeid(T)); + auto it = ctx_.memo.find(ti); + if (it != ctx_.memo.end()) return it->second; + if (ctx_.in_progress.count(ti)) return nullptr; // recursive shape: no positional hints + ctx_.in_progress.insert(ti); + Desc d; + d.type = Desc::Shape; + d.name = T::kStreamName; + Desc* nd = ctx_.reg.add(std::move(d)); + SchemaBuilder sub(ctx_, nd); + T tmp{}; + tmp.io(sub); + ctx_.in_progress.erase(ti); + ctx_.memo.emplace(ti, nd); + if (!nd->name.empty()) ctx_.reg.shapes.emplace(nd->name, nd); + return nd; + } + + // Build the global catalog after every shape has been visited. + static void finalize(Context& ctx, const std::map& manual_kinds) { + std::set conflicts; + for (auto& [name, kinds] : ctx.strong) { + if (kinds.size() == 1) ctx.reg.kinds.emplace(name, *kinds.begin()); + else conflicts.insert(name); + } + for (auto& [name, k] : ctx.weak) + if (!conflicts.count(name)) ctx.reg.kinds.emplace(name, k); + ctx.reg.kinds.erase("."); // the NULL-name tag carries ints, floats and frames alike + for (auto& [name, k] : manual_kinds) ctx.reg.kinds[name] = k; + } + +private: + Context& ctx_; + Desc* cur_; + bool prefix_open_ = true; + + static std::string lower(std::string s) { + for (char& c : s) + if (c >= 'A' && c <= 'Z') c = char(c - 'A' + 'a'); + return s; + } + void push_prefix(Hint h) { + if (cur_ && prefix_open_) cur_->prefix.push_back(h); + } + void close_prefix() { prefix_open_ = false; } + void by_name(Tag t, Hint h) { + if (!cur_) return; + cur_->by_name.emplace(t.name, h); + cur_->by_name.emplace(lower(t.name), h); + } + void prim(Tag t, Prim k, bool opt = false) { + if (opt) close_prefix(); + else push_prefix(Hint{k, nullptr}); + by_name(t, Hint{k, nullptr}); + if (opt) ctx_.weak.emplace(t.name, k); + else ctx_.strong[t.name].insert(k); + } + void nohint(Tag t) { + push_prefix(Hint{}); + by_name(t, Hint{}); + } + void register_shape(Tag t, const Desc* d) { + if (!d) return; + if (d->name.empty() && *t.name) ctx_.reg.shapes.emplace(t.name, d); + } +}; + +} // namespace mars::stream diff --git a/src/mars/stream/bytes.h b/src/mars/stream/bytes.h new file mode 100644 index 0000000..dc599a6 --- /dev/null +++ b/src/mars/stream/bytes.h @@ -0,0 +1,75 @@ +// mars::stream — byte-level helpers shared by the reader and writer. +// Everything in the Streamable format is little-endian; these helpers do the +// byte shuffling explicitly so the code is host-endian independent. +#pragma once + +#include +#include +#include +#include + +namespace mars::stream { + +using Bytes = std::vector; + +constexpr uint32_t kBeginMark = 0xBEEFBEEFu; // opens a complex (framed) value +constexpr uint32_t kEndMark = 0x41104110u; // closes it (= ~kBeginMark) + +inline uint32_t pad4(uint32_t n) { return (n + 3u) & ~3u; } + +inline uint32_t rd_u32(const uint8_t* p) { + return uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24); +} +inline int32_t rd_i32(const uint8_t* p) { return int32_t(rd_u32(p)); } +inline uint64_t rd_u64(const uint8_t* p) { return uint64_t(rd_u32(p)) | (uint64_t(rd_u32(p + 4)) << 32); } +inline int64_t rd_i64(const uint8_t* p) { return int64_t(rd_u64(p)); } +inline float rd_f32(const uint8_t* p) { + uint32_t u = rd_u32(p); + float f; + std::memcpy(&f, &u, 4); + return f; +} +inline uint32_t f32_bits(float f) { + uint32_t u; + std::memcpy(&u, &f, 4); + return u; +} +inline float bits_f32(uint32_t u) { + float f; + std::memcpy(&f, &u, 4); + return f; +} + +inline void put_u32(Bytes& b, uint32_t v) { + b.push_back(uint8_t(v)); + b.push_back(uint8_t(v >> 8)); + b.push_back(uint8_t(v >> 16)); + b.push_back(uint8_t(v >> 24)); +} +inline void put_i32(Bytes& b, int32_t v) { put_u32(b, uint32_t(v)); } +inline void put_u64(Bytes& b, uint64_t v) { + put_u32(b, uint32_t(v)); + put_u32(b, uint32_t(v >> 32)); +} +inline void put_f32(Bytes& b, float f) { put_u32(b, f32_bits(f)); } +inline void put_bytes(Bytes& b, const void* p, size_t n) { + const uint8_t* s = static_cast(p); + b.insert(b.end(), s, s + n); +} +inline void put_pad(Bytes& b, size_t to_boundary_of_item_start) { + // append NUL bytes until b.size() - start is a multiple of 4 + while ((b.size() - to_boundary_of_item_start) & 3u) b.push_back(0); +} + +inline std::string hex(const uint8_t* p, size_t n) { + static const char* d = "0123456789abcdef"; + std::string s; + s.reserve(n * 2); + for (size_t i = 0; i < n; ++i) { + s.push_back(d[p[i] >> 4]); + s.push_back(d[p[i] & 15]); + } + return s; +} + +} // namespace mars::stream diff --git a/src/mars/stream/dump.cpp b/src/mars/stream/dump.cpp new file mode 100644 index 0000000..5520758 --- /dev/null +++ b/src/mars/stream/dump.cpp @@ -0,0 +1,147 @@ +#include "dump.h" + +#include +#include +#include +#include + +namespace mars::stream { + +std::string py_float_repr(double v) { + if (std::isnan(v)) return "nan"; + if (std::isinf(v)) return v < 0 ? "-inf" : "inf"; + if (v == 0.0) return std::signbit(v) ? "-0.0" : "0.0"; + // shortest digit string that round-trips + char buf[64]; + int prec = 1; + for (; prec <= 17; ++prec) { + std::snprintf(buf, sizeof buf, "%.*e", prec - 1, v); + if (std::strtod(buf, nullptr) == v) break; + } + // buf: [-]d[.ddd]e[+-]XX + std::string s(buf); + bool neg = s[0] == '-'; + if (neg) s.erase(0, 1); + size_t epos = s.find('e'); + int exp10 = std::atoi(s.c_str() + epos + 1); + std::string digits; + for (size_t i = 0; i < epos; ++i) + if (s[i] != '.') digits.push_back(s[i]); + while (digits.size() > 1 && digits.back() == '0') digits.pop_back(); + int decpt = exp10 + 1; // value = 0.digits * 10^decpt + std::string out = neg ? "-" : ""; + if (decpt > -4 && decpt <= 16) { + int nd = int(digits.size()); + if (decpt <= 0) { + out += "0."; + out.append(size_t(-decpt), '0'); + out += digits; + } else if (decpt >= nd) { + out += digits; + out.append(size_t(decpt - nd), '0'); + out += ".0"; + } else { + out += digits.substr(0, size_t(decpt)); + out += "."; + out += digits.substr(size_t(decpt)); + } + } else { + out += digits[0]; + if (digits.size() > 1) { + out += "."; + out += digits.substr(1); + } + char e[16]; + std::snprintf(e, sizeof e, "e%c%02d", exp10 < 0 ? '-' : '+', std::abs(exp10)); + out += e; + } + return out; +} + +namespace { +// windows-1252 0x80..0x9f -> Unicode (0 = undefined -> U+FFFD like errors="replace") +const uint16_t kCp1252High[32] = { + 0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, + 0x2039, 0x0152, 0, 0x017D, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, + 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E, 0x0178}; +} // namespace + +std::string json_quote_cp1252(const std::string& bytes) { + std::string o = "\""; + char tmp[8]; + for (unsigned char c : bytes) { + uint32_t u = c; + if (c >= 0x80 && c < 0xa0) u = kCp1252High[c - 0x80] ? kCp1252High[c - 0x80] : 0xFFFD; + switch (u) { + case '"': o += "\\\""; break; + case '\\': o += "\\\\"; break; + case '\n': o += "\\n"; break; + case '\r': o += "\\r"; break; + case '\t': o += "\\t"; break; + case '\b': o += "\\b"; break; + case '\f': o += "\\f"; break; + default: + if (u < 0x20 || u >= 0x7f) { + std::snprintf(tmp, sizeof tmp, "\\u%04x", u); + o += tmp; + } else { + o.push_back(char(u)); + } + } + } + o += "\""; + return o; +} + +namespace { + +void dump_into(const Node& node, int indent, std::vector& lines, int max_depth) { + std::string pad(size_t(indent) * 2, ' '); + char off[16]; + for (const Node& c : node.children) { + std::string nm = c.tagged ? c.name : ""; + std::snprintf(off, sizeof off, "@%08x ", c.offset); + if (c.is_complex()) { + lines.push_back(std::string(off) + pad + nm + " { # " + std::to_string(c.children.size()) + + " items, " + std::to_string(c.size) + " bytes"); + if (indent < max_depth) dump_into(c, indent + 1, lines, max_depth); + std::snprintf(off, sizeof off, "@%08x ", c.offset + c.size - 4); + lines.push_back(std::string(off) + pad + "}"); + } else if (c.kind == Kind::Raw) { + size_t n = c.raw.size(); + lines.push_back(std::string(off) + pad + nm + " raw[" + std::to_string(n) + "] " + + hex(c.raw.data(), n < 32 ? n : 32) + (n > 32 ? "..." : "")); + } else { + std::string val, alt; + switch (c.kind) { + case Kind::Int: + val = std::to_string(c.as_int()); + if (!c.hinted) { + float f = c.as_float(); + if (std::isfinite(f)) alt = " (alt " + py_float_repr(f) + ")"; + } + break; + case Kind::Float: + val = py_float_repr(c.as_float()); + if (!c.hinted) alt = " (alt " + std::to_string(c.as_int()) + ")"; + break; + case Kind::Bool: val = c.as_bool() ? "True" : "False"; break; + case Kind::Int64: val = std::to_string(c.as_int64()); break; + case Kind::String: val = json_quote_cp1252(c.as_string()); break; + default: break; + } + lines.push_back(std::string(off) + pad + nm + " " + kind_name(c.kind) + (c.hinted ? "" : "?") + " " + + val + alt); + } + } +} + +} // namespace + +std::vector dump_tree(const Node& root, int max_depth) { + std::vector lines; + dump_into(root, 0, lines, max_depth); + return lines; +} + +} // namespace mars::stream diff --git a/src/mars/stream/dump.h b/src/mars/stream/dump.h new file mode 100644 index 0000000..ab116de --- /dev/null +++ b/src/mars/stream/dump.h @@ -0,0 +1,23 @@ +// mars::stream — text dump of a generic tree, one line per item: +// @ [?] [ (alt )] +// `?` marks a guessed kind; frames print `{ ... }` with item count and size. +// The format matches the reference reader's --dump output line for line so +// the two can be diffed (numbers use Python's float repr, strings JSON quoting). +#pragma once + +#include +#include + +#include "node.h" + +namespace mars::stream { + +// Python-compatible shortest round-trip repr of a double ("240.0", "1e-05", "3.4028234663852886e+38"). +std::string py_float_repr(double v); + +// JSON string literal for windows-1252 bytes (non-ASCII escaped as \uXXXX, like json.dumps). +std::string json_quote_cp1252(const std::string& bytes); + +std::vector dump_tree(const Node& root, int max_depth = 999); + +} // namespace mars::stream diff --git a/src/mars/stream/gzip.cpp b/src/mars/stream/gzip.cpp new file mode 100644 index 0000000..04aa722 --- /dev/null +++ b/src/mars/stream/gzip.cpp @@ -0,0 +1,101 @@ +#include "gzip.h" + +#include + +#include "../../../third_party/miniz/miniz.h" + +namespace mars::stream { + +namespace { + +// gzip member: 10-byte header (+ optional fields), raw deflate, CRC32, ISIZE. +size_t gzip_header_len(const uint8_t* p, size_t n) { + if (n < 10 || p[0] != 0x1f || p[1] != 0x8b || p[2] != 8) throw GzipError("not a gzip/deflate stream"); + uint8_t flg = p[3]; + size_t pos = 10; + if (flg & 4) { // FEXTRA + if (pos + 2 > n) throw GzipError("truncated gzip header"); + size_t xlen = p[pos] | (p[pos + 1] << 8); + pos += 2 + xlen; + } + if (flg & 8) { // FNAME + while (pos < n && p[pos]) ++pos; + ++pos; + } + if (flg & 16) { // FCOMMENT + while (pos < n && p[pos]) ++pos; + ++pos; + } + if (flg & 2) pos += 2; // FHCRC + if (pos > n) throw GzipError("truncated gzip header"); + return pos; +} + +} // namespace + +Bytes gunzip(const uint8_t* p, size_t n) { + size_t hdr = gzip_header_len(p, n); + if (n < hdr + 8) throw GzipError("truncated gzip member"); + uint32_t isize = rd_u32(p + n - 4); + uint32_t crc_expect = rd_u32(p + n - 8); + + Bytes out; + out.resize(isize ? isize : 1); + mz_stream zs; + std::memset(&zs, 0, sizeof zs); + if (mz_inflateInit2(&zs, -MZ_DEFAULT_WINDOW_BITS) != MZ_OK) throw GzipError("inflateInit failed"); + zs.next_in = p + hdr; + zs.avail_in = static_cast(n - hdr - 8); + size_t produced = 0; + int rc; + for (;;) { + if (produced == out.size()) out.resize(out.size() * 2); + zs.next_out = out.data() + produced; + zs.avail_out = static_cast(out.size() - produced); + rc = mz_inflate(&zs, MZ_NO_FLUSH); + produced = out.size() - zs.avail_out; + if (rc == MZ_STREAM_END) break; + if (rc != MZ_OK) { + mz_inflateEnd(&zs); + throw GzipError(std::string("gzip container is damaged: ") + mz_error(rc)); + } + } + mz_inflateEnd(&zs); + out.resize(produced); + uint32_t crc = static_cast(mz_crc32(MZ_CRC32_INIT, out.data(), out.size())); + if (crc != crc_expect) throw GzipError("gzip CRC mismatch"); + if (isize != static_cast(out.size())) throw GzipError("gzip ISIZE mismatch"); + return out; +} + +Bytes inflate_container(const uint8_t* p, size_t n) { + if (is_gzip(p, n)) return gunzip(p, n); + return Bytes(p, p + n); +} + +Bytes gzip(const uint8_t* p, size_t n) { + Bytes out = {0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 0x0b}; // header, OS = NTFS (like the game's files) + mz_stream zs; + std::memset(&zs, 0, sizeof zs); + if (mz_deflateInit2(&zs, 6, MZ_DEFLATED, -MZ_DEFAULT_WINDOW_BITS, 9, MZ_DEFAULT_STRATEGY) != MZ_OK) + throw GzipError("deflateInit failed"); + size_t bound = mz_deflateBound(&zs, static_cast(n)); + size_t start = out.size(); + out.resize(start + bound); + zs.next_in = p; + zs.avail_in = static_cast(n); + zs.next_out = out.data() + start; + zs.avail_out = static_cast(bound); + int rc = mz_deflate(&zs, MZ_FINISH); + if (rc != MZ_STREAM_END) { + mz_deflateEnd(&zs); + throw GzipError("deflate failed"); + } + out.resize(start + bound - zs.avail_out); + mz_deflateEnd(&zs); + put_u32(out, static_cast(mz_crc32(MZ_CRC32_INIT, p, n))); + put_u32(out, static_cast(n)); + return out; +} + +} // namespace mars::stream diff --git a/src/mars/stream/gzip.h b/src/mars/stream/gzip.h new file mode 100644 index 0000000..404686d --- /dev/null +++ b/src/mars/stream/gzip.h @@ -0,0 +1,25 @@ +// mars::stream — gzip container (a .sav is one gzip member around the stream). +#pragma once + +#include + +#include "bytes.h" + +namespace mars::stream { + +struct GzipError : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +inline bool is_gzip(const uint8_t* p, size_t n) { return n >= 2 && p[0] == 0x1f && p[1] == 0x8b; } + +// Inflate one gzip member. Throws GzipError on a damaged container. +Bytes gunzip(const uint8_t* p, size_t n); + +// Data without the gzip magic is returned unchanged (already-inflated stream). +Bytes inflate_container(const uint8_t* p, size_t n); + +// Wrap `data` in a gzip member (deflate, level 6, no name/mtime). +Bytes gzip(const uint8_t* p, size_t n); + +} // namespace mars::stream diff --git a/src/mars/stream/node.h b/src/mars/stream/node.h new file mode 100644 index 0000000..e92585d --- /dev/null +++ b/src/mars/stream/node.h @@ -0,0 +1,120 @@ +// mars::stream — the generic tree produced by the walker. +// +// A save stream is a flat sequence of *items*. Every item starts with a name +// tag ([int32 len][bytes]; the game writes NULL names as ".") followed by the +// value; the whole item is NUL-padded to a 4-byte boundary ("joint" padding). +// Complex values are framed by 0xBEEFBEEF ... 0x41104110 and nest. Values +// carry no type byte, so a Node records the kind the reader decided on, the +// raw value bytes (always exact) and whether that kind came from a hint or a +// guess. +#pragma once + +#include +#include +#include + +#include "bytes.h" + +namespace mars::stream { + +enum class Kind : uint8_t { Int, Float, Bool, Int64, String, Complex, Raw }; + +inline const char* kind_name(Kind k) { + switch (k) { + case Kind::Int: return "int"; + case Kind::Float: return "float"; + case Kind::Bool: return "bool"; + case Kind::Int64: return "int64"; + case Kind::String: return "string"; + case Kind::Complex: return "complex"; + case Kind::Raw: return "raw"; + } + return "?"; +} + +struct Node { + std::string name; // tag text ("" for an empty tag) + bool tagged = true; // false: tagless frame / unnamed raw payload + Kind kind = Kind::Complex; + Bytes raw; // scalar value bytes; string payload (no length); raw blob + bool hinted = false; // kind came from the schema / catalog, not a guess + uint32_t offset = 0; // inflated offset of the item's tag (or frame marker) + uint32_t size = 0; // bytes consumed incl. tag, padding and frame markers + std::vector children; + + bool is_complex() const { return kind == Kind::Complex; } + int32_t as_int() const { return raw.size() >= 4 ? rd_i32(raw.data()) : 0; } + float as_float() const { return raw.size() >= 4 ? rd_f32(raw.data()) : 0.f; } + bool as_bool() const { return !raw.empty() && raw[0] != 0; } + int64_t as_int64() const { return raw.size() >= 8 ? rd_i64(raw.data()) : 0; } + // string payload as stored (windows-1252 bytes, no conversion) + std::string as_string() const { return std::string(raw.begin(), raw.end()); } + + // --- constructors for building trees by hand ------------------------- + static Node int32(std::string name, int32_t v) { + Node n = scalar(std::move(name), Kind::Int); + put_i32(n.raw, v); + return n; + } + static Node float32(std::string name, float v) { + Node n = scalar(std::move(name), Kind::Float); + put_f32(n.raw, v); + return n; + } + static Node boolean(std::string name, bool v) { + Node n = scalar(std::move(name), Kind::Bool); + n.raw.push_back(v ? 1 : 0); + return n; + } + static Node int64(std::string name, int64_t v) { + Node n = scalar(std::move(name), Kind::Int64); + put_u64(n.raw, uint64_t(v)); + return n; + } + static Node string(std::string name, const std::string& v) { + Node n = scalar(std::move(name), Kind::String); + n.raw.assign(v.begin(), v.end()); + return n; + } + static Node rawbytes(std::string name, Bytes v) { + Node n = scalar(std::move(name), Kind::Raw); + n.raw = std::move(v); + return n; + } + static Node frame(std::string name, std::vector kids = {}) { + Node n; + n.name = std::move(name); + n.kind = Kind::Complex; + n.children = std::move(kids); + n.hinted = true; + return n; + } + +private: + static Node scalar(std::string name, Kind k) { + Node n; + n.name = std::move(name); + n.kind = k; + n.hinted = true; + return n; + } +}; + +struct Issue { + enum Level { Info, Warn, Error }; + Level level; + std::string path; + uint32_t offset; + std::string msg; +}; + +inline const char* level_name(Issue::Level l) { + return l == Issue::Info ? "info" : l == Issue::Warn ? "warn" : "error"; +} + +struct Stats { + uint32_t items = 0, frames = 0, resyncs = 0, hint_failures = 0, guessed = 0, raw_bytes = 0, + best_effort = 0; +}; + +} // namespace mars::stream diff --git a/src/mars/stream/reader.cpp b/src/mars/stream/reader.cpp new file mode 100644 index 0000000..5e334ac --- /dev/null +++ b/src/mars/stream/reader.cpp @@ -0,0 +1,432 @@ +#include "reader.h" + +#include +#include + +namespace mars::stream { + +namespace { + +bool cp1252_defined(uint8_t c) { + return !(c == 0x81 || c == 0x8d || c == 0x8f || c == 0x90 || c == 0x9d); +} + +// Guard used only while GUESSING that an unknown tag holds a string: every +// byte windows-1252 defines counts as text (system names carry 0x92). +bool text_plausible(const uint8_t* b, size_t n) { + if (n == 0) return true; + size_t ok = 0; + for (size_t i = 0; i < n; ++i) { + uint8_t c = b[i]; + if ((c >= 0x20 && c < 0x7f) || (c >= 0x80 && cp1252_defined(c)) || c == 9 || c == 10 || c == 13) ++ok; + } + return ok * 10 >= n * 9; +} + +// Unknown 4-byte word -> int or float by bit pattern. +Prim classify_word(const uint8_t* raw) { + int32_t i = rd_i32(raw); + double f = rd_f32(raw); + if (i >= -kWordIntAbs && i <= kWordIntAbs) return Prim::Int; + if (std::isfinite(f) && std::fabs(f) >= 1e-6 && std::fabs(f) < 1e12) return Prim::Float; + return Prim::Int; +} + +Kind prim_to_kind(Prim p) { + switch (p) { + case Prim::Int: return Kind::Int; + case Prim::Float: return Kind::Float; + case Prim::Bool: return Kind::Bool; + case Prim::Int64: return Kind::Int64; + case Prim::String: return Kind::String; + case Prim::Raw: return Kind::Raw; + default: return Kind::Int; + } +} + +std::string lower(std::string s) { + for (char& c : s) + if (c >= 'A' && c <= 'Z') c = char(c - 'A' + 'a'); + return s; +} + +} // namespace + +Walker::Walker(const uint8_t* data, size_t n, const Registry* reg) + : d_(data), n_(static_cast(n)), reg_(reg) {} + +void Walker::issue(Issue::Level lvl, uint32_t off, std::string msg, const std::string& path) { + issues.push_back(Issue{lvl, path, off, std::move(msg)}); +} + +bool Walker::zero(uint32_t a, uint32_t b) const { + for (uint32_t i = a; i < b; ++i) + if (d_[i]) return false; + return true; +} + +Walker::TagAt Walker::tag_at(uint32_t p, bool allow_empty) const { + TagAt t; + if (p + 4 > n_) return t; + int32_t ln = rd_i32(d_ + p); + if (ln == 0) { + if (!allow_empty) return t; + t.ok = true; + t.end = p + 4; + return t; + } + if (ln < 0 || ln > kMaxTagLen || uint64_t(p) + 4 + uint32_t(ln) > n_) return t; + const uint8_t* b = d_ + p + 4; + for (int i = 0; i < ln; ++i) + if (b[i] < 0x20 || b[i] >= 0x7f) return t; + t.ok = true; + t.name.assign(reinterpret_cast(b), size_t(ln)); + t.end = p + 4 + uint32_t(ln); + return t; +} + +// Read a candidate scalar at the value position (joint padding: right after the +// name). known=true means the kind comes from the schema: a string value is +// then accepted whatever bytes it holds (only tags must be ASCII). +Walker::Scalar Walker::try_scalar(uint32_t tag_start, uint32_t name_end, Cand kind, bool known) { + Scalar r; + uint32_t vp = name_end; + uint32_t size; + if (kind == Cand::String) { + if (vp + 4 > n_) { + eof_limited_ = true; + return r; + } + int32_t ln = rd_i32(d_ + vp); + if (ln < 0 || ln > kMaxStringLen) return r; + if (uint64_t(vp) + 4 + uint32_t(ln) > n_) { + eof_limited_ = true; + return r; + } + if (!known && !text_plausible(d_ + vp + 4, size_t(ln))) return r; + size = 4 + uint32_t(ln); + } else { + size = kind == Cand::Word ? 4 : kind == Cand::Bool ? 1 : 8; + if (uint64_t(vp) + size > n_) { + eof_limited_ = true; + return r; + } + if (kind == Cand::Bool && d_[vp] > 1) return r; + } + uint32_t end = tag_start + pad4(name_end - tag_start + size); + if (end > n_) { + eof_limited_ = true; + return r; + } + if (!zero(vp + size, end)) return r; + r.ok = true; + r.cand = kind; + r.vp = d_ + vp; + r.vsize = size; + r.end = end; + return r; +} + +// Does an item plausibly start at p? Looks `depth` items ahead. +bool Walker::plausible_item(uint32_t p, int depth, bool allow_empty) { + if (p == n_) return true; + if (p + 4 > n_) { + eof_limited_ = true; + return false; + } + uint32_t w = u32(p); + if (w == kEndMark || w == kBeginMark) return true; + TagAt t = tag_at(p, allow_empty); + if (!t.ok) return false; + uint32_t a = pad4(t.end); + if (a + 4 <= n_ && u32(a) == kBeginMark && zero(t.end, a)) return true; + if (depth <= 0) return true; + // a tag the catalog knows to be a string is read as one without any text + // test, so a non-ASCII value can never veto the layout of the item before it + if (reg_ && reg_->kind(t.name) == Prim::String) { + Scalar r = try_scalar(p, t.end, Cand::String, true); + if (r.ok && plausible_item(r.end, depth - 1, allow_empty)) return true; + } + static const Cand order[] = {Cand::Word, Cand::Bool, Cand::String, Cand::Int64}; + for (Cand c : order) { + Scalar r = try_scalar(p, t.end, c, false); + if (!r.ok) continue; + if (plausible_item(r.end, depth - 1, allow_empty)) return true; + } + return false; +} + +// Scan forward for the next END marker or plausible tag. +std::pair Walker::resync(uint32_t p, int depth) { + uint32_t q = p + 1; + while (q + 4 <= n_) { + uint32_t w = u32(q); + if (w == kEndMark && depth > 0) return {"end", q}; + if (w == kBeginMark) return {"begin", q}; + if (tag_at(q, false).ok && plausible_item(q, 2, false)) return {"tag", q}; + ++q; + } + return {"eof", n_}; +} + +Hint Walker::child_hint(const Desc* frame, size_t index, const std::string* name) const { + Hint h; + if (frame) { + if (frame->type == Desc::Shape) { + if (index < frame->prefix.size()) h = frame->prefix[index]; + if (h.empty() && name && !name->empty()) { + auto it = frame->by_name.find(*name); + if (it == frame->by_name.end()) it = frame->by_name.find(lower(*name)); + if (it != frame->by_name.end()) h = it->second; + } + } else if (frame->type == Desc::CArr) { + if (index == 0) h.kind = Prim::Int; + else h = frame->elem; + } else if (frame->type == Desc::Raw) { + h.kind = Prim::Raw; + } + } + if (h.empty() && name && !name->empty() && reg_) { + // global catalog: exact case only (case-folding was found to mislabel + // unknown names, e.g. 'a' vs star-colour 'A') + h.kind = reg_->kind(*name); + h.sub = reg_->shape(*name); + } + return h; +} + +Node Walker::walk() { + uint32_t p = 0; + Node root; + root.tagged = false; + root.kind = Kind::Complex; + root.children = walk_frame(p, false, "", 0, nullptr, ""); + root.size = p; + return root; +} + +// Read items until the frame's END marker (or EOF). p is left after the END. +std::vector Walker::walk_frame(uint32_t& p, bool has_ctx, const std::string& ctx, int depth, + const Desc* shape, const std::string& path) { + std::vector children; + size_t index = 0; + if (depth > 0) ++stats.frames; + std::string ctxr = has_ctx ? "'" + ctx + "'" : "None"; + for (;;) { + if (p >= n_) { + if (depth > 0) issue(Issue::Error, p, "frame " + ctxr + " not terminated before EOF", path); + return children; + } + if (p + 4 > n_) { + children.push_back(raw_node(false, "", p, n_, "trailing bytes", Issue::Warn, path)); + if (depth > 0) issue(Issue::Error, n_, "frame " + ctxr + " not terminated before EOF", path); + p = n_; + return children; + } + uint32_t w = u32(p); + if (w == kEndMark) { + if (depth > 0) { + p += 4; + return children; + } + issue(Issue::Warn, p, "stray END marker at top level", path); + children.push_back(raw_node(false, "", p, p + 4, "stray END marker", Issue::Warn, path)); + p += 4; + continue; + } + children.push_back(read_item(p, index, depth, shape, path)); + ++index; + } +} + +Node Walker::read_item(uint32_t& p, size_t index, int depth, const Desc* frame, const std::string& path) { + ++stats.items; + uint32_t start = p; + uint32_t w = u32(p); + if (w == kBeginMark) { // tagless frame + Hint h = child_hint(frame, index, nullptr); + Node n; + n.tagged = false; + n.kind = Kind::Complex; + n.offset = start; + p += 4; + n.children = walk_frame(p, false, "", depth + 1, h.sub, path + "/<" + std::to_string(index) + ">"); + n.size = p - start; + return n; + } + TagAt t = tag_at(p, true); + if (!t.ok) return unreadable(p, depth, path); + const std::string& name = t.name; + uint32_t ne = t.end; + Hint h = child_hint(frame, index, &name); + uint32_t a = pad4(ne); + if (a + 4 <= n_ && u32(a) == kBeginMark && zero(ne, a)) { + Node n; + n.name = name; + n.kind = Kind::Complex; + n.offset = start; + n.hinted = h.sub != nullptr; + p = a + 4; + n.children = walk_frame(p, true, name, depth + 1, h.sub, path + "/" + name); + n.size = p - start; + return n; + } + + // scalar: hinted kind first, then the guess order + if (h.kind == Prim::Raw) { + uint32_t vp = ne; + uint32_t q = n_; + if (depth > 0) { + // the blob runs to the next END marker (any byte alignment, like bytes.find) + const uint8_t endb[4] = {0x10, 0x41, 0x10, 0x41}; + for (uint32_t i = vp; i + 4 <= n_; ++i) + if (std::memcmp(d_ + i, endb, 4) == 0) { + q = i; + break; + } + } + Node n; + n.name = name; + n.kind = Kind::Raw; + n.offset = start; + n.size = q - start; + n.hinted = true; + n.raw.assign(d_ + vp, d_ + q); + stats.raw_bytes += q - vp; + p = q; + return n; + } + + Cand order[4]; + int norder = 0; + switch (h.kind) { + case Prim::Int: + case Prim::Float: order[norder++] = Cand::Word; break; + case Prim::Bool: order[norder++] = Cand::Bool; break; + case Prim::String: order[norder++] = Cand::String; break; + case Prim::Int64: order[norder++] = Cand::Int64; break; + default: break; + } + for (Cand c : {Cand::Word, Cand::Bool, Cand::String, Cand::Int64}) { + bool have = false; + for (int i = 0; i < norder; ++i) + if (order[i] == c) have = true; + if (!have) order[norder++] = c; + } + bool hinted_kind = h.kind != Prim::None; + + // pass 1: continuation must be a non-empty tag / marker; pass 2 tolerates + // empty ("") tags; pass 3 (only when lookahead was defeated by EOF) takes + // whatever fits the remaining bytes. + eof_limited_ = false; + Scalar chosen; + int chosen_i = -1; + int chosen_mode = -1; + for (int mode = 0; mode < 3 && chosen_i < 0; ++mode) { + if (mode == 2 && !eof_limited_) break; + for (int i = 0; i < norder; ++i) { + Scalar r = try_scalar(start, ne, order[i], i == 0 && hinted_kind); + if (!r.ok) continue; + if (mode == 2 || plausible_item(r.end, 2, mode == 1)) { + chosen = r; + chosen_i = i; + chosen_mode = mode; + break; + } + } + } + if (chosen_i >= 0) { + Node n; + n.name = name; + n.offset = start; + n.size = chosen.end - start; + Prim ck; + switch (chosen.cand) { + case Cand::Word: + ck = (h.kind == Prim::Int || h.kind == Prim::Float) ? h.kind : classify_word(chosen.vp); + break; + case Cand::Bool: ck = Prim::Bool; break; + case Cand::String: ck = Prim::String; break; + default: ck = Prim::Int64; break; + } + n.kind = prim_to_kind(ck); + if (chosen.cand == Cand::String) n.raw.assign(chosen.vp + 4, chosen.vp + chosen.vsize); + else n.raw.assign(chosen.vp, chosen.vp + chosen.vsize); + n.hinted = hinted_kind && chosen_i == 0; + if (hinted_kind && chosen_i != 0) { + ++stats.hint_failures; + issue(Issue::Warn, start, + "'" + name + "': hinted type " + prim_name(h.kind) + " not plausible, read as " + prim_name(ck), + path); + } else if (!hinted_kind) { + ++stats.guessed; + } + if (chosen_mode == 2) { + ++stats.best_effort; + issue(Issue::Warn, start, + "'" + name + "': stream ends before layout can be confirmed; read as " + prim_name(ck), path); + } + p = chosen.end; + return n; + } + + // tag looked fine but no value layout fits: skip to the next sync point + auto [what, q] = resync(ne, depth); + ++stats.resyncs; + issue(Issue::Warn, start, + "'" + name + "': no value layout fits; skipped " + std::to_string(q - ne) + " bytes to " + what, path); + Node n; + n.name = name; + n.kind = Kind::Raw; + n.offset = start; + n.size = q - start; + n.raw.assign(d_ + ne, d_ + q); + stats.raw_bytes += q - ne; + p = q; + return n; +} + +// No tag at p. A small unnamed payload before an END is normal (info); +// anything longer is a resync event. +Node Walker::unreadable(uint32_t& p, int depth, const std::string& path) { + uint32_t start = p; + if (depth > 0) { + uint32_t q = start; + while (q + 4 <= n_ && q - start <= uint32_t(kSmallRaw)) { + if (u32(q) == kEndMark) { + p = q; + return raw_node(false, "", start, q, "unnamed payload", Issue::Info, path); + } + q += 4; + } + } + auto [what, q] = resync(start, depth); + ++stats.resyncs; + p = q; + return raw_node(false, "", start, q, (std::string("unreadable; resynced to ") + what).c_str(), Issue::Warn, + path); +} + +Node Walker::raw_node(bool tagged, const std::string& name, uint32_t p, uint32_t q, const char* why, + Issue::Level lvl, const std::string& path) { + issue(lvl, p, std::string(why) + ": " + std::to_string(q - p) + " raw bytes", path); + stats.raw_bytes += q - p; + Node n; + n.tagged = tagged; + n.name = name; + n.kind = Kind::Raw; + n.offset = p; + n.size = q - p; + n.raw.assign(d_ + p, d_ + q); + return n; +} + +Node read_tree(const Bytes& inflated, std::vector* issues, Stats* stats, const Registry* reg) { + Walker w(inflated.data(), inflated.size(), reg); + Node root = w.walk(); + if (issues) *issues = std::move(w.issues); + if (stats) *stats = w.stats; + return root; +} + +} // namespace mars::stream diff --git a/src/mars/stream/reader.h b/src/mars/stream/reader.h new file mode 100644 index 0000000..f6d3971 --- /dev/null +++ b/src/mars/stream/reader.h @@ -0,0 +1,78 @@ +// mars::stream — the generic walker. +// +// Tokenizes an inflated stream into a Node tree without needing a schema. +// Frames (BEEFBEEF / 41104110) are hard synchronisation points; scalar layout +// is chosen by a hint (registry) or by trying [word, bool, string, int64] and +// keeping the first layout after which another plausible item (or a marker, +// or EOF) follows. Unreadable stretches become raw nodes and the walk resumes +// at the next marker or plausible tag, so a damaged or unknown region never +// derails the rest of the file. +#pragma once + +#include +#include +#include +#include + +#include "node.h" +#include "schema.h" + +namespace mars::stream { + +constexpr int kMaxTagLen = 64; // field names are short identifiers +constexpr int kMaxStringLen = 1 << 20; // sanity bound for string payloads +constexpr int kSmallRaw = 64; // unnamed payload <= this before an END is info, not warn +constexpr int kWordIntAbs = 100000; // |int| <= this -> classified as int, not float + +class Walker { +public: + // `reg` may be null: everything is then guessed. + Walker(const uint8_t* data, size_t n, const Registry* reg); + + Node walk(); + + std::vector issues; + Stats stats; + +private: + enum class Cand { Word, Bool, String, Int64 }; + struct Scalar { + bool ok = false; + Cand cand = Cand::Word; + Prim prim = Prim::None; // resolved kind for the node + const uint8_t* vp = nullptr; + uint32_t vsize = 0; + uint32_t end = 0; + }; + struct TagAt { + bool ok = false; + std::string name; + uint32_t end = 0; + }; + + const uint8_t* d_; + uint32_t n_; + const Registry* reg_; + bool eof_limited_ = false; + + void issue(Issue::Level lvl, uint32_t off, std::string msg, const std::string& path); + uint32_t u32(uint32_t p) const { return rd_u32(d_ + p); } + bool zero(uint32_t a, uint32_t b) const; + TagAt tag_at(uint32_t p, bool allow_empty) const; + Scalar try_scalar(uint32_t tag_start, uint32_t name_end, Cand kind, bool known); + bool plausible_item(uint32_t p, int depth, bool allow_empty); + std::pair resync(uint32_t p, int depth); + Hint child_hint(const Desc* frame, size_t index, const std::string* name) const; + std::vector walk_frame(uint32_t& p, bool has_ctx, const std::string& ctx, int depth, const Desc* shape, + const std::string& path); + Node read_item(uint32_t& p, size_t index, int depth, const Desc* frame, const std::string& path); + Node unreadable(uint32_t& p, int depth, const std::string& path); + Node raw_node(bool tagged, const std::string& name, uint32_t p, uint32_t q, const char* why, Issue::Level lvl, + const std::string& path); +}; + +// Convenience: walk an inflated stream with the SotS save registry. +Node read_tree(const Bytes& inflated, std::vector* issues = nullptr, Stats* stats = nullptr, + const Registry* reg = &save_registry()); + +} // namespace mars::stream diff --git a/src/mars/stream/save.cpp b/src/mars/stream/save.cpp new file mode 100644 index 0000000..1e92963 --- /dev/null +++ b/src/mars/stream/save.cpp @@ -0,0 +1,98 @@ +#include "save.h" + +#include +#include +#include + +#include "gzip.h" +#include "reader.h" +#include "writer.h" + +namespace mars::stream { + +namespace { + +// Kinds for tags that only occur inside opaque ("any") regions; they let the +// walker type those bodies without a shape (tech tree, designs, objectives, +// comms, research, encounters). +const std::map& manual_kinds() { + static const std::map k = { + {"TNm", Prim::String}, {"tfc", Prim::Bool}, {"tResCost", Prim::Int}, {"tResDone", Prim::Int}, + {"tAcq", Prim::Int}, {"tiAcq", Prim::Int}, {"tUnlck", Prim::Int}, {"dName", Prim::String}, + {"wfn", Prim::String}, {"bId", Prim::Bool}, {"faiDes", Prim::Bool}, {"dHide", Prim::Bool}, + {"cmp", Prim::Bool}, {"dsc", Prim::String}, {"xcsn", Prim::String}, {"nm", Prim::String}, + {"ntg", Prim::String}, {"wep", Prim::String}, {"gmch", Prim::Int}, {"drad", Prim::Float}, + {"cst", Prim::Float}, {"aOdd", Prim::Float}, {"aInc", Prim::Float}, {"rMd", Prim::Float}, + {"sctSize", Prim::Float}, {"smx", Prim::Float}, {"nPrvVa", Prim::Float}, {"crPce", Prim::Bool}, + {"maintHf", Prim::Bool}, {"caps2", Prim::Int64}, {"tRsld", Prim::Bool}, {"tRsldd", Prim::Bool}, + {"tRsldc", Prim::Bool}, {"tRsldi", Prim::Bool}, {"tRsldr", Prim::Bool}, {"prm", Prim::Float}, + }; + return k; +} + +Registry build_registry() { + Registry reg; + SchemaBuilder::Context ctx(reg); + SchemaBuilder root(ctx, nullptr); // the file root is an unframed sequence: no positional hints + shapes::SaveGame g; + g.io(root); + SchemaBuilder::finalize(ctx, manual_kinds()); + return reg; +} + +} // namespace + +const Registry& save_registry() { + static const Registry reg = build_registry(); + return reg; +} + +shapes::SaveGame apply_shapes(const Node& root, std::vector& issues) { + shapes::SaveGame game; + ReadArchive ar(root.children, issues, ""); + game.io(ar); + if (!ar.done()) { + issues.push_back(Issue{Issue::Warn, "", ar.peek()->offset, + std::to_string(ar.remaining()) + " unexpected trailing items"}); + } + return game; +} + +SaveDocument read_save_bytes(const uint8_t* p, size_t n) { + SaveDocument doc; + doc.inflated = inflate_container(p, n); + Walker w(doc.inflated.data(), doc.inflated.size(), &save_registry()); + doc.tree = w.walk(); + doc.issues = std::move(w.issues); + doc.stats = w.stats; + doc.game = apply_shapes(doc.tree, doc.issues); + return doc; +} + +SaveDocument read_save_file(const std::string& path) { + std::ifstream f(path, std::ios::binary); + if (!f) throw GzipError("cannot open " + path); + Bytes data((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + return read_save_bytes(data.data(), data.size()); +} + +Bytes write_save(shapes::SaveGame& game) { + Writer w; + WriteArchive ar(w); + game.io(ar); + return w.take(); +} + +Bytes write_tree(const Node& root) { + Writer w; + for (const Node& c : root.children) w.node(c); + return w.take(); +} + +std::string format_issue(const Issue& i) { + char off[24]; + std::snprintf(off, sizeof off, "0x%x", i.offset); + return std::string("[") + level_name(i.level) + "] @" + off + " " + i.path + ": " + i.msg; +} + +} // namespace mars::stream diff --git a/src/mars/stream/save.h b/src/mars/stream/save.h new file mode 100644 index 0000000..cedef09 --- /dev/null +++ b/src/mars/stream/save.h @@ -0,0 +1,47 @@ +// mars::stream — top-level save-file API. +// +// SaveDocument doc = read_save_file(path); // or read_save_bytes(...) +// doc.tree generic Node tree (always complete: unknown regions are raw) +// doc.game typed shapes applied on top of the tree +// doc.issues walker + shape deviations (error / warn / info) +// write_save(doc.game) -> inflated bytes; gzip() them for a .sav +#pragma once + +#include +#include + +#include "node.h" +#include "shapes.h" + +namespace mars::stream { + +struct SaveDocument { + Bytes inflated; + Node tree; + shapes::SaveGame game; + std::vector issues; // walker issues first, then shape issues + Stats stats; + + size_t count(Issue::Level l) const { + size_t n = 0; + for (const Issue& i : issues) n += i.level == l; + return n; + } +}; + +// Parse a .sav (gzip) or an already-inflated stream. Throws GzipError on a damaged container. +SaveDocument read_save_bytes(const uint8_t* p, size_t n); +SaveDocument read_save_file(const std::string& path); + +// Apply the typed shapes to a generic tree (root children = file items). +shapes::SaveGame apply_shapes(const Node& root, std::vector& issues); + +// Serialize typed shapes back to an inflated stream. +Bytes write_save(shapes::SaveGame& game); + +// Re-emit a generic tree (root children) as bytes. +Bytes write_tree(const Node& root); + +std::string format_issue(const Issue& i); + +} // namespace mars::stream diff --git a/src/mars/stream/savedump_main.cpp b/src/mars/stream/savedump_main.cpp new file mode 100644 index 0000000..474beda --- /dev/null +++ b/src/mars/stream/savedump_main.cpp @@ -0,0 +1,116 @@ +// sots_savedump — CLI over mars::stream: dump / summarize / round-trip a save. +// +// sots_savedump SAVE [--dump] [--strict] [--roundtrip] [--issues N] [--max-depth D] +// [--inflate FILE] [--rewrite FILE] +// +// --dump generic tree, one item per line (same format as the reference reader) +// --strict exit 2 on any error or warning +// --roundtrip re-emit the tree and the typed shapes and compare with the input +// --rewrite write the typed shapes back out as a gzip .sav +#include +#include +#include +#include + +#include "dump.h" +#include "gzip.h" +#include "save.h" + +using namespace mars::stream; + +static void summary(const SaveDocument& d, const char* path) { + const auto& s = d.game.summary; + const auto& sim = d.game.sim; + std::printf("file: %s inflated: %zu bytes\n", path, d.inflated.size()); + std::printf("items: %u frames: %u guessed: %u hint-failures: %u resyncs: %u raw-bytes: %u\n", + d.stats.items, d.stats.frames, d.stats.guessed, d.stats.hint_failures, d.stats.resyncs, + d.stats.raw_bytes); + std::printf("issues: %zu error, %zu warn, %zu info\n", d.count(Issue::Error), d.count(Issue::Warn), + d.count(Issue::Info)); + std::printf("summary: game='%s' turn=%d numSys=%d players=%zu scenario='%s'\n", s.gameName.c_str(), s.turn, + s.numSys, s.players.size(), s.scenario.c_str()); + std::printf("sim: gameName='%s' players=%zu systems=%zu fleets=%zu\n", sim.gameName.c_str(), + sim.players.size(), sim.systems.size(), sim.fleets.size()); + for (const auto& p : sim.players) + std::printf(" player %d: '%s' species=%d home=%d sav=%d designs=%zu\n", p.playerID, + p.player.plryName.c_str(), p.player.species, p.player.homeSys, p.player.sav, + p.player.designs.size()); +} + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: sots_savedump SAVE [--dump] [--strict] [--roundtrip] [--issues N] " + "[--max-depth D] [--inflate FILE] [--rewrite FILE]\n"); + return 2; + } + const char* path = argv[1]; + bool dump = false, strict = false, roundtrip = false; + int max_issues = 30, max_depth = 999; + const char* inflate_to = nullptr; + const char* rewrite_to = nullptr; + for (int i = 2; i < argc; ++i) { + std::string a = argv[i]; + if (a == "--dump") dump = true; + else if (a == "--strict") strict = true; + else if (a == "--roundtrip") roundtrip = true; + else if (a == "--issues" && i + 1 < argc) max_issues = std::atoi(argv[++i]); + else if (a == "--max-depth" && i + 1 < argc) max_depth = std::atoi(argv[++i]); + else if (a == "--inflate" && i + 1 < argc) inflate_to = argv[++i]; + else if (a == "--rewrite" && i + 1 < argc) rewrite_to = argv[++i]; + else { + std::fprintf(stderr, "unknown option %s\n", argv[i]); + return 2; + } + } + SaveDocument doc; + try { + doc = read_save_file(path); + } catch (const std::exception& e) { + std::fprintf(stderr, "error: %s\n", e.what()); + return 2; + } + if (inflate_to) { + std::ofstream f(inflate_to, std::ios::binary); + f.write(reinterpret_cast(doc.inflated.data()), std::streamsize(doc.inflated.size())); + } + if (dump) { + for (const std::string& l : dump_tree(doc.tree, max_depth)) std::puts(l.c_str()); + } else { + summary(doc, path); + int shown = 0; + for (const Issue& i : doc.issues) { + if (i.level == Issue::Info) continue; + if (shown == 0) std::printf("issues (errors/warnings, first %d):\n", max_issues); + if (shown++ >= max_issues) break; + std::printf(" %s\n", format_issue(i).c_str()); + } + } + int rc = doc.count(Issue::Error) ? 1 : 0; + if (roundtrip) { + Bytes tree_bytes = write_tree(doc.tree); + bool tree_ok = tree_bytes == doc.inflated; + Bytes typed_bytes = write_save(doc.game); + bool typed_ok = typed_bytes == doc.inflated; + size_t first_diff = 0; + if (!typed_ok) { + size_t n = std::min(typed_bytes.size(), doc.inflated.size()); + while (first_diff < n && typed_bytes[first_diff] == doc.inflated[first_diff]) ++first_diff; + } + char where[48] = ""; + if (!typed_ok) std::snprintf(where, sizeof where, ", first difference at 0x%zx", first_diff); + std::printf("roundtrip: tree %s (%zu bytes), typed %s (%zu bytes%s)\n", tree_ok ? "identical" : "DIFFERS", + tree_bytes.size(), typed_ok ? "identical" : "DIFFERS", typed_bytes.size(), where); + if (!tree_ok || !typed_ok) rc = rc ? rc : 1; + } + if (rewrite_to) { + Bytes out = write_save(doc.game); + Bytes gz = gzip(out.data(), out.size()); + std::ofstream f(rewrite_to, std::ios::binary); + f.write(reinterpret_cast(gz.data()), std::streamsize(gz.size())); + } + if (strict && (doc.count(Issue::Error) || doc.count(Issue::Warn))) { + std::fprintf(stderr, "strict: %zu error(s), %zu warning(s)\n", doc.count(Issue::Error), doc.count(Issue::Warn)); + return 2; + } + return rc; +} diff --git a/src/mars/stream/schema.h b/src/mars/stream/schema.h new file mode 100644 index 0000000..806a89f --- /dev/null +++ b/src/mars/stream/schema.h @@ -0,0 +1,75 @@ +// mars::stream — type hints the walker consults while tokenizing. +// +// The stream carries no type bytes, so the generic walker decides scalar kinds +// from (a) a positional / by-name description of the frame it is inside +// (`Desc`), (b) a global (tag -> kind) catalog, and (c) a lookahead +// plausibility test. The descriptors are built once from the typed shapes in +// shapes.h by running their io() under a SchemaBuilder archive (archive.h); +// nothing here is required for the walk to succeed — hints only make the +// output better typed. +#pragma once + +#include +#include +#include +#include + +namespace mars::stream { + +// Scalar kinds a hint can name. Hint::None means "no scalar hint". +enum class Prim : uint8_t { None, Int, Float, Bool, Int64, String, Raw }; + +struct Desc; + +struct Hint { + Prim kind = Prim::None; + const Desc* sub = nullptr; // descriptor for a framed child (Shape / CArr / Raw) + bool empty() const { return kind == Prim::None && sub == nullptr; } +}; + +struct Desc { + enum Type { Shape, CArr, Raw }; + Type type = Shape; + std::string name; // Shape: on-disk frame tag ("" when unnamed) + std::vector prefix; // Shape: positional hints up to the first variable-length field + std::map by_name; // Shape: fallback by child tag (exact, then lower-cased) + Hint elem; // CArr: hint for every element after the "." count +}; + +struct Registry { + std::deque owned; // stable storage for descriptors + std::map kinds; // global tag -> scalar kind + std::map shapes; // global tag -> frame descriptor + std::map string_tags; // tags known to be strings (never text-tested) + + Desc* add(Desc d) { + owned.push_back(std::move(d)); + return &owned.back(); + } + const Desc* shape(const std::string& n) const { + auto it = shapes.find(n); + return it == shapes.end() ? nullptr : it->second; + } + Prim kind(const std::string& n) const { + auto it = kinds.find(n); + return it == kinds.end() ? Prim::None : it->second; + } +}; + +inline const char* prim_name(Prim p) { + switch (p) { + case Prim::None: return "-"; + case Prim::Int: return "int"; + case Prim::Float: return "float"; + case Prim::Bool: return "bool"; + case Prim::Int64: return "int64"; + case Prim::String: return "string"; + case Prim::Raw: return "raw"; + } + return "?"; +} + +// The registry describing the SotS save layout (built from shapes.h). +const Registry& save_registry(); + +} // namespace mars::stream diff --git a/src/mars/stream/shapes.h b/src/mars/stream/shapes.h new file mode 100644 index 0000000..b468012 --- /dev/null +++ b/src/mars/stream/shapes.h @@ -0,0 +1,1585 @@ +// mars::stream — typed shapes of the SotS 1.8 save stream. +// +// Field orders and types follow verify/save-reader/SAVE_FORMAT.md (the +// confirmed format). A("x") tags are on-disk names; R("x") fields are written +// by the game with a NULL name ("." on disk) or carry a reference name whose +// disk spelling differs, and are matched by position. Bodies the format keeps +// opaque (TechTree, Events, ShipRecs, spy2, civr, comms, Ojvs, Attrib, sprjs, +// SvSctOb, trdmgr, spymgr, CD, ...) are held as generic Nodes so a shape can +// be re-emitted byte-for-byte. +#pragma once + +#include +#include +#include +#include + +#include "archive.h" + +namespace mars::stream::shapes { + +// Game::PlayerColorID — "." int index; iff -1 the three "." ints r,g,b follow. +struct PlayerColor { + static constexpr const char* kStreamName = ""; + int32_t idx = 0; + int32_t r = 0, g = 0, b = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("idx"), idx); + ar.when(idx == -1, [&](Ar& a) { + a.i32(R("r"), r); + a.i32(R("g"), g); + a.i32(R("b"), b); + }); + ar.rest(extra); + } +}; + +// ---- Summary (StrategyGameInfo) --------------------------------------------- +struct PlayerSettings { // StrategyPlayerGameSettings: 4 x "." int + static constexpr const char* kStreamName = "Settings"; + int32_t treasury = 0, colonies = 0, techs = 0, difficulty = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("treasury"), treasury); + ar.i32(R("colonies"), colonies); + ar.i32(R("techs"), techs); + ar.i32(R("difficulty"), difficulty); + ar.rest(extra); + } +}; + +struct Slot { // SlotDef + static constexpr const char* kStreamName = "Slot"; + bool isPlay = false, isDead = false, isReq = false, isRec = false; + bool isFxNm = false; + std::string fxNm; + bool isFxSp = false; + int32_t fxSp = 0; + bool isFxCr = false; + PlayerColor fxCrID; + bool isFxBd = false; + std::string fxBd; + bool isFxAv = false; + std::string fxAv; + int32_t tag = 0; + std::string pwd; + int32_t team = 0; + PlayerSettings settings; + std::vector extra; + template + void io(Ar& ar) { + ar.b(A("IsPlay"), isPlay); + ar.b(A("IsDead"), isDead); + ar.b(A("IsReq"), isReq); + ar.b(A("IsRec"), isRec); + ar.b(A("IsFxNm"), isFxNm); + ar.str(A("FxNm"), fxNm); + ar.b(A("IsFxSp"), isFxSp); + ar.i32(A("FxSp"), fxSp); + ar.b(A("IsFxCr"), isFxCr); + ar.obj(A("FxCrID"), fxCrID); + ar.b(A("IsFxBd"), isFxBd); + ar.str(A("FxBd"), fxBd); + ar.b(A("IsFxAv"), isFxAv); + ar.str(A("FxAv"), fxAv); + ar.i32(A("Tag"), tag); + ar.str(A("Pwd"), pwd); + ar.i32(A("Team"), team); + ar.obj(A("Settings"), settings); + ar.rest(extra); + } +}; + +struct PlayerInfo { // element of Summary.Players ("." frame) + static constexpr const char* kStreamName = ""; + Slot slot; + int32_t rank = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.obj(A("Slot"), slot); + ar.i32(A("Rank"), rank); + ar.rest(extra); + } +}; + +struct Tmrs { + static constexpr const char* kStreamName = "TMRS"; + float tstl = 0, tctl = 0, tqtl = 0, tqtle = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.f32(A("TSTL"), tstl); + ar.f32(A("TCTL"), tctl); + ar.f32(A("TQTL"), tqtl); + ar.f32(A("TQTLE"), tqtle); + ar.rest(extra); + } +}; + +struct Session { + static constexpr const char* kStreamName = "Session"; + Tmrs tmrs; + std::vector extra; + template + void io(Ar& ar) { + ar.obj(A("TMRS"), tmrs); + ar.rest(extra); + } +}; + +struct Summary { + static constexpr const char* kStreamName = "Summary"; + std::string gameName; + int32_t turn = 0, numSys = 0, checksum = 0; + std::vector players; + Session session; + int32_t mapShape = 0; + float incMod = 0, resMod = 0; + bool alliances = false, teams = false, encounters = false; + std::string scenario; + std::vector extra; + template + void io(Ar& ar) { + ar.str(A("GameName"), gameName); + ar.i32(A("Turn"), turn); + ar.i32(A("NumSys"), numSys); + ar.i32(A("Checksum"), checksum); + ar.carr(A("Players"), players); + ar.obj(A("Session"), session); + ar.i32(A("MapShape"), mapShape); + ar.f32(A("IncMod"), incMod); + ar.f32(A("ResMod"), resMod); + ar.b(A("Alliances"), alliances); + ar.b(A("Teams"), teams); + ar.b(A("Encounters"), encounters); + ar.str(A("Scenario"), scenario); + ar.rest(extra); + } +}; + +// ---- CreateParams (StrategyGameCreateParams) ---------------------------------- +struct Planet { // SystemParams element, all tags "." + static constexpr const char* kStreamName = ""; + Vec3 pos; + int32_t p1 = 0, p2 = 0, p3 = 0; + float p4 = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.vec3(R("pos"), pos); + ar.i32(R("p1"), p1); + ar.i32(R("p2"), p2); + ar.i32(R("p3"), p3); + ar.f32(R("p4"), p4); + ar.rest(extra); + } +}; + +struct MapP { // StarMapParams, all tags "." + static constexpr const char* kStreamName = "MapP"; + int32_t mapType = 0; + std::vector planets; // VectorHelper + std::vector> players; // "." count, n x "." frame{ "." count, n x "." int } + std::vector nodePaths; // VectorHelper, empty in the real saves + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("mapType"), mapType); + ar.carr(R("planets"), planets); + ar.narr(R("players"), players, [](Ar& a, std::vector& e) { a.carr(R("."), e); }); + ar.carr(R("nodePaths"), nodePaths); + ar.rest(extra); + } +}; + +struct ScriptParam { + std::string spsn, sppn, sppv; + template + void io(Ar& ar) { + ar.str(A("spsn"), spsn); + ar.str(A("sppn"), sppn); + ar.str(A("sppv"), sppv); + } +}; + +struct Scrp { // StrategyScriptParams + static constexpr const char* kStreamName = "scrp"; + std::vector params; + std::vector extra; + template + void io(Ar& ar) { + ar.narr(A("spc"), params, [](Ar& a, ScriptParam& e) { e.io(a); }); + ar.rest(extra); + } +}; + +struct CreateParams { + static constexpr const char* kStreamName = "CreateParams"; + std::string name; + int32_t id = 0, rseed = 0, aid = 0; + std::string key; + MapP mapP; + int32_t mapS = 0; + std::string mapF; + int32_t nSys = 0; + float rEnc = 0, sDist = 0, sSize = 0, sRes = 0, sSuit = 0; + int32_t maxP = 0, aSpec = 0; + bool bAlly = false; + int32_t nTeam = 0; + bool tmgrp = false; + int32_t pSav = 0, pCol = 0, pTech = 0; + float incM = 0, resM = 0; + Scrp scrp; + std::vector extra; + template + void io(Ar& ar) { + ar.str(A("Name"), name); + ar.i32(A("ID"), id); + ar.i32(A("RSeed"), rseed); + ar.i32(A("AID"), aid); + ar.str(A("Key"), key); + ar.obj(A("MapP"), mapP); + ar.i32(A("MapS"), mapS); + ar.str(A("MapF"), mapF); + ar.i32(A("NSys"), nSys); + ar.f32(A("REnc"), rEnc); + ar.f32(A("SDist"), sDist); + ar.f32(A("SSize"), sSize); + ar.f32(A("SRes"), sRes); + ar.f32(A("SSuit"), sSuit); + ar.i32(A("MaxP"), maxP); + ar.i32(A("ASpec"), aSpec); + ar.b(A("bAlly"), bAlly); + ar.i32(A("NTeam"), nTeam); + ar.b(A("tmgrp"), tmgrp); + ar.i32(A("PSav"), pSav); + ar.i32(A("PCol"), pCol); + ar.i32(A("PTech"), pTech); + ar.f32(A("IncM"), incM); + ar.f32(A("ResM"), resM); + ar.obj(A("scrp"), scrp); + ar.rest(extra); + } +}; + +// ---- shared sim types --------------------------------------------------------- +struct PopG { + static constexpr const char* kStreamName = "PopG"; + int32_t popT = 0, popS = 0; + int64_t popC = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("PopT"), popT); + ar.i32(A("PopS"), popS); + ar.i64(A("PopC"), popC); + ar.rest(extra); + } +}; + +struct Population { + static constexpr const char* kStreamName = ""; + std::vector groups; + std::vector extra; + template + void io(Ar& ar) { + ar.narr(A("PopNG"), groups, [](Ar& a, PopG& e) { a.obj(A("PopG"), e); }); + ar.rest(extra); + } +}; + +struct MoraleEntry { + int32_t msp = 0, mv = 0; + template + void io(Ar& ar) { + ar.i32(A("msp"), msp); + ar.i32(A("mv"), mv); + } +}; + +struct Morale { + static constexpr const char* kStreamName = ""; + std::vector entries; + std::vector extra; + template + void io(Ar& ar) { + ar.narr(A("mnsp"), entries, [](Ar& a, MoraleEntry& e) { e.io(a); }); + ar.rest(extra); + } +}; + +struct MoraleEvent { + static constexpr const char* kStreamName = ""; + int32_t mid = 0, mtr = 0, mn = 0, mtp = 0; + Morale mfx; + std::string mdsc; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("mid"), mid); + ar.i32(A("mtr"), mtr); + ar.i32(A("mn"), mn); + ar.i32(A("mtp"), mtp); + ar.obj(A("mfx"), mfx); + ar.str(A("mdsc"), mdsc); + ar.rest(extra); + } +}; + +struct BuildOrder { + static constexpr const char* kStreamName = ""; + int32_t desID = 0, con = 0, conleft = 0, sav = 0, ordID = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("desID"), desID); + ar.i32(A("con"), con); + ar.i32(A("conleft"), conleft); + ar.i32(A("sav"), sav); + ar.i32(A("ordID"), ordID); + ar.rest(extra); + } +}; + +struct BuildQueue { + static constexpr const char* kStreamName = "BQ"; + std::vector orders; + bool ordersFramed = true; + std::vector extra; + template + void io(Ar& ar) { + ar.carr_flex(R("ords", "ords"), orders, ordersFramed); + ar.rest(extra); + } +}; + +struct IndependenceInfo { + static constexpr const char* kStreamName = "indi"; + int32_t indsp = 0; + PlayerColor indcl; + std::string indnm, indav, indba; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("indsp"), indsp); + ar.obj(A("indcl"), indcl); + ar.str(A("indnm"), indnm); + ar.str(A("indav"), indav); + ar.str(A("indba"), indba); + ar.rest(extra); + } +}; + +struct Rts { + static constexpr const char* kStreamName = "Rts"; + float srs = 0, srt = 0, srsc = 0, srtf = 0, sri = 0, sroh = 0; + int32_t srnr = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.f32(A("SRs"), srs); + ar.f32(A("SRt"), srt); + ar.f32(A("SRsc"), srsc); + ar.f32(A("SRtf"), srtf); + ar.f32(A("SRi"), sri); + ar.f32(A("SRoh"), sroh); + ar.i32(A("SRnr"), srnr); + ar.rest(extra); + } +}; + +struct PlayerView { + static constexpr const char* kStreamName = "pview"; + int32_t vtrn = 0, pop = 0; + Population pop2; + float infra = 0, suit = 0; + int32_t res = 0; + std::optional aRes; + int32_t aRes2 = 0, mRes = 0; + bool noRebAI = false; + int32_t pbon = 0; + Population pbon2; + float ibon = 0; + int32_t terrFl = 0; + bool footer = false; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("VTrn"), vtrn); + ar.i32(A("Pop"), pop); + ar.obj(A("Pop2"), pop2); + ar.f32(A("Infra"), infra); + ar.f32(A("Suit"), suit); + ar.i32(A("Res"), res); + ar.opt_i32(A("ARes"), aRes); + ar.i32(A("ARes2"), aRes2); + ar.i32(A("MRes"), mRes); + ar.b(A("NoRebAI"), noRebAI); + ar.i32(A("pbon"), pbon); + ar.obj(A("pbon2"), pbon2); + ar.f32(A("ibon"), ibon); + ar.i32(A("TerrFl"), terrFl); + ar.b(A("footer"), footer); + ar.rest(extra); + } +}; + +// ---- star system (ServerSystem) ----------------------------------------------- +struct HaltEntry { + int32_t haltt = 0; + bool haltv = false; + template + void io(Ar& ar) { + ar.i32(A("haltt"), haltt); + ar.b(A("haltv"), haltv); + } +}; +struct AdctEntry { + int32_t ads = 0, adt = 0; + template + void io(Ar& ar) { + ar.i32(A("ads"), ads); + ar.i32(A("adt"), adt); + } +}; +struct PlagueEntry { + int32_t plgT = 0; + Node plg; + template + void io(Ar& ar) { + ar.i32(A("PlgT"), plgT); + ar.any(A("Plg"), plg); + } +}; +struct Colony { // NVO entry; indi is an inline member, always present + int32_t pid = 0, tshn = 0, oid = 0; + bool isind = false; + IndependenceInfo indi; + template + void io(Ar& ar) { + ar.i32(A("PID"), pid); + ar.i32(A("TShn"), tshn); + ar.i32(A("OID"), oid); + ar.b(A("isind"), isind); + ar.obj(A("indi"), indi); + } +}; +struct NveEntry { + int32_t ePid = 0, ets = 0, eid = 0; + template + void io(Ar& ar) { + ar.i32(A("EPid"), ePid); + ar.i32(A("ETS"), ets); + ar.i32(A("Eid"), eid); + } +}; +struct ViewEntry { + int32_t pid = 0; + PlayerView pview; + template + void io(Ar& ar) { + ar.i32(R("pid"), pid); + ar.obj(R("pview", "pview"), pview); + } +}; + +struct Sys { + static constexpr const char* kStreamName = "Sys"; + Vec3 pos; + float r = 0, g = 0, b = 0, a = 0; + int32_t idx = 0, size = 0; + std::optional iSuit; + float suit = 0; + int32_t res = 0; + std::optional aRes; + int32_t aRes2 = 0, mRes = 0; + bool noRebAI = false; + int32_t tRes = 0, pop = 0; + Population pop2; + float infra = 0; + int32_t pvPop = 0; + Population pvPop2; + float pvInfra = 0, pvSuit = 0; + int32_t pvRes = 0, pvARes2 = 0, pvMRes = 0; + bool pvNoRebAI = false; + Rts rts; + bool abdn = false, dstyd = false; + int32_t tnsOH = 0; + float outMod = 0, repCur = 0, repMax = 0; + int32_t ntdev = 0, pbon = 0; + Population pbon2; + float ibon = 0; + int32_t ltis = 0, rbfl = 0, rbtn = 0, rbfr = 0, rbwn = 0; + bool hsrg = false; + std::vector halt; + bool vnh = false; + bool vnd = false, vnex3 = false, vnpex3 = false; + std::string name; + int32_t vFlags = 0, eFlags = 0, aFlags = 0, fFlags = 0, gFlags = 0; + int64_t bats2 = 0, rcex = 0; + int32_t mnRFlags = 0, rfRFlags = 0, clkFlags = 0; + int32_t eggScio = 0, terrFl = 0, tAcq = 0, tfAcq = 0, tDst = 0; + Population dcs; + float dsu = 0; + Morale cm, pvCM; + std::vector cme2; + bool cme2Framed = true; + Node spies2; + int32_t pid = 0, defF = 0, defSF = 0; + std::optional bq; + std::vector adct; + std::vector plagues; + std::vector fleets, gates, stations, monitors; + std::vector colonies; + std::vector nve; + std::vector views; + bool hindi = false; + IndependenceInfo indi; + std::vector extra; + + template + void io(Ar& ar) { + ar.vec3(A("Pos"), pos); + ar.f32(A("R"), r); + ar.f32(A("G"), g); + ar.f32(A("B"), b); + ar.f32(A("A"), a); + ar.i32(A("Idx"), idx); + ar.i32(A("Size"), size); + ar.opt_f32(A("ISuit"), iSuit); + ar.f32(A("Suit"), suit); + ar.i32(A("Res"), res); + ar.opt_i32(A("ARes"), aRes); + ar.i32(A("ARes2"), aRes2); + ar.i32(A("MRes"), mRes); + ar.b(A("NoRebAI"), noRebAI); + ar.i32(A("TRes"), tRes); + ar.i32(A("Pop"), pop); + ar.obj(A("Pop2"), pop2); + ar.f32(A("Infra"), infra); + ar.i32(A("PvPop"), pvPop); + ar.obj(A("PvPop2"), pvPop2); + ar.f32(A("PvInfra"), pvInfra); + ar.f32(A("PvSuit"), pvSuit); + ar.i32(A("PvRes"), pvRes); + ar.i32(A("PvARes2"), pvARes2); + ar.i32(A("PvMRes"), pvMRes); + ar.b(A("PvNoRebAI"), pvNoRebAI); + ar.obj(A("Rts"), rts); + ar.b(A("Abdn"), abdn); + ar.b(A("Dstyd"), dstyd); + ar.i32(A("TnsOH"), tnsOH); + ar.f32(A("OutMod"), outMod); + ar.f32(A("RepCur"), repCur); + ar.f32(A("RepMax"), repMax); + ar.i32(A("ntdev"), ntdev); + ar.i32(A("pbon"), pbon); + ar.obj(A("pbon2"), pbon2); + ar.f32(A("ibon"), ibon); + ar.i32(A("ltis"), ltis); + ar.i32(A("rbfl"), rbfl); + ar.i32(A("rbtn"), rbtn); + ar.i32(A("rbfr"), rbfr); + ar.i32(A("rbwn"), rbwn); + ar.b(A("hsrg"), hsrg); + ar.narr(A("haltc"), halt, [](Ar& a, HaltEntry& e) { e.io(a); }); + ar.b(A("vnh"), vnh); + ar.when(vnh, [&](Ar& a) { + a.b(A("vnd"), vnd); + a.b(A("vnex3"), vnex3); + a.b(A("vnpex3"), vnpex3); + }); + ar.str(A("Name"), name); + ar.i32(A("VFlags"), vFlags); + ar.i32(A("EFlags"), eFlags); + ar.i32(A("AFlags"), aFlags); + ar.i32(A("FFlags"), fFlags); + ar.i32(A("GFlags"), gFlags); + ar.i64(A("Bats2"), bats2); + ar.i64(A("rcex"), rcex); + ar.i32(A("MnRFlags"), mnRFlags); + ar.i32(A("RfRFlags"), rfRFlags); + ar.i32(A("ClkFlags"), clkFlags); + ar.i32(A("EggScio"), eggScio); + ar.i32(A("TerrFl"), terrFl); + ar.i32(A("TAcq"), tAcq); + ar.i32(A("TFAcq"), tfAcq); + ar.i32(A("TDst"), tDst); + ar.obj(A("dcs"), dcs); + ar.f32(A("dsu"), dsu); + ar.obj(A("cm"), cm); + ar.obj(A("PvCM"), pvCM); + ar.carr_flex(A("cme2"), cme2, cme2Framed); + ar.any(A("spies2"), spies2); + ar.i32(A("PID"), pid); + ar.i32(A("DefF"), defF); + ar.i32(A("DefSF"), defSF); + ar.opt_obj(A("BQ"), bq); + ar.narr(A("nadct"), adct, [](Ar& a, AdctEntry& e) { e.io(a); }); + ar.narr(A("NumPlgs2"), plagues, [](Ar& a, PlagueEntry& e) { e.io(a); }); + ar.narr(A("NumFlts"), fleets, [](Ar& a, int32_t& e) { a.i32(A("Flt"), e); }); + ar.narr(A("NumGFs"), gates, [](Ar& a, int32_t& e) { a.i32(A("GF"), e); }); + ar.narr(A("NumSnF"), stations, [](Ar& a, int32_t& e) { a.i32(A("SnF"), e); }); + ar.narr(A("NumMnF"), monitors, [](Ar& a, int32_t& e) { a.i32(A("MnF"), e); }); + ar.narr(A("NVO"), colonies, [](Ar& a, Colony& e) { e.io(a); }); + ar.narr(A("NVE"), nve, [](Ar& a, NveEntry& e) { e.io(a); }); + ar.narr(A("NVs"), views, [](Ar& a, ViewEntry& e) { e.io(a); }); + ar.b(A("hindi"), hindi); + ar.when(hindi, [&](Ar& a) { a.obj(A("indi"), indi); }); + ar.rest(extra); + } +}; + +// ---- player / empire ------------------------------------------------------------ +struct Alliances { // the second "Team" item in Player + static constexpr const char* kStreamName = "Team"; + int32_t alid = 0, al = 0, na = 0, cf = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("ALid"), alid); + ar.i32(A("AL"), al); + ar.i32(A("NA"), na); + ar.i32(A("CF"), cf); + ar.rest(extra); + } +}; + +struct DipStat { + static constexpr const char* kStreamName = ""; + int32_t other = 0; + int32_t lastnap = 0, lastnapbty = 0, bknnap = 0, btynap = 0; + int32_t lastally = 0, lastallybty = 0, bknally = 0, btyally = 0; + int32_t lastcf = 0, lastcfbty = 0, bkncf = 0, btycf = 0; + int32_t deadhome = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("other"), other); + ar.i32(A("lastnap"), lastnap); + ar.i32(A("lastnapbty"), lastnapbty); + ar.i32(A("bknnap"), bknnap); + ar.i32(A("btynap"), btynap); + ar.i32(A("lastally"), lastally); + ar.i32(A("lastallybty"), lastallybty); + ar.i32(A("bknally"), bknally); + ar.i32(A("btyally"), btyally); + ar.i32(A("lastcf"), lastcf); + ar.i32(A("lastcfbty"), lastcfbty); + ar.i32(A("bkncf"), bkncf); + ar.i32(A("btycf"), btycf); + ar.i32(A("deadhome"), deadhome); + ar.rest(extra); + } +}; + +struct Prep { + static constexpr const char* kStreamName = ""; + int32_t oid = 0, pid = 0, flds = 0, sav = 0, home = 0, ncol = 0, mpwr = 0, mcls = 0, mmsl = 0, nshp = 0, nsat = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("oid"), oid); + ar.i32(A("pid"), pid); + ar.i32(A("flds"), flds); + ar.i32(A("sav"), sav); + ar.i32(A("home"), home); + ar.i32(A("ncol"), ncol); + ar.i32(A("mpwr"), mpwr); + ar.i32(A("mcls"), mcls); + ar.i32(A("mmsl"), mmsl); + ar.i32(A("nshp"), nshp); + ar.i32(A("nsat"), nsat); + ar.rest(extra); + } +}; + +struct Odes { + static constexpr const char* kStreamName = ""; + int32_t ontF = 0, otnL = 0, odid = 0, opid = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("ontF", "otnF"), ontF); + ar.i32(R("otnL", "otnL"), otnL); + ar.i32(R("odid", "odid"), odid); + ar.i32(R("opid", "opid"), opid); + ar.rest(extra); + } +}; +struct Owep { + static constexpr const char* kStreamName = ""; + int32_t ontF = 0, otnL = 0, odet = 0; + std::string owep; + int32_t owith = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("ontF", "otnF"), ontF); + ar.i32(R("otnL", "otnL"), otnL); + ar.i32(R("odet", "odet"), odet); + ar.str(R("owep", "owep"), owep); + ar.i32(R("owith", "owith"), owith); + ar.rest(extra); + } +}; +struct Otch { + static constexpr const char* kStreamName = ""; + int32_t ontF = 0, otnL = 0, odet = 0; + std::string otch; + int32_t owith = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("ontF", "otnF"), ontF); + ar.i32(R("otnL", "otnL"), otnL); + ar.i32(R("odet", "odet"), odet); + ar.str(R("otch", "otch"), otch); + ar.i32(R("owith", "owith"), owith); + ar.rest(extra); + } +}; + +struct Note { + static constexpr const char* kStreamName = "Nts"; + int32_t ntSys = 0; + std::string ntTxt; + int32_t ntTrn = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("NtSys"), ntSys); + ar.str(A("NtTxt"), ntTxt); + ar.i32(A("NtTrn"), ntTrn); + ar.rest(extra); + } +}; + +struct Design { // on-disk tags are case variants of the reference names (FAIDes, DHide, DWep, DName) + static constexpr const char* kStreamName = "Des"; + bool faiDes = false, dHide = false; + int32_t dWep = 0; + std::string dName; + std::vector sections; + template + void io(Ar& ar) { + ar.b(R("faiDes", "FAIDes"), faiDes); + ar.b(R("dHide", "DHide"), dHide); + ar.i32(R("dWep", "DWep"), dWep); + ar.str(R("dName", "DName"), dName); + ar.rest(sections); + } +}; + +struct ConMods { + static constexpr const char* kStreamName = "ConMods"; + float conMod0 = 0, savMod0 = 0, conMod1 = 0, savMod1 = 0, conMod2 = 0, savMod2 = 0; + template + void io(Ar& ar) { + ar.f32(A("ConMod"), conMod0); + ar.f32(A("SavMod"), savMod0); + ar.f32(A("ConMod"), conMod1); + ar.f32(A("SavMod"), savMod1); + ar.f32(A("ConMod"), conMod2); + ar.f32(A("SavMod"), savMod2); + } +}; + +struct DesignEntry { + int32_t desID = 0; + Design des; + template + void io(Ar& ar) { + ar.i32(A("DesID"), desID); + ar.obj(A("Des"), des); + } +}; +struct PrEntry { + float prm = 0; + int32_t prbt = 0; + template + void io(Ar& ar) { + ar.f32(A("PRm"), prm); + ar.i32(A("PRBt"), prbt); + } +}; +struct SprjEntry { + int32_t sprjT = 0; + Node sprj; + template + void io(Ar& ar) { + ar.i32(A("SprjT"), sprjT); + ar.any(A("Sprj"), sprj); + } +}; +struct NexpEntry { + int32_t xid = 0, xmin = 0, xmax = 0; + float xper = 0; + template + void io(Ar& ar) { + ar.i32(A("xid"), xid); + ar.i32(A("xmin"), xmin); + ar.i32(A("xmax"), xmax); + ar.f32(A("xper"), xper); + } +}; + +struct Player { + static constexpr const char* kStreamName = "Player"; + Node techTree; + int32_t homeSys = 0, plyrIdx = 0; + std::string plryName; + int32_t species = 0; + PlayerColor clrID; + std::string bdg, avt; + int32_t team = 0, sav = 0; + float idealSuit = 0, suitTol = 0, maxOH = 0, resRate = 0, resMod = 0, resScl = 0, trm = 0; + int32_t trp = 0, tra = 0; + float outMod = 0, rebOutMod = 0, scOutMod = 0, incMod = 0; + std::optional sensMod; + std::optional exPopSys; + float popMod = 0, terraMod = 0; + bool aMine = false; + float minPure = 0, minRate = 0; + int32_t nGts = 0, prGtTrf = 0, gTraf = 0; + float cstR = 0, cstE = 0, cstT = 0; + int32_t maint = 0; + float shrm = 0; + int32_t status = 0; + bool elim = false, npc = false, rebAI = false, reqCL = false; + Alliances alliances; + int32_t hasVac = 0, hasImm = 0, npTrk = 0, hasDisc = 0, hasDiscSp = 0, hasDiscCl = 0, hasEnc = 0, hasEng = 0; + Node events, fng; + int32_t pvSav = 0; + bool pvMA = false, aiBn = false, cnTrd = false, cnRad = false, hgs = false, hadvs = false, harcc = false, + cnVItl = false; + float pddm = 0; + int32_t bnkWrn = 0, bnkTrn = 0, bnkPr = 0, bnkEl = 0; + Node shipRecs; + int32_t nextPrjID = 0, plcy = 0; + std::string pswd; + int32_t lret = 0, nmeid = 0; + bool cdp = false; + Node spy2, civr; + int32_t aidf = 0; + bool srn = false; + int32_t srnTo = 0, lboid = 0; + std::optional lcid; + int32_t lcid2 = 0; + std::string resTNm; + bool resErrRoll = false; + ConMods conMods; + bool conModsFramed = false; + std::vector owners; + std::vector designs, legacyDesigns; + std::vector notes; + std::vector pr; + bool hasAIR = false; + Node air; + bool cta = false; + Node aiEnf; + std::vector specialProjects; + std::vector nexp; + std::vector weapXcl; + Node ojvs; + std::vector dipstats; + Node comms; + std::vector preps; + std::vector odes; + std::vector owep; + std::vector otch; + Node aid; + std::vector defLayouts, raidTargets; + int32_t tnc = 0; + std::vector extra; + + template + void io(Ar& ar) { + ar.any(A("TechTree"), techTree); + ar.i32(A("HomeSys"), homeSys); + ar.i32(A("PlyrIdx"), plyrIdx); + ar.str(A("PlryName"), plryName); + ar.i32(A("Species"), species); + ar.obj(A("ClrID"), clrID); + ar.str(A("Bdg"), bdg); + ar.str(A("Avt"), avt); + ar.i32(A("Team"), team); + ar.i32(A("Sav"), sav); + ar.f32(A("IdealSuit"), idealSuit); + ar.f32(A("SuitTol"), suitTol); + ar.f32(A("MaxOH"), maxOH); + ar.f32(A("ResRate"), resRate); + ar.f32(A("ResMod"), resMod); + ar.f32(A("ResScl"), resScl); + ar.f32(A("TRM"), trm); + ar.i32(A("TRP"), trp); + ar.i32(A("TRA"), tra); + ar.f32(A("OutMod"), outMod); + ar.f32(A("RebOutMod"), rebOutMod); + ar.f32(A("ScOutMod"), scOutMod); + ar.f32(A("IncMod"), incMod); + ar.opt_f32(A("SensMod"), sensMod); + ar.opt_i32(A("ExPopSys"), exPopSys); + ar.f32(A("PopMod"), popMod); + ar.f32(A("TerraMod"), terraMod); + ar.b(A("AMine"), aMine); + ar.f32(A("MinPure"), minPure); + ar.f32(A("MinRate"), minRate); + ar.i32(A("NGts"), nGts); + ar.i32(A("PrGtTrf"), prGtTrf); + ar.i32(A("GTraf"), gTraf); + ar.f32(A("CstR"), cstR); + ar.f32(A("CstE"), cstE); + ar.f32(A("CstT"), cstT); + ar.i32(A("Maint"), maint); + ar.f32(A("shrm"), shrm); + ar.i32(A("Status"), status); + ar.b(A("Elim"), elim); + ar.b(A("NPC"), npc); + ar.b(A("RebAI"), rebAI); + ar.b(A("ReqCL"), reqCL); + ar.obj(A("Team"), alliances); + ar.i32(A("HasVac"), hasVac); + ar.i32(A("HasImm"), hasImm); + ar.i32(A("NPTrk"), npTrk); + ar.i32(A("HasDisc"), hasDisc); + ar.i32(A("HasDiscSp"), hasDiscSp); + ar.i32(A("HasDiscCl"), hasDiscCl); + ar.i32(A("HasEnc"), hasEnc); + ar.i32(A("HasEng"), hasEng); + ar.any(A("Events"), events); + ar.any(A("FNG"), fng); + ar.i32(A("PvSav"), pvSav); + ar.b(A("PvMA"), pvMA); + ar.b(A("AIBn"), aiBn); + ar.b(A("CnTrd"), cnTrd); + ar.b(A("CnRad"), cnRad); + ar.b(A("hgs"), hgs); + ar.b(A("hadvs"), hadvs); + ar.b(A("harcc"), harcc); + ar.b(A("CnVItl"), cnVItl); + ar.f32(A("pddm"), pddm); + ar.i32(A("BnkWrn"), bnkWrn); + ar.i32(A("BnkTrn"), bnkTrn); + ar.i32(A("BnkPr"), bnkPr); + ar.i32(A("BnkEl"), bnkEl); + ar.any(A("ShipRecs"), shipRecs); + ar.i32(A("NextPrjID"), nextPrjID); + ar.i32(A("plcy"), plcy); + ar.str(A("pswd"), pswd); + ar.i32(A("lret"), lret); + ar.i32(A("nmeid"), nmeid); + ar.b(A("cdp"), cdp); + ar.any(A("spy2"), spy2); + ar.any(A("civr"), civr); + ar.i32(A("aidf"), aidf); + ar.b(A("Srn"), srn); + ar.i32(A("SrnTo"), srnTo); + ar.i32(A("lboid"), lboid); + ar.opt_i32(A("lcid"), lcid); + ar.i32(A("lcid2"), lcid2); + ar.str(A("ResTNm"), resTNm); + ar.b(A("ResErrRoll"), resErrRoll); + ar.obj_flex(R("conMods", "ConMods"), conMods, conModsFramed); + ar.narr(A("NumOwn"), owners, [](Ar& a, int32_t& e) { a.i32(A("OwnId"), e); }); + ar.narr(A("NumDes"), designs, [](Ar& a, DesignEntry& e) { e.io(a); }); + ar.narr(A("NumLeg"), legacyDesigns, [](Ar& a, DesignEntry& e) { e.io(a); }); + ar.narr(A("NumNotes"), notes, [](Ar& a, Note& e) { a.obj(A("Nts"), e); }); + ar.narr(A("NumPR"), pr, [](Ar& a, PrEntry& e) { e.io(a); }); + ar.b(A("HasAIR"), hasAIR); + ar.when(hasAIR, [&](Ar& a) { a.any(A("AIR"), air); }); + ar.b(A("cta"), cta); + ar.any(A("AIEnf"), aiEnf); + ar.narr(A("NSprj"), specialProjects, [](Ar& a, SprjEntry& e) { e.io(a); }); + ar.narr(A("Nexp"), nexp, [](Ar& a, NexpEntry& e) { e.io(a); }); + ar.narr(A("NWeapXcl"), weapXcl, [](Ar& a, int32_t& e) { a.i32(A("WeapXcl"), e); }); + ar.any(A("Ojvs"), ojvs); + ar.carr(A("dipstats"), dipstats); + ar.any(A("comms"), comms); + ar.carr(A("preps"), preps); + ar.carr(A("odes"), odes); + ar.carr(A("owep"), owep); + ar.carr(A("otch"), otch); + ar.any(A("aid"), aid); + ar.narr(A("ndeflay"), defLayouts, [](Ar& a, Node& e) { a.any(A("deflay"), e); }); + ar.narr(A("rdtc"), raidTargets, [](Ar& a, Node& e) { a.any(A("rdt"), e); }); + ar.i32(A("tnc"), tnc); + ar.rest(extra); + } +}; + +// ---- fleets & ships -------------------------------------------------------------- +struct NodeRoute { + static constexpr const char* kStreamName = "nrt"; + int32_t nrp = 0, nrf = 0, nrt = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("nrp"), nrp); + ar.i32(A("nrf"), nrf); + ar.i32(A("nrt"), nrt); + ar.rest(extra); + } +}; +struct Waypoint { + static constexpr const char* kStreamName = ""; + int32_t wpt = 0, tp = 0; + NodeRoute nrt; + bool nrtFramed = true; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("Wpt"), wpt); + ar.i32(A("Tp"), tp); + ar.obj_flex(R("nrt", "nrt"), nrt, nrtFramed); + ar.rest(extra); + } +}; +struct FlightPlan { + static constexpr const char* kStreamName = "FPlan"; + std::vector wpts; + bool wptsFramed = true; + float fpsp2 = 0; + int32_t fpeta2 = 0; + Vec3 fpogn2, fpdpos; + int32_t pnd = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.carr_flex(R("wpts", "wpts"), wpts, wptsFramed); + ar.f32(A("FPsp2"), fpsp2); + ar.i32(A("FPeta2"), fpeta2); + ar.vec3(A("FPogn2"), fpogn2); + ar.vec3(A("FPdpos"), fpdpos); + ar.i32(A("pnd"), pnd); + ar.rest(extra); + } +}; +struct PrisonerEntry { + int32_t prSp = 0, prNum = 0; + template + void io(Ar& ar) { + ar.i32(A("PrSp"), prSp); + ar.i32(A("PrNum"), prNum); + } +}; +struct PrisonerHold { + static constexpr const char* kStreamName = "PrisH"; + int32_t prMax = 0; + std::vector prisoners; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("PrMax"), prMax); + ar.when(prMax > 0, [&](Ar& a) { a.narr(A("PrNSp"), prisoners, [](Ar& b, PrisonerEntry& e) { e.io(b); }); }); + ar.rest(extra); + } +}; +struct Thruster { + float th = 0, thm = 0; + template + void io(Ar& ar) { + ar.f32(A("TH"), th); + ar.f32(A("THM"), thm); + } +}; +struct Ship { + static constexpr const char* kStreamName = "Ship"; + int32_t desID = 0, fltID = 0, plrID = 0; + float range = 0; + Vec3 health; + int32_t conCap = 0; + float refCap = 0, repCap = 0; + int32_t mineCap = 0, plg = 0, act = 0; + bool dep = false, atq = false; + int32_t encID = 0; + PrisonerHold prisH; + int32_t lct = 0, tsd = 0, atsp = 0, tblt = 0; + bool hbq = false; + BuildQueue bq2; + bool hsp = false; + Population pop, ppop; + std::vector thrusters; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("DesID"), desID); + ar.i32(A("FltID"), fltID); + ar.i32(A("PlrID"), plrID); + ar.f32(A("Range"), range); + ar.vec3(A("Health"), health); + ar.i32(A("ConCap"), conCap); + ar.f32(A("RefCap"), refCap); + ar.f32(A("RepCap"), repCap); + ar.i32(A("MineCap"), mineCap); + ar.i32(A("Plg"), plg); + ar.i32(A("Act"), act); + ar.b(A("Dep"), dep); + ar.b(A("Atq"), atq); + ar.i32(A("EncID"), encID); + ar.obj(A("PrisH"), prisH); + ar.i32(A("LCT"), lct); + ar.i32(A("tsd"), tsd); + ar.i32(A("atsp"), atsp); + ar.i32(A("tblt"), tblt); + ar.b(A("hbq"), hbq); + ar.when(hbq, [&](Ar& a) { a.obj(A("BQ2"), bq2); }); + ar.b(A("hsp"), hsp); + ar.when(hsp, [&](Ar& a) { + a.obj(A("pop"), pop); + a.obj(A("ppop"), ppop); + }); + ar.narr(A("NTH"), thrusters, [](Ar& a, Thruster& e) { e.io(a); }); + ar.rest(extra); + } +}; +struct ShipEntry { + int32_t shipID = 0; + Ship ship; + template + void io(Ar& ar) { + ar.i32(A("ShipID"), shipID); + ar.obj(A("Ship"), ship); + } +}; +struct Fleet { + static constexpr const char* kStreamName = "Flt"; + Vec3 pos; + int32_t pid = 0, locID = 0; + std::optional sysID, trdID; + bool hfPlan = false; + FlightPlan fplan; + std::string ftName; + std::optional caps, gtTrf; + std::optional ftSens; + std::optional ftInc; + int32_t ftTrans = 0; + Vec3 ftOrig; + int32_t ftFlg = 0, ftae = 0, ftpae = 0, ftEnc = 0, ftMS = 0; + bool perm = false; + Vec3 prvPos; + bool hLay = false; + Node lay; + std::vector ships; + std::vector extra; + template + void io(Ar& ar) { + ar.vec3(A("Pos"), pos); + ar.i32(A("PID"), pid); + ar.i32(A("LocID"), locID); + ar.opt_i32(A("SysID"), sysID); + ar.opt_i32(A("TrdID"), trdID); + ar.b(A("HFPlan"), hfPlan); + ar.when(hfPlan, [&](Ar& a) { a.obj(A("FPlan"), fplan); }); + ar.str(A("FtName"), ftName); + ar.opt_i32(A("Caps"), caps); + ar.opt_i32(A("GtTrf"), gtTrf); + ar.opt_f32(A("FtSens"), ftSens); + ar.opt_i32(A("FtInc"), ftInc); + ar.i32(A("FtTrans"), ftTrans); + ar.vec3(A("FtOrig"), ftOrig); + ar.i32(A("FtFlg"), ftFlg); + ar.i32(A("Ftae"), ftae); + ar.i32(A("Ftpae"), ftpae); + ar.i32(A("FtEnc"), ftEnc); + ar.i32(A("FtMS"), ftMS); + ar.b(A("Perm"), perm); + ar.vec3(A("PrvPos"), prvPos); + ar.b(A("HLay"), hLay); + ar.when(hLay, [&](Ar& a) { a.any(A("Lay"), lay); }); + ar.narr(A("NShips"), ships, [](Ar& a, ShipEntry& e) { e.io(a); }); + ar.rest(extra); + } +}; + +// ---- combat reports, node grid --------------------------------------------------- +struct Dams { + static constexpr const char* kStreamName = "dams"; + int32_t dams = 0, damp = 0, dami = 0, damt = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("dams", "dams"), dams); + ar.i32(R("damp", "damp"), damp); + ar.i32(R("dami", "dami"), dami); + ar.i32(R("damt", "damt"), damt); + ar.rest(extra); + } +}; +struct Wrep { + static constexpr const char* kStreamName = ""; + std::string wep; + Dams dams; + bool damsFramed = true; + std::vector extra; + template + void io(Ar& ar) { + ar.str(R("wep", "wep"), wep); + ar.obj_flex(R("dams", "dams"), dams, damsFramed); + ar.rest(extra); + } +}; +struct CrepPrep { + static constexpr const char* kStreamName = ""; + int32_t plr = 0; + bool ai = false; + int32_t ally = 0, status = 0, mxeng = 0, mxcls = 0, mxmsl = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("plr", "plr"), plr); + ar.b(R("ai", "ai"), ai); + ar.i32(R("ally", "ally"), ally); + ar.i32(R("status", "status"), status); + ar.i32(R("mxeng", "mxeng"), mxeng); + ar.i32(R("mxcls", "mxcls"), mxcls); + ar.i32(R("mxmsl", "mxmsl"), mxmsl); + ar.rest(extra); + } +}; +struct Crep { + static constexpr const char* kStreamName = "crep"; + int32_t cid = 0, trn = 0; + Vec3 pos; + int32_t sid = 0, autoF = 0, dur = 0, cow = 0, cdst = 0, cpk = 0, cpt = 0, cdt = 0, cdi = 0; + std::vector prep; + bool prepFramed = true; + std::vector wrep; + bool wrepFramed = true; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("cid", "cid"), cid); + ar.i32(R("trn", "trn"), trn); + ar.vec3(R("pos", "pos"), pos); + ar.i32(R("sid", "sid"), sid); + ar.i32(R("auto", "auto"), autoF); + ar.i32(R("dur", "dur"), dur); + ar.i32(R("cow", "cow"), cow); + ar.i32(R("cdst", "cdst"), cdst); + ar.i32(R("cpk", "cpk"), cpk); + ar.i32(R("cpt", "cpt"), cpt); + ar.i32(R("cdt", "cdt"), cdt); + ar.i32(R("cdi", "cdi"), cdi); + ar.carr_flex(R("prep", "prep"), prep, prepFramed); + ar.carr_flex(R("wrep", "wrep"), wrep, wrepFramed); + ar.rest(extra); + } +}; +struct NodePath { + static constexpr const char* kStreamName = ""; + int32_t npt = 0, npid = 0, npfr = 0, npto = 0, npctm = 0, npcby = 0, npdtn = 0, npdtf = 0, npenp = 0, npuse = 0, + nptf = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(R("npt", "npt"), npt); + ar.i32(R("npid", "npid"), npid); + ar.i32(R("npfr", "npfr"), npfr); + ar.i32(R("npto", "npto"), npto); + ar.i32(R("npctm", "npctm"), npctm); + ar.i32(R("npcby", "npcby"), npcby); + ar.i32(R("npdtn", "npdtn"), npdtn); + ar.i32(R("npdtf", "npdtf"), npdtf); + ar.i32(R("npenp", "npenp"), npenp); + ar.i32(R("npuse", "npuse"), npuse); + ar.i32(R("nptf", "nptf"), nptf); + ar.rest(extra); + } +}; +struct NodeGrid { + static constexpr const char* kStreamName = "NdGr2"; + std::vector paths; + bool pathsFramed = true; + int32_t nextId = 0; + std::vector extra; + template + void io(Ar& ar) { + ar.carr_flex(R("paths", "paths"), paths, pathsFramed); + ar.i32(R("nextId", "nextid"), nextId); + ar.rest(extra); + } +}; + +// ---- turn statistics --------------------------------------------------------------- +struct SystemEvent { + static constexpr const char* kStreamName = ""; + int32_t set = 0, ses = 0, seop = 0, senp = 0; + std::vector others; + std::vector extra; + template + void io(Ar& ar) { + ar.i32(A("set"), set); + ar.i32(A("ses"), ses); + ar.i32(A("seop"), seop); + ar.i32(A("senp"), senp); + ar.narr(A("seno2"), others, [](Ar& a, int32_t& e) { a.i32(A("seot2"), e); }); + ar.rest(extra); + } +}; +struct ClassStats { + int32_t cls = 0, shpt = 0, shpl = 0, shpk = 0, satt = 0, satl = 0, satk = 0; + template + void io(Ar& ar) { + ar.i32(A("cls"), cls); + ar.i32(A("shpt"), shpt); + ar.i32(A("shpl"), shpl); + ar.i32(A("shpk"), shpk); + ar.i32(A("satt"), satt); + ar.i32(A("satl"), satl); + ar.i32(A("satk"), satk); + } +}; +struct PlayerTurnStats { + static constexpr const char* kStreamName = "stats"; + int64_t pop = 0; + std::vector sacq, slost; + int32_t trn = 0, almem = 0, inc = 0, tdinc = 0, sav = 0, col = 0, bat = 0, tch = 0; + std::vector classes; + std::vector extra; + template + void io(Ar& ar) { + ar.i64(A("pop"), pop); + ar.carr(A("sacq"), sacq); + ar.carr(A("slost"), slost); + ar.i32(A("trn"), trn); + ar.i32(A("almem"), almem); + ar.i32(A("inc"), inc); + ar.i32(A("tdinc"), tdinc); + ar.i32(A("sav"), sav); + ar.i32(A("col"), col); + ar.i32(A("bat"), bat); + ar.i32(A("tch"), tch); + ar.narr(A("ncls"), classes, [](Ar& a, ClassStats& e) { e.io(a); }); + ar.rest(extra); + } +}; +struct PlayerTurnHistory { + static constexpr const char* kStreamName = "hist"; + int32_t ply = 0; + std::vector stats; // no count: until the frame END + template + void io(Ar& ar) { + ar.i32(A("ply"), ply); + ar.repeat("stats", stats, [](Ar& a, PlayerTurnStats& e) { a.obj(A("stats"), e); }); + } +}; +struct TurnHistoryEntry { + int32_t ply = 0; + PlayerTurnHistory hist; + template + void io(Ar& ar) { + ar.i32(A("ply"), ply); + ar.obj(A("hist"), hist); + } +}; +struct TurnStats { + static constexpr const char* kStreamName = "turnstats"; + std::vector players; + std::vector extra; + template + void io(Ar& ar) { + ar.narr(A("nply"), players, [](Ar& a, TurnHistoryEntry& e) { e.io(a); }); + ar.rest(extra); + } +}; + +// ---- Sim block (StrategyServer) ------------------------------------------------------ +struct Invasion { + int32_t invs = 0, inve = 0, invt = 0, invtb = 0; + template + void io(Ar& ar) { + ar.i32(A("invs"), invs); + ar.i32(A("inve"), inve); + ar.i32(A("invt"), invt); + ar.i32(A("invtb"), invtb); + } +}; +struct IntPair { + int32_t a = 0, b = 0; +}; +struct Species { + std::string issp; + float issu = 0; + template + void io(Ar& ar) { + ar.str(A("ISsp"), issp); + ar.f32(A("ISsu"), issu); + } +}; +struct PlayerEntry { + int32_t playerID = 0; + Player player; + template + void io(Ar& ar) { + ar.i32(A("PlayerID"), playerID); + ar.obj(A("Player"), player); + } +}; +struct SysEntry { + int32_t sysID = 0; + Sys sys; + template + void io(Ar& ar) { + ar.i32(A("SysID"), sysID); + ar.obj(A("Sys"), sys); + } +}; +struct FleetEntry { + int32_t fltID = 0; + Fleet flt; + template + void io(Ar& ar) { + ar.i32(A("FltID"), fltID); + ar.obj(A("Flt"), flt); + } +}; +struct ZoneDefence { + int32_t zdsi = 0, zdst = 0; + template + void io(Ar& ar) { + ar.i32(A("zdsi"), zdsi); + ar.i32(A("zdst"), zdst); + } +}; + +struct Sim { + static constexpr const char* kStreamName = "Sim"; + std::string keyPath; + int32_t nmSz = 0, nmLc = 0, nmnx = 0; + std::vector playerIds, designIds, systemIds, fleetIds, shipIds, tradeIds; + int32_t modCount = 0, frame = 0, gameID = 0; + std::optional aiDifficultyID; + Node attrib; + std::optional rand; + Node rng; // opaque MT19937 blob: frame with one "." raw item (mt[624] + left, 3 pad bytes) + std::string gameName; + int32_t map = 0; + float incMod = 0, resMod = 0; + std::optional randEnc; + bool enAl = false, enTm = false; + int32_t goTurn = 0; + std::vector goWinPly; + int32_t npcm = 0, npco = 0, npci = 0, npcv = 0, npca = 0; + std::optional npc; + float szadj = 0, rsadj = 0, suadj = 0; + Node sprjs; + float randEncAdj = 0; + int32_t cmbtid = 0; + TurnStats turnstats; + std::vector combatReports; + std::vector invasions; + std::vector exclusions3, exclusions2, exclusionsCF; + std::vector players; + std::vector species; + std::vector systems; + NodeGrid ndGr2; + Node trdmgr, spymgr; + std::vector fleets; + std::vector acts; + std::optional svSctOb; + std::vector zoneDefence; + std::vector extra; + + template + void io(Ar& ar) { + auto idlist = [](Ar& a, int32_t& e) { a.i32(R("."), e); }; + ar.str(A("KeyPath"), keyPath); + ar.i32(A("NMSz"), nmSz); + ar.i32(A("NMLc"), nmLc); + ar.i32(A("NMnx"), nmnx); + ar.narr(A("PlayerIDs"), playerIds, idlist); + ar.narr(A("DesignIDs"), designIds, idlist); + ar.narr(A("SystemIDs"), systemIds, idlist); + ar.narr(A("FleetIDs"), fleetIds, idlist); + ar.narr(A("ShipIDs"), shipIds, idlist); + ar.narr(A("TradeIDs"), tradeIds, idlist); + ar.i32(A("ModCount"), modCount); + ar.i32(A("Frame"), frame); + ar.i32(A("GameID"), gameID); + ar.opt_i32(A("AIDifficultyID"), aiDifficultyID); + ar.any(A("Attrib"), attrib); + ar.opt_i32(A("Rand"), rand); + ar.raw_frame(A("RNG"), rng); + ar.str(A("GameName"), gameName); + ar.i32(A("Map"), map); + ar.f32(A("IncMod"), incMod); + ar.f32(A("ResMod"), resMod); + ar.opt_b(A("RandEnc"), randEnc); + ar.b(A("EnAl"), enAl); + ar.b(A("EnTm"), enTm); + ar.i32(A("GOTurn"), goTurn); + ar.carr(A("GOWinPly"), goWinPly); + ar.i32(A("NPCm"), npcm); + ar.i32(A("NPCo"), npco); + ar.i32(A("NPCi"), npci); + ar.i32(A("NPCv"), npcv); + ar.i32(A("NPCa"), npca); + ar.opt_i32(A("NPC"), npc); + ar.f32(A("szadj"), szadj); + ar.f32(A("rsadj"), rsadj); + ar.f32(A("suadj"), suadj); + ar.any(A("sprjs"), sprjs); + ar.f32(A("RandEncAdj"), randEncAdj); + ar.i32(A("cmbtid"), cmbtid); + ar.obj(A("turnstats"), turnstats); + ar.narr(A("numcreps"), combatReports, [](Ar& a, Crep& e) { a.obj(A("crep"), e); }); + ar.narr(A("ninv"), invasions, [](Ar& a, Invasion& e) { e.io(a); }); + auto excl = [](Ar& a, IntPair& e) { + a.i32(A("AllExc"), e.a); + a.i32(A("AllExc"), e.b); + }; + ar.narr(A("AllExc"), exclusions3, excl); + ar.narr(A("AllExc"), exclusions2, excl); + ar.narr(A("AllExcCF"), exclusionsCF, [](Ar& a, IntPair& e) { + a.i32(A("AllExcCFp"), e.a); + a.i32(A("AllExcCFp"), e.b); + }); + ar.narr(A("NumPlrs"), players, [](Ar& a, PlayerEntry& e) { e.io(a); }); + ar.repeat("ISsp", species, [](Ar& a, Species& e) { e.io(a); }); // 7 pairs, no count + ar.narr(A("NumSys"), systems, [](Ar& a, SysEntry& e) { e.io(a); }); + ar.obj(A("NdGr2"), ndGr2); + ar.any(A("trdmgr"), trdmgr); + ar.any(A("spymgr"), spymgr); + ar.narr(A("NumFlts"), fleets, [](Ar& a, FleetEntry& e) { e.io(a); }); + ar.narr(A("NumActs"), acts, [](Ar& a, int32_t& e) { a.i32(A("Act"), e); }); + ar.opt_any(A("SvSctOb"), svSctOb); // only if the pointer was non-NULL + ar.narr(A("zdsc"), zoneDefence, [](Ar& a, ZoneDefence& e) { e.io(a); }); + ar.rest(extra); + } +}; + +// ---- custom-data table + file root ----------------------------------------------------- +struct CdTable { + static constexpr const char* kStreamName = "CDT"; + std::vector ids; + std::vector extra; + template + void io(Ar& ar) { + ar.narr(A("NumIDs"), ids, [](Ar& a, std::string& e) { a.str(A("ID"), e); }); + ar.rest(extra); + } +}; + +struct SaveGame { // root: Summary, CreateParams, Sim, CDT, then one opaque CD frame per id with data + static constexpr const char* kStreamName = ""; + Summary summary; + CreateParams createParams; + Sim sim; + CdTable cdTable; + std::vector customData; + template + void io(Ar& ar) { + ar.obj(A("Summary"), summary); + ar.obj(A("CreateParams"), createParams); + ar.obj(A("Sim"), sim); + ar.obj(A("CDT"), cdTable); + ar.repeat("CD", customData, [](Ar& a, Node& e) { a.any(A("CD"), e); }); + } +}; + +} // namespace mars::stream::shapes diff --git a/src/mars/stream/writer.cpp b/src/mars/stream/writer.cpp new file mode 100644 index 0000000..3d58902 --- /dev/null +++ b/src/mars/stream/writer.cpp @@ -0,0 +1,31 @@ +#include "writer.h" + +namespace mars::stream { + +void Writer::node(const Node& n) { + switch (n.kind) { + case Kind::Complex: + if (n.tagged) begin(n.name); + else begin_tagless(); + for (const Node& c : n.children) node(c); + end(); + return; + case Kind::String: + string(n.name, n.as_string()); + return; + case Kind::Raw: + // raw payloads keep whatever the reader captured, including any + // padding bytes that preceded the END marker + if (n.tagged) raw(n.name, n.raw); + else bare(n.raw.data(), n.raw.size()); + return; + default: + // int / float / bool / int64: the raw bytes are the value + begin_item(n.name); + put_bytes(buf_, n.raw.data(), n.raw.size()); + end_item(); + return; + } +} + +} // namespace mars::stream diff --git a/src/mars/stream/writer.h b/src/mars/stream/writer.h new file mode 100644 index 0000000..2660bbb --- /dev/null +++ b/src/mars/stream/writer.h @@ -0,0 +1,66 @@ +// mars::stream — emits the Streamable byte format (joint padding, framed +// complex values). Mirrors what the reader accepts so read -> write is +// byte-identical. +#pragma once + +#include + +#include "bytes.h" +#include "node.h" + +namespace mars::stream { + +class Writer { +public: + // --- scalars: [len][name][value][pad to 4] ------------------------------ + void int32(const std::string& name, int32_t v) { begin_item(name); put_i32(buf_, v); end_item(); } + void float32(const std::string& name, float v) { begin_item(name); put_f32(buf_, v); end_item(); } + void boolean(const std::string& name, bool v) { begin_item(name); buf_.push_back(v ? 1 : 0); end_item(); } + void int64(const std::string& name, int64_t v) { begin_item(name); put_u64(buf_, uint64_t(v)); end_item(); } + // string value = [int32 len][cp1252 bytes]; an empty string is 4 zero bytes + void string(const std::string& name, const std::string& v) { + begin_item(name); + put_i32(buf_, int32_t(v.size())); + put_bytes(buf_, v.data(), v.size()); + end_item(); + } + // opaque payload whose length only the writer knows (RNG state) + void raw(const std::string& name, const uint8_t* p, size_t n) { + begin_item(name); + put_bytes(buf_, p, n); + end_item(); + } + void raw(const std::string& name, const Bytes& b) { raw(name, b.data(), b.size()); } + + // --- frames: [len][name][pad] BEEFBEEF ... 41104110 ----------------------- + void begin(const std::string& name) { + begin_item(name); + end_item(); // the tag alone is padded to 4 + put_u32(buf_, kBeginMark); + } + void begin_tagless() { put_u32(buf_, kBeginMark); } + void end() { put_u32(buf_, kEndMark); } + + // unnamed bytes (unnamed raw payload before an END marker) + void bare(const uint8_t* p, size_t n) { put_bytes(buf_, p, n); } + + // re-emit a generic tree node (and its subtree) exactly + void node(const Node& n); + + const Bytes& bytes() const { return buf_; } + Bytes take() { return std::move(buf_); } + size_t size() const { return buf_.size(); } + +private: + Bytes buf_; + size_t item_start_ = 0; + + void begin_item(const std::string& name) { + item_start_ = buf_.size(); + put_i32(buf_, int32_t(name.size())); + put_bytes(buf_, name.data(), name.size()); + } + void end_item() { put_pad(buf_, item_start_); } +}; + +} // namespace mars::stream diff --git a/tests/mars_stream/build_and_run.sh b/tests/mars_stream/build_and_run.sh new file mode 100755 index 0000000..32c5a02 --- /dev/null +++ b/tests/mars_stream/build_and_run.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Self-contained build + test for src/mars/stream and src/mars/rng (plain g++, no CMake). +# +# tests/mars_stream/build_and_run.sh [BUILD_DIR] +# +# Environment (all optional): +# SOTS_SAVES_DIR directory with the owner's *.sav files -> real-save test runs +# (skipped cleanly when unset; saves never enter the repo) +# SOTS_SAVE_READER path to the reference save_reader.py -> oracle dump comparison +# PYTHON interpreter for the oracle script (default: python3) +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +root="$(cd "$here/../.." && pwd)" +out="${1:-$root/build-mars-stream}" +mkdir -p "$out" + +CXX="${CXX:-g++}" +CC="${CC:-gcc}" +CXXFLAGS="-std=c++17 -O2 -Wall -Wextra -Wpedantic -I$root/src" +MINIZ_DEFS="-DMINIZ_NO_STDIO -DMINIZ_NO_ARCHIVE_APIS -DMINIZ_NO_TIME" + +echo "== build ($out)" +$CC -O2 -w $MINIZ_DEFS -c "$root/third_party/miniz/miniz.c" -o "$out/miniz.o" +objs=() +for src in gzip reader writer dump save; do + $CXX $CXXFLAGS $MINIZ_DEFS -c "$root/src/mars/stream/$src.cpp" -o "$out/stream_$src.o" + objs+=("$out/stream_$src.o") +done +$CXX $CXXFLAGS -c "$root/src/mars/rng/mt19937.cpp" -o "$out/rng_mt19937.o" +$CXX $CXXFLAGS -o "$out/sots_savedump" "$root/src/mars/stream/savedump_main.cpp" "${objs[@]}" "$out/miniz.o" +$CXX $CXXFLAGS -o "$out/test_rng" "$here/test_rng.cpp" "$out/rng_mt19937.o" +$CXX $CXXFLAGS -o "$out/test_stream" "$here/test_stream.cpp" "${objs[@]}" "$out/miniz.o" +$CXX $CXXFLAGS -o "$out/test_save" "$here/test_save.cpp" "${objs[@]}" "$out/miniz.o" "$out/rng_mt19937.o" + +echo "== test_rng" +"$out/test_rng" +echo "== test_stream" +"$out/test_stream" +echo "== test_save" +if [ -n "${SOTS_SAVES_DIR:-}" ]; then + mkdir -p "$out/dumps" + SOTS_DUMP_DIR="$out/dumps" "$out/test_save" + if [ -n "${SOTS_SAVE_READER:-}" ]; then + echo "== oracle comparison" + ${PYTHON:-python3} "$here/oracle/compare.py" --reader "$SOTS_SAVE_READER" --saves "$SOTS_SAVES_DIR" \ + --dumps "$out/dumps" --python "${PYTHON:-python3}" + else + echo "oracle comparison skipped (SOTS_SAVE_READER not set)" + fi +else + "$out/test_save" +fi +echo "== all ok" diff --git a/tests/mars_stream/oracle/compare.py b/tests/mars_stream/oracle/compare.py new file mode 100755 index 0000000..df60b73 --- /dev/null +++ b/tests/mars_stream/oracle/compare.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""compare.py -- oracle check of the C++ walker against the reference reader. + +usage: compare.py --reader SAVE_READER.py --saves DIR --dumps DIR [--python PY] + +For every DIR/*.sav it runs `save_reader.py SAVE --dump --padding joint` and +reads the C++ dump written by test_save (`.cpp.dump` in --dumps), then +reports two agreement figures per save: + + exact lines identical (including the `?` guess marks and `(alt ...)`) + canonical lines identical after normalising what is presentation only: + guess marks and alt readings dropped, 4-byte words compared by + their raw bytes (an int reading and a float reading of the same + word are the same datum), strings compared by content. + +The typed summary (game name, turn, numSys, players) is compared as well. +Exit status 1 when any canonical line differs or a summary disagrees. +""" +import argparse +import ast +import glob +import json +import os +import re +import shlex +import struct +import subprocess +import sys + +LINE = re.compile(r"^@([0-9a-f]{8}) (\s*)(\S+|) (\S+?)(\??) ?(.*)$") + + +def canon(line: str) -> str: + m = LINE.match(line) + if not m: + return line + off, indent, name, kind, _guess, rest = m.groups() + rest = re.sub(r"\s+\(alt .*\)$", "", rest) + if kind in ("int", "float"): + try: + raw = struct.pack(" +#include +#include + +#include "mars/rng/mt19937.h" + +using mars::rng::MT19937; + +static int fails = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++fails; \ + } \ + } while (0) + +int main() { + // --- standard MT19937 outputs for init_genrand(5489) ------------------------ + { + MT19937 r(5489u); + static const uint32_t expect[10] = {3499211612u, 581869302u, 3890346734u, 3586334585u, 545404204u, + 4161255391u, 3922919429u, 949333985u, 2715962298u, 1323567403u}; + for (uint32_t e : expect) CHECK(r.next_u32() == e); + // the 10000th output of the 5489 stream (well-known check value) + MT19937 r2(5489u); + uint32_t v = 0; + for (int i = 0; i < 10000; ++i) v = r2.next_u32(); + CHECK(v == 4123659995u); + } + // --- a fresh generator has twisted once: left == N ------------------------ + { + MT19937 r(1u); + CHECK(r.left() == MT19937::N); + CHECK(r.index() == 0); + r.next_u32(); + CHECK(r.left() == MT19937::N - 1); + for (int i = 1; i < MT19937::N; ++i) r.next_u32(); + CHECK(r.left() == 0); // block exhausted; the next draw twists + r.next_u32(); + CHECK(r.left() == MT19937::N - 1); + } + // --- float mapping: y * 2^-32 narrowed to float ----------------------------- + { + MT19937 r(5489u); + float f = r.next_float(); + float expect = static_cast(3499211612.0 / 4294967296.0); + CHECK(f == expect); + CHECK(f >= 0.f && f <= 1.f); + } + // --- next_int: in range, and consumes exactly one word when mask == n-1 ---- + { + MT19937 r(7u); + for (int i = 0; i < 1000; ++i) CHECK(r.next_int(10) < 10); + MT19937 a(9u), b(9u); + uint32_t x = a.next_int(256); + CHECK(x == (b.next_u32() & 255u)); + } + // --- save_state / load_state round trip, blob layout mt[624] + left -------- + { + MT19937 r(123456u); + for (int i = 0; i < 700; ++i) r.next_u32(); // past one twist + uint8_t blob[MT19937::kStateBytes]; + r.save_state(blob); + CHECK(MT19937::kStateBytes == 0x9c4); + uint32_t left_in_blob = uint32_t(blob[2496]) | (uint32_t(blob[2497]) << 8) | (uint32_t(blob[2498]) << 16) | + (uint32_t(blob[2499]) << 24); + CHECK(int(left_in_blob) == r.left()); + CHECK(std::memcmp(blob, r.state(), 4) == 0 || true); // first word is mt[0] little-endian + uint32_t w0 = uint32_t(blob[0]) | (uint32_t(blob[1]) << 8) | (uint32_t(blob[2]) << 16) | (uint32_t(blob[3]) << 24); + CHECK(w0 == r.state()[0]); + + MT19937 s(1u); + CHECK(s.load_state(blob, sizeof blob)); + CHECK(s.left() == r.left()); + for (int i = 0; i < 2000; ++i) CHECK(s.next_u32() == r.next_u32()); + + // truncated / out-of-range blobs are rejected + CHECK(!s.load_state(blob, sizeof blob - 1)); + uint8_t bad[MT19937::kStateBytes]; + std::memcpy(bad, blob, sizeof bad); + bad[2496] = 0x71; // left = 625 > N + bad[2497] = 0x02; + CHECK(!s.load_state(bad, sizeof bad)); + } + // --- load_state(mt, left) positions the next word at mt[N - left] ----------- + { + MT19937 r(42u); + uint32_t st[MT19937::N]; + std::memcpy(st, r.state(), sizeof st); + MT19937 s(0u); + s.load_state(st, 5); + for (int i = 0; i < MT19937::N - 5; ++i) r.next_u32(); + for (int i = 0; i < 100; ++i) CHECK(s.next_u32() == r.next_u32()); + } + std::printf("test_rng: %s\n", fails ? "FAILED" : "ok"); + return fails ? 1 : 0; +} diff --git a/tests/mars_stream/test_save.cpp b/tests/mars_stream/test_save.cpp new file mode 100644 index 0000000..7741398 --- /dev/null +++ b/tests/mars_stream/test_save.cpp @@ -0,0 +1,156 @@ +// Real-save test: reads every *.sav in $SOTS_SAVES_DIR (owner's data, never +// in the repo) and checks that the walker parses clean, the typed shapes +// load, both round trips are byte-identical and the RNG blob is a valid +// MT19937 state. Skips (exit 0) when the variable is unset. +// +// With SOTS_DUMP_DIR set, writes .cpp.dump / .cpp.summary there +// for tests/mars_stream/oracle/compare.py. +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mars/rng/mt19937.h" +#include "mars/stream/dump.h" +#include "mars/stream/save.h" + +using namespace mars::stream; + +static int fails = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++fails; \ + } \ + } while (0) + +static std::vector list_saves(const std::string& dir) { + std::vector out; + DIR* d = opendir(dir.c_str()); + if (!d) return out; + while (dirent* e = readdir(d)) { + std::string n = e->d_name; + if (n.size() > 4 && n.compare(n.size() - 4, 4, ".sav") == 0) out.push_back(dir + "/" + n); + } + closedir(d); + std::sort(out.begin(), out.end()); + return out; +} + +static void check_save(const std::string& path, const char* dump_dir) { + std::printf("== %s\n", path.c_str()); + SaveDocument doc = read_save_file(path); + std::printf(" inflated %zu bytes, items %u, frames %u, resyncs %u, hint-failures %u, raw-bytes %u\n", + doc.inflated.size(), doc.stats.items, doc.stats.frames, doc.stats.resyncs, doc.stats.hint_failures, + doc.stats.raw_bytes); + std::printf(" issues: %zu error, %zu warn, %zu info\n", doc.count(Issue::Error), doc.count(Issue::Warn), + doc.count(Issue::Info)); + for (const Issue& i : doc.issues) + if (i.level != Issue::Info) std::printf(" %s\n", format_issue(i).c_str()); + + // --- the confirmed format parses clean -------------------------------------- + CHECK(doc.stats.resyncs == 0); + CHECK(doc.stats.hint_failures == 0); + CHECK(doc.count(Issue::Error) == 0); + CHECK(doc.count(Issue::Warn) == 0); + CHECK(doc.stats.raw_bytes == 2503); // only the opaque RNG blob + + // --- typed shapes ------------------------------------------------------------- + const auto& s = doc.game.summary; + const auto& sim = doc.game.sim; + std::printf(" summary: game='%s' turn=%d numSys=%d players=%zu\n", s.gameName.c_str(), s.turn, s.numSys, + s.players.size()); + CHECK(!s.gameName.empty()); + CHECK(s.turn >= 1); + CHECK(s.numSys > 0); + CHECK(!s.players.empty()); + CHECK(sim.gameName == s.gameName); + CHECK(int(sim.systems.size()) == s.numSys); + CHECK(sim.systems.size() == sim.systemIds.size()); + CHECK(sim.players.size() == sim.playerIds.size()); + CHECK(sim.fleets.size() == sim.fleetIds.size()); + CHECK(sim.species.size() == 7); + CHECK(!sim.players.empty() && !sim.players[0].player.plryName.empty()); + CHECK(doc.game.createParams.name == s.gameName); + CHECK(doc.game.createParams.nSys == s.numSys); + CHECK(doc.game.createParams.mapP.planets.size() == size_t(s.numSys)); + for (const auto& se : sim.systems) CHECK(!se.sys.name.empty()); + for (const auto& fe : sim.fleets) CHECK(fe.flt.ships.size() >= 1); + CHECK(!doc.game.cdTable.ids.empty()); + + // --- RNG blob: mt[624] + left, produced by our MT19937 from RSeed -------------- + CHECK(sim.rng.is_complex() && sim.rng.children.size() == 1); + const Node& blob = sim.rng.children[0]; + CHECK(blob.kind == Kind::Raw && blob.raw.size() == 2503); + mars::rng::MT19937 saved(1u); + CHECK(saved.load_state(blob.raw.data(), blob.raw.size())); + std::printf(" rng: left=%d (index %d), RSeed=%d\n", saved.left(), saved.index(), doc.game.createParams.rseed); + CHECK(saved.left() >= 0 && saved.left() <= mars::rng::MT19937::N); + { + // the saved block must be reachable from seed(RSeed) by whole twists + mars::rng::MT19937 gen(uint32_t(doc.game.createParams.rseed)); + int twists = -1; + for (int k = 0; k < 16 && twists < 0; ++k) { + if (std::memcmp(gen.state(), saved.state(), sizeof(uint32_t) * mars::rng::MT19937::N) == 0) twists = k; + else + for (int i = 0; i < mars::rng::MT19937::N; ++i) gen.next_u32(); // consume a block -> next twist + } + std::printf(" rng: state == seed(RSeed) after %d twist(s)\n", twists); + CHECK(twists >= 0); + } + + // --- round trips ------------------------------------------------------------------ + Bytes tree_bytes = write_tree(doc.tree); + CHECK(tree_bytes == doc.inflated); + Bytes typed_bytes = write_save(doc.game); + if (typed_bytes != doc.inflated) { + size_t i = 0, n = std::min(typed_bytes.size(), doc.inflated.size()); + while (i < n && typed_bytes[i] == doc.inflated[i]) ++i; + std::printf(" typed round trip differs at 0x%zx (sizes %zu vs %zu)\n", i, typed_bytes.size(), + doc.inflated.size()); + } + CHECK(typed_bytes == doc.inflated); + std::printf(" round trip: tree %s, typed %s\n", tree_bytes == doc.inflated ? "identical" : "DIFFERS", + typed_bytes == doc.inflated ? "identical" : "DIFFERS"); + + // --- optional dump for the oracle comparison ------------------------------------- + if (dump_dir) { + std::string base = path.substr(path.find_last_of('/') + 1); + std::ofstream d(std::string(dump_dir) + "/" + base + ".cpp.dump"); + for (const std::string& l : dump_tree(doc.tree)) d << l << '\n'; + std::ofstream sm(std::string(dump_dir) + "/" + base + ".cpp.summary"); + sm << "summary: game=" << json_quote_cp1252(s.gameName) << " turn=" << s.turn << " numSys=" << s.numSys + << " players=" << s.players.size() << '\n'; + sm << "sim: players=" << sim.players.size() << " systems=" << sim.systems.size() + << " fleets=" << sim.fleets.size() << '\n'; + } +} + +int main() { + const char* dir = std::getenv("SOTS_SAVES_DIR"); + if (!dir || !*dir) { + std::printf("test_save: SKIPPED (SOTS_SAVES_DIR not set)\n"); + return 0; + } + std::vector saves = list_saves(dir); + if (saves.empty()) { + std::printf("test_save: SKIPPED (no *.sav in %s)\n", dir); + return 0; + } + const char* dump_dir = std::getenv("SOTS_DUMP_DIR"); + for (const std::string& p : saves) { + try { + check_save(p, dump_dir && *dump_dir ? dump_dir : nullptr); + } catch (const std::exception& e) { + std::printf("FAIL %s: %s\n", p.c_str(), e.what()); + ++fails; + } + } + std::printf("test_save: %s (%zu save(s), %d failures)\n", fails ? "FAILED" : "ok", saves.size(), fails); + return fails ? 1 : 0; +} diff --git a/tests/mars_stream/test_stream.cpp b/tests/mars_stream/test_stream.cpp new file mode 100644 index 0000000..3c17b18 --- /dev/null +++ b/tests/mars_stream/test_stream.cpp @@ -0,0 +1,530 @@ +// mars::stream unit tests on hand-built byte fixtures (no game data). +#include +#include +#include +#include +#include + +#include "mars/stream/dump.h" +#include "mars/stream/gzip.h" +#include "mars/stream/reader.h" +#include "mars/stream/save.h" +#include "mars/stream/writer.h" + +using namespace mars::stream; + +static int fails = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++fails; \ + } \ + } while (0) +#define CHECK_EQ(a, b) \ + do { \ + auto _a = (a); \ + auto _b = (b); \ + if (!(_a == _b)) { \ + std::printf("FAIL %s:%d: %s == %s\n", __FILE__, __LINE__, #a, #b); \ + ++fails; \ + } \ + } while (0) + +static Bytes B(std::initializer_list v) { + Bytes b; + for (int x : v) b.push_back(uint8_t(x)); + return b; +} +static Bytes cat(std::initializer_list parts) { + Bytes b; + for (const Bytes& p : parts) b.insert(b.end(), p.begin(), p.end()); + return b; +} +static size_t count(const std::vector& v, Issue::Level l) { + size_t n = 0; + for (const Issue& i : v) n += i.level == l; + return n; +} + +// --- 1. primitive encodings and joint padding (bytes written by hand) ----------- +static void test_primitives_bytes() { + // [len][name][value][pad to 4], padding computed over the whole item + const Bytes turn = B({4, 0, 0, 0, 'T', 'u', 'r', 'n', 1, 0, 0, 0}); // 12: no pad + const Bytes haltv = B({5, 0, 0, 0, 'h', 'a', 'l', 't', 'v', 1, 0, 0}); // 10 -> 12 + const Bytes vnh = B({3, 0, 0, 0, 'v', 'n', 'h', 0}); // 8: no pad + const Bytes name = B({4, 0, 0, 0, 'N', 'a', 'm', 'e', 3, 0, 0, 0, 'S', 'o', 'l', 0}); // 15 -> 16 + const Bytes key = B({3, 0, 0, 0, 'K', 'e', 'y', 0, 0, 0, 0, 0}); // empty string = 4 zero bytes + const Bytes incmod = B({6, 0, 0, 0, 'I', 'n', 'c', 'M', 'o', 'd', 0, 0, 0x80, 0x3f, 0, 0}); // float 1.0, 14 -> 16 + const Bytes bats2 = B({5, 0, 0, 0, 'B', 'a', 't', 's', '2', 0x2a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); // int64 42: 17 -> 20 + const Bytes dot = B({1, 0, 0, 0, '.', 7, 0, 0, 0, 0, 0, 0}); // "." int 7: 9 -> 12 + + Writer w; + w.int32("Turn", 1); + w.boolean("haltv", true); + w.boolean("vnh", false); + w.string("Name", "Sol"); + w.string("Key", ""); + w.float32("IncMod", 1.0f); + w.int64("Bats2", 42); + w.int32(".", 7); + CHECK(w.bytes() == cat({turn, haltv, vnh, name, key, incmod, bats2, dot})); + + std::vector issues; + Stats st; + Node root = read_tree(w.bytes(), &issues, &st); // with the save registry: all tags hinted + CHECK_EQ(root.children.size(), size_t(8)); + const auto& c = root.children; + CHECK(c[0].name == "Turn" && c[0].kind == Kind::Int && c[0].as_int() == 1 && c[0].hinted); + CHECK(c[0].offset == 0 && c[0].size == 12); + CHECK(c[1].name == "haltv" && c[1].kind == Kind::Bool && c[1].as_bool() && c[1].size == 12); + CHECK(c[2].name == "vnh" && c[2].kind == Kind::Bool && !c[2].as_bool() && c[2].size == 8); + CHECK(c[3].name == "Name" && c[3].kind == Kind::String && c[3].as_string() == "Sol" && c[3].size == 16); + CHECK(c[4].name == "Key" && c[4].kind == Kind::String && c[4].as_string().empty() && c[4].size == 12); + CHECK(c[5].name == "IncMod" && c[5].kind == Kind::Float && c[5].as_float() == 1.0f && c[5].size == 16); + CHECK(c[6].name == "Bats2" && c[6].kind == Kind::Int64 && c[6].as_int64() == 42 && c[6].size == 20); + CHECK(c[7].name == "." && c[7].kind == Kind::Int && c[7].as_int() == 7 && !c[7].hinted); + CHECK_EQ(st.resyncs, 0u); + CHECK_EQ(count(issues, Issue::Warn), size_t(0)); + CHECK_EQ(count(issues, Issue::Error), size_t(0)); + + // without any registry everything is guessed by layout + bit pattern + Node g = read_tree(w.bytes(), nullptr, nullptr, nullptr); + CHECK(g.children[0].kind == Kind::Int && !g.children[0].hinted); + CHECK(g.children[1].kind == Kind::Bool); + CHECK(g.children[3].kind == Kind::String); + CHECK(g.children[4].kind == Kind::Int && g.children[4].as_int() == 0); // empty string looks like int 0 + CHECK(g.children[5].kind == Kind::Float); // 0x3f800000 is not a small int + CHECK(g.children[6].kind == Kind::Int64); + + // round trip: the tree re-emits byte-identically + CHECK(write_tree(root) == w.bytes()); + CHECK(write_tree(g) == w.bytes()); +} + +// --- 2. frames: nesting, tagless frame, empty frame, "." arrays ------------------ +static void test_frames() { + Writer w; + w.begin("Summary"); + w.string("GameName", "x"); + w.begin("Players"); // VectorHelper: "." count + "." elements + w.int32(".", 2); + w.begin("."); + w.int32("Rank", 1); + w.end(); + w.begin("."); + w.int32("Rank", 2); + w.end(); + w.end(); + w.begin("Empty"); + w.end(); + w.begin_tagless(); + w.float32(".", 2.5f); + w.end(); + w.end(); + // expected framing bytes for the head: [7]"Summary"[pad] BEEFBEEF + const Bytes head = B({7, 0, 0, 0, 'S', 'u', 'm', 'm', 'a', 'r', 'y', 0, 0xef, 0xbe, 0xef, 0xbe}); + CHECK(std::equal(head.begin(), head.end(), w.bytes().begin())); + const Bytes tail = B({0x10, 0x41, 0x10, 0x41}); + CHECK(std::equal(tail.begin(), tail.end(), w.bytes().end() - 4)); + + std::vector issues; + Stats st; + Node root = read_tree(w.bytes(), &issues, &st); + CHECK_EQ(root.children.size(), size_t(1)); + const Node& s = root.children[0]; + CHECK(s.is_complex() && s.name == "Summary" && s.offset == 0 && s.size == w.size()); + CHECK_EQ(s.children.size(), size_t(4)); + const Node& players = s.children[1]; + CHECK(players.is_complex() && players.children.size() == 3); + CHECK(players.children[0].name == "." && players.children[0].as_int() == 2); + CHECK(players.children[1].is_complex() && players.children[1].name == "."); + CHECK(players.children[2].children[0].name == "Rank" && players.children[2].children[0].as_int() == 2); + CHECK(s.children[2].is_complex() && s.children[2].children.empty()); + CHECK(s.children[3].is_complex() && !s.children[3].tagged && s.children[3].children.size() == 1); + CHECK(s.children[3].children[0].as_float() == 2.5f); + CHECK_EQ(st.frames, 6u); + CHECK_EQ(st.resyncs, 0u); + CHECK(write_tree(root) == w.bytes()); + + // the dump prints frames with item count and size, tagless as + auto lines = dump_tree(root); + CHECK(lines.size() == 17); + CHECK(lines[0] == "@00000000 Summary { # 4 items, " + std::to_string(w.size()) + " bytes"); + CHECK(lines[1] == "@00000010 GameName string \"x\""); + CHECK(lines[13].find(" {") != std::string::npos); +} + +// --- 3. resync: garbage inside a frame becomes raw; the walk continues ---------- +// (both fixtures were checked against the reference reader: same tree, same +// stats, one warning each) +static void test_resync() { + Bytes tail; + { + Writer t; + t.begin("B"); + t.int32("z", 2); + t.end(); + tail = t.take(); + } + // case A: no tag at all after a nested frame -> unnamed raw, resync to the next plausible tag + { + Writer w; + w.begin("A"); + w.begin("C"); + w.end(); + Bytes s = w.take(); + s.insert(s.end(), 12, 0xff); + Writer y; + y.string("y", std::string(80, 'y')); + y.end(); + Bytes yb = y.take(); + s = cat({s, yb, tail}); + std::vector issues; + Stats st; + Node root = read_tree(s, &issues, &st); + CHECK_EQ(root.children.size(), size_t(2)); + const Node& a = root.children[0]; + CHECK(a.name == "A" && a.children.size() == 3 && a.size == 136); + CHECK(a.children[0].name == "C" && a.children[0].children.empty() && a.children[0].size == 16); + const Node& r = a.children[1]; + CHECK(r.kind == Kind::Raw && !r.tagged && r.raw.size() == 12 && r.offset == 0x1c); + CHECK(a.children[2].name == "y" && a.children[2].kind == Kind::String && a.children[2].as_string().size() == 80); + CHECK(root.children[1].name == "B" && root.children[1].children[0].as_int() == 2); + CHECK_EQ(st.resyncs, 1u); + CHECK_EQ(st.raw_bytes, 12u); + CHECK_EQ(count(issues, Issue::Warn), size_t(1)); + CHECK(write_tree(root) == s); // raw nodes re-emit exactly + auto lines = dump_tree(root); + CHECK(lines[3] == "@0000001c raw[12] ffffffffffffffffffffffff"); + } + // case B: a fine-looking tag whose value fits no layout -> tagged raw up to the next tag + { + Bytes s = B({3, 0, 0, 0, 'a', 'b', 'c', 5}); + s.insert(s.end(), 12, 0xff); + Writer w; + w.int32("y", 2); + w.end(); + Bytes head; + { + Writer h; + h.begin("A"); + head = h.take(); + } + s = cat({head, s, w.take(), tail}); + std::vector issues; + Stats st; + Node root = read_tree(s, &issues, &st); + const Node& a = root.children[0]; + CHECK(a.children.size() == 2 && a.size == 48); + CHECK(a.children[0].name == "abc" && a.children[0].kind == Kind::Raw && a.children[0].tagged); + CHECK(a.children[0].raw.size() == 13 && a.children[0].raw[0] == 5); + CHECK(a.children[1].name == "y" && a.children[1].as_int() == 2 && a.children[1].offset == 0x20); + CHECK_EQ(st.resyncs, 1u); + CHECK_EQ(st.raw_bytes, 13u); + CHECK_EQ(count(issues, Issue::Warn), size_t(1)); + CHECK(issues[0].msg.find("no value layout fits; skipped 13 bytes to tag") != std::string::npos); + CHECK(write_tree(root) == s); + } + + // unnamed 12-byte payload right before END is only an info (Vector3 fallback) + Bytes v3; + { + Writer t; + t.begin("Pos"); + t.end(); + v3 = t.take(); + Bytes body = B({0, 0, 0x80, 0x3f, 0, 0, 0, 0x40, 0, 0, 0x40, 0x40}); + v3.insert(v3.end() - 4, body.begin(), body.end()); + } + std::vector issues; + Stats st; + Node r3 = read_tree(v3, &issues, &st); + CHECK(r3.children[0].children.size() == 1 && r3.children[0].children[0].kind == Kind::Raw); + CHECK_EQ(count(issues, Issue::Info), size_t(1)); + CHECK_EQ(count(issues, Issue::Warn), size_t(0)); + // and the shape layer reads it as a vec3 + std::vector si; + ReadArchive ar(r3.children, si, ""); + Vec3 v; + ar.vec3(A("Pos"), v); + CHECK(v.x == 1.f && v.y == 2.f && v.z == 3.f); + + // a stray END at top level and trailing bytes are reported, not fatal + Bytes stray = B({0x10, 0x41, 0x10, 0x41, 1, 2}); + issues.clear(); + Node r4 = read_tree(stray, &issues, &st); + CHECK(r4.children.size() == 2); + CHECK(count(issues, Issue::Warn) >= 2); + CHECK(write_tree(r4) == stray); +} + +// --- 4. cp1252 string values never veto layout when the tag is known ------------ +static void test_cp1252() { + Writer w; + w.boolean("haltv", true); + w.string("Name", std::string("Kor\x92Voth")); // 0x92 = right single quote + w.int32("VFlags", 3); + std::vector issues; + Stats st; + Node root = read_tree(w.bytes(), &issues, &st); + CHECK_EQ(root.children.size(), size_t(3)); + CHECK(root.children[0].kind == Kind::Bool && root.children[0].hinted); + CHECK(root.children[1].kind == Kind::String && root.children[1].as_string() == "Kor\x92Voth"); + CHECK(root.children[2].kind == Kind::Int && root.children[2].as_int() == 3); + CHECK_EQ(st.hint_failures, 0u); + auto lines = dump_tree(root); + CHECK(lines[1] == "@0000000c Name string \"Kor\\u2019Voth\""); + // guessing an unknown tag still accepts every cp1252-defined byte + Writer g; + g.string("zz", std::string("caf\xe9")); + g.int32("q", 1); + Node r2 = read_tree(g.bytes(), nullptr, nullptr, nullptr); + CHECK(r2.children[0].kind == Kind::String); +} + +// --- 5. hints: registry types Summary children positionally and by name --------- +static void test_hints() { + Writer w; + w.begin("Summary"); + w.string("GameName", "g"); + w.int32("Turn", 0); // 0 either way + w.int32("NumSys", 0); + w.int32("Checksum", 0); + w.begin("Players"); + w.int32(".", 0); + w.end(); + w.begin("Session"); + w.begin("TMRS"); + w.float32("TSTL", 0.0f); // 0.0f: guessed would be int 0 + w.end(); + w.end(); + w.int32("MapShape", 0); + w.float32("IncMod", 0.0f); + w.end(); + Node root = read_tree(w.bytes()); + const Node& s = root.children[0]; + CHECK(s.hinted); + CHECK(s.children[0].kind == Kind::String && s.children[0].hinted); + CHECK(s.children[4].hinted); // Players CArr + CHECK(s.children[4].children[0].kind == Kind::Int && s.children[4].children[0].hinted); + const Node& tstl = s.children[5].children[0].children[0]; + CHECK(tstl.kind == Kind::Float && tstl.hinted && tstl.as_float() == 0.f); + CHECK(s.children[7].kind == Kind::Float && s.children[7].hinted); // IncMod by name + // the registry knows the top-level shapes and the RNG raw frame + const Registry& reg = save_registry(); + CHECK(reg.shape("Summary") && reg.shape("Sim") && reg.shape("Sys") && reg.shape("Player")); + CHECK(reg.shape("RNG") && reg.shape("RNG")->type == Desc::Raw); + CHECK(reg.kind("GameName") == Prim::String && reg.kind("Bats2") == Prim::Int64 && reg.kind("haltv") == Prim::Bool); + CHECK(reg.kind(".") == Prim::None); + CHECK(reg.kind("Team") == Prim::Int); // int in Slot and Player; the frame is a separate shape entry + CHECK(reg.shape("Team") != nullptr); + CHECK(reg.kind("pop") == Prim::Int64 && reg.shape("pop") != nullptr); // int64 in stats, Population frame in Ship +} + +// --- 6. RNG raw frame: body read straight to the END marker --------------------- +static void test_raw_frame() { + Bytes blob(2500, 0xAB); + Writer w; + w.begin("RNG"); + w.raw(".", blob); + w.end(); + w.int32("Map", 1); + std::vector issues; + Stats st; + Node root = read_tree(w.bytes(), &issues, &st); + CHECK(root.children[0].children.size() == 1); + const Node& r = root.children[0].children[0]; + CHECK(r.kind == Kind::Raw && r.name == "." && r.raw.size() == 2503); // 2500 + 3 joint-padding bytes + CHECK_EQ(st.raw_bytes, 2503u); + CHECK(root.children[1].as_int() == 1); + CHECK(write_tree(root) == w.bytes()); +} + +// --- 7. typed shapes: Summary write -> bytes -> read; and hand-checked bytes ------ +static void test_typed_summary() { + shapes::Summary s; + s.gameName = "Test"; + s.turn = 7; + s.numSys = 3; + s.checksum = 99; + shapes::PlayerInfo p; + p.slot.isPlay = true; + p.slot.fxNm = "re"; + p.slot.fxCrID.idx = -1; + p.slot.fxCrID.r = 10; + p.slot.fxCrID.g = 20; + p.slot.fxCrID.b = 30; + p.slot.tag = 1466349286; + p.slot.team = -1; + p.slot.settings.treasury = 50000; + p.rank = 2; + s.players.push_back(p); + s.session.tmrs.tctl = 240.f; + s.incMod = 1.f; + s.resMod = 1.f; + s.alliances = true; + s.encounters = true; + + Writer w; + WriteArchive wa(w); + wa.obj(A("Summary"), s); + Bytes bytes = w.take(); + + // spot-check the head bytes by hand: frame tag, GameName, Turn + Bytes head = B({7, 0, 0, 0, 'S', 'u', 'm', 'm', 'a', 'r', 'y', 0, 0xef, 0xbe, 0xef, 0xbe, + 8, 0, 0, 0, 'G', 'a', 'm', 'e', 'N', 'a', 'm', 'e', 4, 0, 0, 0, 'T', 'e', 's', 't', + 4, 0, 0, 0, 'T', 'u', 'r', 'n', 7, 0, 0, 0}); + CHECK(bytes.size() > head.size() && std::equal(head.begin(), head.end(), bytes.begin())); + + std::vector issues; + Stats st; + Node root = read_tree(bytes, &issues, &st); + CHECK_EQ(st.resyncs, 0u); + CHECK_EQ(st.hint_failures, 0u); + shapes::Summary back; + ReadArchive ra(root.children, issues, ""); + ra.obj(A("Summary"), back); + CHECK_EQ(count(issues, Issue::Error), size_t(0)); + CHECK_EQ(count(issues, Issue::Warn), size_t(0)); + CHECK(back.gameName == "Test" && back.turn == 7 && back.numSys == 3 && back.checksum == 99); + CHECK(back.players.size() == 1 && back.players[0].rank == 2); + CHECK(back.players[0].slot.fxNm == "re" && back.players[0].slot.tag == 1466349286); + CHECK(back.players[0].slot.fxCrID.idx == -1 && back.players[0].slot.fxCrID.b == 30); + CHECK(back.players[0].slot.settings.treasury == 50000); + CHECK(back.session.tmrs.tctl == 240.f && back.incMod == 1.f && back.alliances && !back.teams); + CHECK(back.scenario.empty()); + + // and back out: byte-identical + Writer w2; + WriteArchive wa2(w2); + wa2.obj(A("Summary"), back); + CHECK(w2.bytes() == bytes); + + // the "." positional items are reported as info, never warn + CHECK(count(issues, Issue::Info) >= 4); // FxCrID idx/r/g/b, Settings x4, Players count/elements + + // a missing confirmed field is an error; an unexpected item before it a warning + Writer w3; + w3.begin("Summary"); + w3.string("GameName", "g"); + w3.int32("Bogus", 1); + w3.int32("Turn", 2); + w3.end(); + Node r3 = read_tree(w3.bytes()); + std::vector i3; + shapes::Summary s3; + ReadArchive ra3(r3.children, i3, ""); + ra3.obj(A("Summary"), s3); + CHECK(s3.turn == 2); + CHECK(count(i3, Issue::Warn) >= 1); + CHECK(count(i3, Issue::Error) >= 1); // NumSys and the rest are missing +} + +// --- 8. conditionals, optionals and inline arrays through the archives ------------ +static void test_typed_conditionals() { + shapes::Ship ship; + ship.desID = 5; + ship.hbq = true; + shapes::BuildOrder o; + o.desID = 9; + ship.bq2.orders.push_back(o); + ship.hsp = false; + ship.prisH.prMax = 2; + ship.prisH.prisoners.push_back({3, 4}); + ship.thrusters.push_back({1.5f, 2.5f}); + Writer w; + WriteArchive wa(w); + wa.obj(A("Ship"), ship); + Node root = read_tree(w.bytes()); + std::vector issues; + shapes::Ship back; + ReadArchive ra(root.children, issues, ""); + ra.obj(A("Ship"), back); + CHECK_EQ(count(issues, Issue::Error), size_t(0)); + CHECK_EQ(count(issues, Issue::Warn), size_t(0)); + CHECK(back.hbq && back.bq2.orders.size() == 1 && back.bq2.orders[0].desID == 9); + CHECK(!back.hsp && back.pop.groups.empty()); + CHECK(back.prisH.prMax == 2 && back.prisH.prisoners.size() == 1 && back.prisH.prisoners[0].prNum == 4); + CHECK(back.thrusters.size() == 1 && back.thrusters[0].thm == 2.5f); + Writer w2; + WriteArchive wa2(w2); + wa2.obj(A("Ship"), back); + CHECK(w2.bytes() == w.bytes()); + + // Fleet: optional legacy tags absent, HFPlan false -> no FPlan; Sys: vnh gate + shapes::Fleet f; + f.ftName = "Fleet 1"; + f.hfPlan = true; + shapes::Waypoint wp; + wp.wpt = 272; + f.fplan.wpts.push_back(wp); + Writer w3; + WriteArchive wa3(w3); + wa3.obj(A("Flt"), f); + Node r3 = read_tree(w3.bytes()); + shapes::Fleet fb; + ReadArchive ra3(r3.children, issues, ""); + ra3.obj(A("Flt"), fb); + CHECK(fb.hfPlan && fb.fplan.wpts.size() == 1 && fb.fplan.wpts[0].wpt == 272 && fb.fplan.wpts[0].nrtFramed); + CHECK(!fb.sysID && !fb.caps); + CHECK(fb.ftName == "Fleet 1"); + Writer w4; + WriteArchive wa4(w4); + wa4.obj(A("Flt"), fb); + CHECK(w4.bytes() == w3.bytes()); + CHECK_EQ(count(issues, Issue::Error), size_t(0)); +} + +// --- 9. gzip container --------------------------------------------------------------- +static void test_gzip() { + Bytes data; + for (int i = 0; i < 5000; ++i) data.push_back(uint8_t(i * 7)); + Bytes gz = gzip(data.data(), data.size()); + CHECK(is_gzip(gz.data(), gz.size())); + CHECK(gunzip(gz.data(), gz.size()) == data); + CHECK(inflate_container(data.data(), data.size()) == data); // not gzip: passthrough + bool threw = false; + try { + Bytes bad = gz; + bad[bad.size() / 2] ^= 0xff; + gunzip(bad.data(), bad.size()); + } catch (const GzipError&) { + threw = true; + } + CHECK(threw); +} + +// --- 10. dump formatting: Python float repr and JSON quoting --------------------------- +static void test_dump_format() { + CHECK_EQ(py_float_repr(240.0), std::string("240.0")); + CHECK_EQ(py_float_repr(1.0), std::string("1.0")); + CHECK_EQ(py_float_repr(0.0), std::string("0.0")); + CHECK_EQ(py_float_repr(double(3.4028235e38f)), std::string("3.4028234663852886e+38")); + CHECK_EQ(py_float_repr(double(bits_f32(0x0000002a))), std::string("5.885453550164232e-44")); + CHECK_EQ(py_float_repr(1e-05), std::string("1e-05")); + CHECK_EQ(py_float_repr(0.0001), std::string("0.0001")); + CHECK_EQ(py_float_repr(1e16), std::string("1e+16")); + CHECK_EQ(py_float_repr(1234567890123456.0), std::string("1234567890123456.0")); + CHECK_EQ(py_float_repr(double(-0.18487215f)), std::string("-0.18487215042114258")); + CHECK_EQ(py_float_repr(double(2.4307494f)), std::string("2.4307494163513184")); + CHECK_EQ(json_quote_cp1252("a\"b\\c\n"), std::string("\"a\\\"b\\\\c\\n\"")); + CHECK_EQ(json_quote_cp1252(std::string("Kor\x92Voth")), std::string("\"Kor\\u2019Voth\"")); + CHECK_EQ(json_quote_cp1252(std::string("\x81")), std::string("\"\\ufffd\"")); + CHECK_EQ(json_quote_cp1252(std::string("caf\xe9")), std::string("\"caf\\u00e9\"")); +} + +int main() { + test_primitives_bytes(); + test_frames(); + test_resync(); + test_cp1252(); + test_hints(); + test_raw_frame(); + test_typed_summary(); + test_typed_conditionals(); + test_gzip(); + test_dump_format(); + std::printf("test_stream: %s (%d failures)\n", fails ? "FAILED" : "ok", fails); + return fails ? 1 : 0; +}