merge mars/stream + mars/rng; single vendored miniz (3.1.2); wire into builds
This commit is contained in:
alex
commit
563dbe3a87
28 changed files with 5022 additions and 1 deletions
|
|
@ -22,6 +22,8 @@ add_subdirectory(src/mars/parse) # brace-block + .effect readers (lib mars_p
|
|||
add_subdirectory(src/mars/text) # flat-kv, id-manifest, csv (lib mars_text)
|
||||
add_subdirectory(src/game/sim) # strategic formulas, pure (lib sots_game_sim)
|
||||
add_subdirectory(src/mars/vfs) # .gob ZIP reader + native override (lib mars_vfs, miniz)
|
||||
add_subdirectory(src/mars/stream) # Streamable save format + gzip (lib mars_stream)
|
||||
add_subdirectory(src/mars/rng) # MT19937 (lib mars_rng)
|
||||
|
||||
if(WIN32)
|
||||
# ---- shim: proxy binkw32.dll that the original game loads (Phase 2 frontend) ----
|
||||
|
|
@ -45,7 +47,7 @@ else()
|
|||
add_executable(addr_smoke tests/addr_smoke.cpp)
|
||||
target_link_libraries(addr_smoke PRIVATE sots_addresses)
|
||||
add_test(NAME addr_smoke COMMAND addr_smoke)
|
||||
foreach(_t mars_parse mars_text game_sim mars_vfs)
|
||||
foreach(_t mars_parse mars_text game_sim mars_vfs mars_stream)
|
||||
if(EXISTS ${CMAKE_SOURCE_DIR}/tests/${_t}/CMakeLists.txt)
|
||||
add_subdirectory(tests/${_t})
|
||||
endif()
|
||||
|
|
|
|||
60
docs/mars-rng.md
Normal file
60
docs/mars-rng.md
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
# mars::rng — the engine PRNG
|
||||
|
||||
`src/mars/rng/mt19937.h` — a textbook 32-bit Mersenne Twister (MT19937), the generator the
|
||||
strategy simulation draws every roll from (map generation, research, encounters, raids, …).
|
||||
Because all lockstep peers share one seeded stream, the reimplementation has to be bit-exact
|
||||
and consume words in the same order; the save file carries the generator state verbatim.
|
||||
|
||||
## API
|
||||
|
||||
```cpp
|
||||
mars::rng::MT19937 r(seed); // seed(): Knuth initializer, then one twist (left == 624)
|
||||
uint32_t y = r.next_u32(); // tempered output
|
||||
float f = r.next_float(); // (float)(y * 2^-32), see below
|
||||
uint32_t k = r.next_int(n); // uniform [0, n): power-of-two mask + rejection
|
||||
r.load_state(mt, left); // or load_state(blob, 0x9c4) from a save's "RNG" item
|
||||
r.save_state(out); // mt[624] + left, 0x9c4 bytes little-endian
|
||||
r.left(); r.index(); r.state();
|
||||
```
|
||||
|
||||
## State model
|
||||
|
||||
| member | meaning |
|
||||
|---|---|
|
||||
| `mt[624]` | the untempered state block |
|
||||
| `left` | words still unread in the current block; next output is `mt[624 - left]` |
|
||||
|
||||
`next_u32()` twists when `left` is 0, hands out `mt[624 - left]`, decrements `left`, and
|
||||
tempers. A freshly seeded generator has already twisted once, so `left == 624` and the
|
||||
first draw is `mt[0]`.
|
||||
|
||||
## Serialized form (the save's `RNG` frame)
|
||||
|
||||
`Sim → RNG { "." raw[2503] }`: 624 × uint32 (`mt`) followed by one int32 (`left`) = 0x9c4 =
|
||||
2500 bytes, plus the 3 joint-padding bytes of the item. `MT19937::load_state(blob, n)` parses
|
||||
it and rejects `left` outside 0..624; `save_state` writes the same layout.
|
||||
|
||||
**Verified on the real saves** (`tests/mars_stream/test_save.cpp`): the 624-word block in all
|
||||
three saves equals `seed(CreateParams.RSeed)` followed by exactly two whole twists, and `left`
|
||||
decreases turn over turn (454 → 432 → 413, i.e. ~20 draws per turn). That confirms the
|
||||
initializer, the twist, the seed source (`RSeed`) and the blob layout. It does not exercise
|
||||
the tempering or the float mapping (those never touch the saved state).
|
||||
|
||||
## Reference vectors (`tests/mars_stream/test_rng.cpp`)
|
||||
|
||||
* seed 5489 → 3499211612, 581869302, 3890346734, … ; the 10000th output is 4123659995.
|
||||
* `save_state`/`load_state` round trip, `left` positioning, malformed-blob rejection.
|
||||
|
||||
## Choices that still need binary confirmation
|
||||
|
||||
1. **Float mapping.** `next_float()` returns `(float)((double)y * 2^-32)`. This is the mapping
|
||||
recorded in the RE notes for the engine's float roll (product in double, then narrowed).
|
||||
Note the narrowing rounds `y >= 0xFFFFFF80` up to exactly `1.0f`, so the range is `[0, 1]`
|
||||
in practice. Confirm against a captured sequence before relying on the exact bits.
|
||||
2. **Twist timing at the block boundary.** We twist lazily when `left` reaches 0 (so a saved
|
||||
state may carry `left == 0`). If the original twists eagerly after the last word of a block
|
||||
(`left` then never saved as 0, block already advanced), the output sequence is identical but
|
||||
the saved blob at that one boundary differs. The three saves (`left` = 454/432/413) do not
|
||||
distinguish the two.
|
||||
3. **`next_int(n)`** — mask = smallest `2^k - 1 >= n - 1`, reject while `r >= n`. The rejection
|
||||
scheme matches the RE description; the exact mask computation is unconfirmed.
|
||||
122
docs/mars-stream.md
Normal file
122
docs/mars-stream.md
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
# mars::stream — Streamable serialization (save files)
|
||||
|
||||
`src/mars/stream/` reimplements the engine's self-describing "Streamable" stream as used by
|
||||
`.sav` files: a generic walker that recovers the item tree from any stream, a writer that
|
||||
emits the same framing, and typed shapes for the confirmed top-level structures. The format
|
||||
facts come from the RE repo's confirmed description (`verify/save-reader/SAVE_FORMAT.md`);
|
||||
the reference Python reader is the oracle the C++ is tested against.
|
||||
|
||||
## The format in one screen
|
||||
|
||||
```
|
||||
container one gzip member; everything below is the inflated stream, little-endian
|
||||
item [int32 len][name bytes][value][NUL pad] pad brings the WHOLE item to 4 bytes
|
||||
("joint" padding; a NULL name is written as ".")
|
||||
scalars int32 | float32 | bool (1 byte) | int64 | string = [int32 len][cp1252 bytes]
|
||||
(an empty string is 4 zero bytes — byte-identical to int 0; no type bytes anywhere)
|
||||
frame [len][name][pad] BE EF BE EF ...items... 10 41 10 41 (0x41104110 = ~0xBEEFBEEF)
|
||||
arrays VectorHelper<T>: a frame holding "." count + n × "." elements (frames or scalars)
|
||||
inline arrays: a named count followed by n × element in the same frame
|
||||
Vector3 a frame of 3 × "." float
|
||||
root Summary → CreateParams → Sim → CDT → n × CD
|
||||
RNG Sim.RNG { "." raw[2503] }: MT19937 mt[624] + left (0x9c4) + 3 pad bytes (see mars-rng.md)
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
| file | contents |
|
||||
|---|---|
|
||||
| `bytes.h` | LE read/write helpers, `pad4`, the two markers |
|
||||
| `node.h` | `Node{name, tagged, kind, raw, hinted, offset, size, children}`, `Issue`, `Stats` |
|
||||
| `gzip.h/.cpp` | `gunzip`, `inflate_container` (passthrough for already-inflated data), `gzip` — via vendored miniz |
|
||||
| `reader.h/.cpp` | `Walker` — the generic tokenizer; `read_tree()` |
|
||||
| `writer.h/.cpp` | `Writer` — scalar/frame/raw emitters with joint padding; `Writer::node()` re-emits a tree |
|
||||
| `dump.h/.cpp` | `dump_tree()` text form (identical to the reference `--dump`), Python float repr, JSON quoting |
|
||||
| `schema.h` | `Desc`/`Hint`/`Registry` — the type hints the walker consults |
|
||||
| `archive.h` | `ReadArchive`, `WriteArchive`, `SchemaBuilder` — drive a shape's `io()` field list |
|
||||
| `shapes.h` | the typed shapes (Summary, CreateParams/MapP, Sim, Player, Sys, Fleet, Ship, …) |
|
||||
| `save.h/.cpp` | `read_save_file/bytes()` → `SaveDocument{inflated, tree, game, issues, stats}`, `write_save()`, `write_tree()`, `save_registry()` |
|
||||
| `savedump_main.cpp` | `sots_savedump SAVE [--dump] [--strict] [--roundtrip] [--rewrite OUT]` |
|
||||
|
||||
## Walker (generic reader)
|
||||
|
||||
The walker needs no schema. Frames are hard synchronisation points; between markers each
|
||||
item's scalar layout is chosen by
|
||||
|
||||
1. a **hint** — positional (`Desc::prefix`, from the shape describing the frame) or by tag
|
||||
(`Desc::by_name`, then the global tag → kind catalog), or
|
||||
2. **guessing**: try `[word, bool, string, int64]` and keep the first layout after which
|
||||
another plausible item, a marker or EOF follows (two items of lookahead; string values
|
||||
only text-checked while guessing, since a known string may hold any cp1252 byte); a bare
|
||||
4-byte word is int if `|i| <= 100000`, float if finite with `1e-6 <= |f| < 1e12`, else int.
|
||||
|
||||
Nothing readable is lost: when no layout fits, the bytes up to the next marker or plausible
|
||||
tag become a `raw` node (a warning), a small unnamed payload before an END is a `raw` node
|
||||
(info), and the walk resumes. Every node keeps its exact value bytes and offset, so
|
||||
`write_tree(read_tree(x)) == x` for any input the walker accepts, damaged or not. These rules
|
||||
are the reference reader's; the test suite checks the two implementations tokenise the three
|
||||
real saves identically (every line of the dump, including the guessed/hinted marks).
|
||||
|
||||
Hints are not hand-maintained: `save_registry()` runs every shape's `io()` under
|
||||
`SchemaBuilder`, which records prefixes, by-name tables and the global catalog (a tag
|
||||
claimed with two different kinds is dropped from the catalog; optional legacy tags register
|
||||
at lowest priority; `"."` is never a hint key because it carries ints, floats and frames alike).
|
||||
|
||||
## Shapes and archives
|
||||
|
||||
A shape is a struct with `kStreamName` and a `template<class Ar> void io(Ar&)` listing its
|
||||
fields in disk order:
|
||||
|
||||
```cpp
|
||||
struct Summary {
|
||||
static constexpr const char* kStreamName = "Summary";
|
||||
std::string gameName; int32_t turn = 0; ...
|
||||
template <class Ar> void io(Ar& ar) {
|
||||
ar.str(A("GameName"), gameName); // A(): confirmed on-disk tag, matched by name
|
||||
ar.i32(A("Turn"), turn);
|
||||
ar.carr(A("Players"), players); // VectorHelper: "." count + "." frames
|
||||
...
|
||||
ar.rest(extra); // anything the shape does not describe, kept generic
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
`R("idx")` fields are positional (the game wrote a NULL name — `"."` on disk — or the
|
||||
reference name differs from the disk spelling, e.g. `faiDes` → `FAIDes`, `ontF` → `otnF`,
|
||||
`nextId` → `nextid`); the second argument of `R` is the tag to write. Conditionals are plain
|
||||
`ar.when(vnh, ...)`, optional legacy tags `ar.opt_i32(A("ARes"), aRes)`, uncounted lists
|
||||
`ar.repeat("stats", ...)`, inline arrays `ar.narr(A("NumSys"), systems, elem)`. Bodies the
|
||||
format keeps opaque (`TechTree`, `Events`, `ShipRecs`, `spy2`, `civr`, `comms`, `Ojvs`,
|
||||
`Attrib`, `sprjs`, `SvSctOb`, `trdmgr`, `spymgr`, `CD`, the RNG blob) are `Node` members and
|
||||
re-emit verbatim.
|
||||
|
||||
`ReadArchive` mirrors the reference applier: a confirmed tag that is missing is an error,
|
||||
unexpected items before it a warning, a positional item read under another tag an info;
|
||||
kinds are coerced with the same width/zero rules (4 zero bytes are the empty string). The
|
||||
same `io()` drives `WriteArchive`, so a shape loaded from a real save writes back
|
||||
**byte-identically** — the whole file, typed shapes plus retained generic bodies.
|
||||
|
||||
## Tests (`tests/mars_stream/build_and_run.sh`, plain g++)
|
||||
|
||||
* `test_stream.cpp` — hand-built byte fixtures: primitive encodings and joint padding (bytes
|
||||
written out by hand and compared with the `Writer`), framing/nesting/tagless/empty frames,
|
||||
`"."` arrays, resync (two fixtures cross-checked with the reference reader), cp1252 values,
|
||||
hints from the registry, the raw RNG frame, typed Summary/Ship/Fleet write → read → write,
|
||||
gzip, Python float repr and JSON quoting.
|
||||
* `test_rng.cpp` — MT19937 reference vectors and state (de)serialization.
|
||||
* `test_save.cpp` — **real saves** when `SOTS_SAVES_DIR` is set (skips otherwise, saves never
|
||||
enter the repo): 0 errors/warnings/resyncs/hint failures, only the 2503-byte RNG blob raw,
|
||||
typed values consistent (Summary ↔ Sim ↔ CreateParams counts, names, 7 species), RNG state
|
||||
reachable from `seed(RSeed)`, tree and typed round trips byte-identical; with
|
||||
`SOTS_SAVE_READER` set, `oracle/compare.py` diffs the C++ dump against the reference
|
||||
`--dump` (exact and canonical agreement, summary values).
|
||||
|
||||
Result on the owner's three saves (SotS 1.8, turns 1–3): 100% exact line agreement, both
|
||||
round trips identical, summaries agree.
|
||||
|
||||
## Not done / open
|
||||
|
||||
* Only joint padding is implemented (the split convention never matched a real file).
|
||||
* Shapes for the opaque bodies listed above (kept generic on purpose, as in the reference).
|
||||
* The float-vs-int guess for unhinted words inside opaque bodies is a heuristic; the typed
|
||||
layer never depends on it (it coerces from the raw bytes).
|
||||
6
src/mars/rng/CMakeLists.txt
Normal file
6
src/mars/rng/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
# mars::rng — the engine PRNG (MT19937).
|
||||
# Included from the root CMakeLists.txt via add_subdirectory(src/mars/rng).
|
||||
|
||||
add_library(mars_rng STATIC mt19937.cpp)
|
||||
target_include_directories(mars_rng PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../..)
|
||||
target_compile_features(mars_rng PUBLIC cxx_std_17)
|
||||
102
src/mars/rng/mt19937.cpp
Normal file
102
src/mars/rng/mt19937.cpp
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
#include "mt19937.h"
|
||||
|
||||
namespace mars::rng {
|
||||
|
||||
namespace {
|
||||
constexpr uint32_t kMatrixA = 0x9908b0dfu;
|
||||
constexpr uint32_t kUpper = 0x80000000u;
|
||||
constexpr uint32_t kLower = 0x7fffffffu;
|
||||
|
||||
inline uint32_t temper(uint32_t y) {
|
||||
y ^= y >> 11;
|
||||
y ^= (y << 7) & 0x9d2c5680u;
|
||||
y ^= (y << 15) & 0xefc60000u;
|
||||
y ^= y >> 18;
|
||||
return y;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void MT19937::seed(uint32_t s) {
|
||||
mt_[0] = s;
|
||||
for (int i = 1; i < N; ++i)
|
||||
mt_[i] = 1812433253u * (mt_[i - 1] ^ (mt_[i - 1] >> 30)) + uint32_t(i);
|
||||
twist();
|
||||
}
|
||||
|
||||
void MT19937::twist() {
|
||||
int kk = 0;
|
||||
for (; kk < N - M; ++kk) {
|
||||
uint32_t y = (mt_[kk] & kUpper) | (mt_[kk + 1] & kLower);
|
||||
mt_[kk] = mt_[kk + M] ^ (y >> 1) ^ ((y & 1u) ? kMatrixA : 0u);
|
||||
}
|
||||
for (; kk < N - 1; ++kk) {
|
||||
uint32_t y = (mt_[kk] & kUpper) | (mt_[kk + 1] & kLower);
|
||||
mt_[kk] = mt_[kk + (M - N)] ^ (y >> 1) ^ ((y & 1u) ? kMatrixA : 0u);
|
||||
}
|
||||
uint32_t y = (mt_[N - 1] & kUpper) | (mt_[0] & kLower);
|
||||
mt_[N - 1] = mt_[M - 1] ^ (y >> 1) ^ ((y & 1u) ? kMatrixA : 0u);
|
||||
left_ = N;
|
||||
}
|
||||
|
||||
uint32_t MT19937::next_u32() {
|
||||
if (left_ <= 0) twist();
|
||||
uint32_t y = mt_[N - left_];
|
||||
--left_;
|
||||
return temper(y);
|
||||
}
|
||||
|
||||
float MT19937::next_float() {
|
||||
return static_cast<float>(static_cast<double>(next_u32()) * (1.0 / 4294967296.0));
|
||||
}
|
||||
|
||||
uint32_t MT19937::next_int(uint32_t n) {
|
||||
if (n <= 1) return 0;
|
||||
uint32_t mask = n - 1;
|
||||
mask |= mask >> 1;
|
||||
mask |= mask >> 2;
|
||||
mask |= mask >> 4;
|
||||
mask |= mask >> 8;
|
||||
mask |= mask >> 16;
|
||||
for (;;) {
|
||||
uint32_t r = next_u32() & mask;
|
||||
if (r < n) return r;
|
||||
}
|
||||
}
|
||||
|
||||
void MT19937::load_state(const uint32_t mt[N], int left) {
|
||||
for (int i = 0; i < N; ++i) mt_[i] = mt[i];
|
||||
left_ = left;
|
||||
}
|
||||
|
||||
bool MT19937::load_state(const uint8_t* blob, size_t n) {
|
||||
if (n < kStateBytes) return false;
|
||||
uint32_t tmp[N];
|
||||
for (int i = 0; i < N; ++i) {
|
||||
const uint8_t* p = blob + size_t(i) * 4;
|
||||
tmp[i] = uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24);
|
||||
}
|
||||
const uint8_t* p = blob + size_t(N) * 4;
|
||||
int32_t left = int32_t(uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24));
|
||||
if (left < 0 || left > N) return false;
|
||||
load_state(tmp, left);
|
||||
return true;
|
||||
}
|
||||
|
||||
void MT19937::save_state(uint8_t out[kStateBytes]) const {
|
||||
for (int i = 0; i < N; ++i) {
|
||||
uint32_t v = mt_[i];
|
||||
uint8_t* p = out + size_t(i) * 4;
|
||||
p[0] = uint8_t(v);
|
||||
p[1] = uint8_t(v >> 8);
|
||||
p[2] = uint8_t(v >> 16);
|
||||
p[3] = uint8_t(v >> 24);
|
||||
}
|
||||
uint32_t v = uint32_t(left_);
|
||||
uint8_t* p = out + size_t(N) * 4;
|
||||
p[0] = uint8_t(v);
|
||||
p[1] = uint8_t(v >> 8);
|
||||
p[2] = uint8_t(v >> 16);
|
||||
p[3] = uint8_t(v >> 24);
|
||||
}
|
||||
|
||||
} // namespace mars::rng
|
||||
64
src/mars/rng/mt19937.h
Normal file
64
src/mars/rng/mt19937.h
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
// mars::rng — the engine's PRNG: a textbook 32-bit Mersenne Twister (MT19937).
|
||||
//
|
||||
// State model (matches the save-file blob, see docs/mars-rng.md):
|
||||
// uint32_t mt[624] the untempered state block
|
||||
// int left words still unread in the current block; the next
|
||||
// word to hand out is mt[624 - left]
|
||||
// The serialized form is exactly mt[624] followed by left as an int32
|
||||
// (0x9c4 = 2500 bytes). A fresh generator seeds the block with the standard
|
||||
// Knuth-style initializer and twists once immediately, so left == 624 right
|
||||
// after seeding.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace mars::rng {
|
||||
|
||||
class MT19937 {
|
||||
public:
|
||||
static constexpr int N = 624;
|
||||
static constexpr int M = 397;
|
||||
static constexpr size_t kStateBytes = size_t(N) * 4 + 4; // 0x9c4
|
||||
|
||||
explicit MT19937(uint32_t seed = 5489u) { this->seed(seed); }
|
||||
|
||||
// mt[0] = seed; mt[i] = 1812433253 * (mt[i-1] ^ (mt[i-1] >> 30)) + i; then twist.
|
||||
void seed(uint32_t s);
|
||||
|
||||
// Next tempered 32-bit output.
|
||||
uint32_t next_u32();
|
||||
|
||||
// Uniform float in [0, 1): (float)(next_u32() * 2^-32), the product formed
|
||||
// in double precision then narrowed. NOTE: the narrowing can round the
|
||||
// largest outputs (y >= 0xFFFFFF80) up to exactly 1.0f. Mapping recorded
|
||||
// from the RE notes; still needs binary confirmation against a captured
|
||||
// sequence (see docs/mars-rng.md).
|
||||
float next_float();
|
||||
|
||||
// Uniform integer in [0, n) by rejection sampling with the smallest
|
||||
// power-of-two mask covering n-1 (n == 0 returns 0). Mask/rejection
|
||||
// details need binary confirmation.
|
||||
uint32_t next_int(uint32_t n);
|
||||
|
||||
// --- state access / serialization ---------------------------------------
|
||||
const uint32_t* state() const { return mt_; }
|
||||
int left() const { return left_; }
|
||||
int index() const { return N - left_; } // words consumed in the current block
|
||||
|
||||
// Load mt[624] + left from the save's RNG blob layout.
|
||||
void load_state(const uint32_t mt[N], int left);
|
||||
// Parse the 0x9c4-byte blob (little-endian). Returns false if n < kStateBytes
|
||||
// or left is out of range.
|
||||
bool load_state(const uint8_t* blob, size_t n);
|
||||
// Serialize to the same 0x9c4-byte layout.
|
||||
void save_state(uint8_t out[kStateBytes]) const;
|
||||
|
||||
private:
|
||||
uint32_t mt_[N];
|
||||
int left_ = 0;
|
||||
|
||||
void twist();
|
||||
};
|
||||
|
||||
} // namespace mars::rng
|
||||
22
src/mars/stream/CMakeLists.txt
Normal file
22
src/mars/stream/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# mars::stream — Streamable serialization (save-file format) + gzip container.
|
||||
# Included from the root CMakeLists.txt via add_subdirectory(src/mars/stream).
|
||||
|
||||
if(NOT TARGET miniz) # mars/vfs defines the same vendored target when added first
|
||||
add_library(miniz STATIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party/miniz/miniz.c)
|
||||
target_include_directories(miniz PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party/miniz)
|
||||
target_compile_definitions(miniz PUBLIC MINIZ_NO_STDIO MINIZ_NO_ARCHIVE_APIS MINIZ_NO_TIME)
|
||||
set_target_properties(miniz PROPERTIES LINKER_LANGUAGE C)
|
||||
endif()
|
||||
|
||||
add_library(mars_stream STATIC
|
||||
gzip.cpp
|
||||
reader.cpp
|
||||
writer.cpp
|
||||
dump.cpp
|
||||
save.cpp)
|
||||
target_include_directories(mars_stream PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../..)
|
||||
target_link_libraries(mars_stream PUBLIC miniz)
|
||||
target_compile_features(mars_stream PUBLIC cxx_std_17)
|
||||
|
||||
add_executable(sots_savedump savedump_main.cpp)
|
||||
target_link_libraries(sots_savedump PRIVATE mars_stream)
|
||||
684
src/mars/stream/archive.h
Normal file
684
src/mars/stream/archive.h
Normal file
|
|
@ -0,0 +1,684 @@
|
|||
// mars::stream — archives that drive a typed shape's io() field list.
|
||||
//
|
||||
// A shape is a plain struct with
|
||||
// static constexpr const char* kStreamName = "Sys"; // "" when the frame is unnamed ("." elements)
|
||||
// template <class Ar> void io(Ar& ar) { ar.i32(A("Idx"), idx); ... }
|
||||
// listing its fields in on-disk order. The same io() is driven by three
|
||||
// archives:
|
||||
// ReadArchive fills the struct from a generic Node tree (walker output),
|
||||
// matching tags by name (A) or position (R), coercing kinds
|
||||
// and reporting deviations as Issues;
|
||||
// WriteArchive emits the struct through a Writer in the same order;
|
||||
// SchemaBuilder records the field list as a Desc so the walker can type the
|
||||
// frame's children before any shape is applied.
|
||||
//
|
||||
// Tag helpers: A("Turn") = on-disk tag confirmed, matched by name.
|
||||
// R("treasury") = reference/positional field, written as "." (or
|
||||
// the given disk tag); matched by position.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <typeindex>
|
||||
#include <vector>
|
||||
|
||||
#include "node.h"
|
||||
#include "schema.h"
|
||||
#include "writer.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
struct Tag {
|
||||
const char* name; // schema name
|
||||
const char* disk; // tag emitted when writing
|
||||
bool positional;
|
||||
};
|
||||
constexpr Tag A(const char* n) { return Tag{n, n, false}; }
|
||||
constexpr Tag R(const char* n, const char* disk = ".") { return Tag{n, disk, true}; }
|
||||
|
||||
struct Vec3 {
|
||||
float x = 0, y = 0, z = 0;
|
||||
bool operator==(const Vec3& o) const { return x == o.x && y == o.y && z == o.z; }
|
||||
};
|
||||
|
||||
template <class T, class = void>
|
||||
struct has_io : std::false_type {};
|
||||
template <class T>
|
||||
struct has_io<T, std::void_t<decltype(T::kStreamName)>> : std::true_type {};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ReadArchive
|
||||
// ---------------------------------------------------------------------------
|
||||
class ReadArchive {
|
||||
public:
|
||||
static constexpr bool reading = true, writing = false, building = false;
|
||||
|
||||
ReadArchive(const std::vector<Node>& nodes, std::vector<Issue>& issues, std::string path)
|
||||
: nodes_(&nodes), issues_(&issues), path_(std::move(path)) {}
|
||||
|
||||
// --- scalars ------------------------------------------------------------
|
||||
void i32(Tag t, int32_t& v) {
|
||||
if (const Node* n = take_named(t)) v = coerce_int(*n, fpath(t));
|
||||
}
|
||||
void f32(Tag t, float& v) {
|
||||
if (const Node* n = take_named(t)) v = coerce_float(*n, fpath(t));
|
||||
}
|
||||
void b(Tag t, bool& v) {
|
||||
if (const Node* n = take_named(t)) v = coerce_bool(*n, fpath(t));
|
||||
}
|
||||
void i64(Tag t, int64_t& v) {
|
||||
if (const Node* n = take_named(t)) v = coerce_int64(*n, fpath(t));
|
||||
}
|
||||
void str(Tag t, std::string& v) {
|
||||
if (const Node* n = take_named(t)) v = coerce_string(*n, fpath(t));
|
||||
}
|
||||
void vec3(Tag t, Vec3& v) {
|
||||
if (const Node* n = take_named(t)) v = coerce_vec3(*n, fpath(t));
|
||||
}
|
||||
// generic item kept as a node copy ("any")
|
||||
void any(Tag t, Node& v) {
|
||||
if (const Node* n = take_named(t)) v = *n;
|
||||
}
|
||||
void raw_frame(Tag t, Node& v) { any(t, v); }
|
||||
|
||||
// --- optional named items (consumed only when the next tag matches) ------
|
||||
void opt_i32(Tag t, std::optional<int32_t>& v) {
|
||||
if (next_is(t)) v = coerce_int(*take(), fpath(t));
|
||||
}
|
||||
void opt_f32(Tag t, std::optional<float>& v) {
|
||||
if (next_is(t)) v = coerce_float(*take(), fpath(t));
|
||||
}
|
||||
void opt_b(Tag t, std::optional<bool>& v) {
|
||||
if (next_is(t)) v = coerce_bool(*take(), fpath(t));
|
||||
}
|
||||
void opt_any(Tag t, std::optional<Node>& v) {
|
||||
if (next_is(t)) v = *take();
|
||||
}
|
||||
template <class T>
|
||||
void opt_obj(Tag t, std::optional<T>& v) {
|
||||
if (next_is(t)) {
|
||||
v.emplace();
|
||||
obj(t, *v);
|
||||
}
|
||||
}
|
||||
|
||||
// --- framed struct ---------------------------------------------------------
|
||||
template <class T>
|
||||
void obj(Tag t, T& v) {
|
||||
const Node* n = take_named(t);
|
||||
if (!n) return;
|
||||
read_frame(*n, v, fpath(t));
|
||||
}
|
||||
// struct that may also appear inline (unframed); `framed` records which
|
||||
template <class T>
|
||||
void obj_flex(Tag t, T& v, bool& framed) {
|
||||
const Node* n = peek();
|
||||
if (n && !n->is_complex()) {
|
||||
framed = false;
|
||||
v.io(*this);
|
||||
return;
|
||||
}
|
||||
framed = true;
|
||||
obj(t, v);
|
||||
}
|
||||
|
||||
// --- framed array: "." count + n x "." element -----------------------------
|
||||
template <class T>
|
||||
void carr(Tag t, std::vector<T>& v) {
|
||||
const Node* n = take_named(t);
|
||||
if (!n) return;
|
||||
std::string p = fpath(t);
|
||||
if (!n->is_complex()) {
|
||||
issue(Issue::Error, p, n, "expected framed array, found " + std::string(kind_name(n->kind)) + " '" +
|
||||
n->name + "'");
|
||||
return;
|
||||
}
|
||||
ReadArchive sub(n->children, *issues_, p);
|
||||
v.clear();
|
||||
if (sub.done()) return;
|
||||
int32_t count = coerce_int(*sub.take(), p);
|
||||
size_t remaining = sub.remaining();
|
||||
if (count < 0 || size_t(count) > remaining) {
|
||||
issue(Issue::Error, p, n,
|
||||
"framed-array count " + std::to_string(count) + " exceeds " + std::to_string(remaining) + " item(s)");
|
||||
count = int32_t(std::min<size_t>(size_t(std::max(count, 0)), remaining));
|
||||
}
|
||||
for (int32_t i = 0; i < count; ++i) {
|
||||
if (sub.done()) {
|
||||
issue(Issue::Error, p, n, "framed array truncated: " + std::to_string(i) + " of " + std::to_string(count));
|
||||
break;
|
||||
}
|
||||
v.emplace_back();
|
||||
sub.read_elem(v.back(), p + "[" + std::to_string(i) + "]");
|
||||
}
|
||||
if (!sub.done())
|
||||
issue(Issue::Warn, p, sub.peek(),
|
||||
std::to_string(sub.remaining()) + " item(s) after framed array elements");
|
||||
}
|
||||
template <class T>
|
||||
void carr_flex(Tag t, std::vector<T>& v, bool& framed) {
|
||||
const Node* n = peek();
|
||||
if (n && !n->is_complex()) {
|
||||
framed = false;
|
||||
narr(t, v, [](ReadArchive& a, T& e) { a.read_elem(e, a.path_); });
|
||||
return;
|
||||
}
|
||||
framed = true;
|
||||
carr(t, v);
|
||||
}
|
||||
|
||||
// --- inline array: named count + n x element -------------------------------
|
||||
template <class T, class F>
|
||||
void narr(Tag t, std::vector<T>& v, F elem) {
|
||||
const Node* cn = take_named(t);
|
||||
if (!cn) return;
|
||||
std::string p = fpath(t);
|
||||
int32_t count = coerce_int(*cn, p);
|
||||
size_t rem = remaining();
|
||||
v.clear();
|
||||
if (count < 0 || size_t(count) > rem) {
|
||||
issue(Issue::Error, p, cn,
|
||||
"array count " + std::to_string(count) + " exceeds the " + std::to_string(rem) +
|
||||
" item(s) left in the frame");
|
||||
count = int32_t(std::min<size_t>(size_t(std::max(count, 0)), rem));
|
||||
}
|
||||
for (int32_t i = 0; i < count; ++i) {
|
||||
if (done()) {
|
||||
issue(Issue::Error, p, nullptr, "array truncated: " + std::to_string(i) + " of " + std::to_string(count) +
|
||||
" elements present");
|
||||
break;
|
||||
}
|
||||
size_t before = i_;
|
||||
v.emplace_back();
|
||||
std::string save = path_;
|
||||
path_ = p + "[" + std::to_string(i) + "]";
|
||||
elem(*this, v.back());
|
||||
path_ = save;
|
||||
if (i_ == before) {
|
||||
issue(Issue::Error, p, peek(), "array element " + std::to_string(i) + " consumed nothing; stopping");
|
||||
v.pop_back();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- conditional group -----------------------------------------------------
|
||||
template <class F>
|
||||
void when(bool cond, F body) {
|
||||
if (cond) body(*this);
|
||||
}
|
||||
|
||||
// --- uncounted repetition while the next tag is `lead` ---------------------
|
||||
template <class T, class F>
|
||||
void repeat(const char* lead, std::vector<T>& v, F elem) {
|
||||
v.clear();
|
||||
while (!done() && peek()->tagged && peek()->name == lead) {
|
||||
size_t before = i_;
|
||||
v.emplace_back();
|
||||
std::string save = path_;
|
||||
path_ = path_ + "/" + lead + "[" + std::to_string(v.size() - 1) + "]";
|
||||
elem(*this, v.back());
|
||||
path_ = save;
|
||||
if (i_ == before) {
|
||||
v.pop_back();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- everything left in the frame, kept generic ----------------------------
|
||||
void rest(std::vector<Node>& v) {
|
||||
v.assign(nodes_->begin() + long(i_), nodes_->end());
|
||||
i_ = nodes_->size();
|
||||
absorbed_ = true;
|
||||
}
|
||||
|
||||
// --- cursor ----------------------------------------------------------------
|
||||
bool done() const { return i_ >= nodes_->size(); }
|
||||
size_t remaining() const { return nodes_->size() - i_; }
|
||||
const Node* peek() const { return done() ? nullptr : &(*nodes_)[i_]; }
|
||||
const Node* take() { return &(*nodes_)[i_++]; }
|
||||
|
||||
// After io(): leftover items are unexpected unless rest() absorbed them.
|
||||
void finish(const Node* frame) {
|
||||
if (!done() && !absorbed_) {
|
||||
issue(Issue::Warn, path_, peek(), std::to_string(remaining()) + " unexpected item(s) at end of frame");
|
||||
(void)frame;
|
||||
i_ = nodes_->size();
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void read_frame(const Node& n, T& v, const std::string& p) {
|
||||
if (!n.is_complex()) {
|
||||
issue(Issue::Error, p, &n, "expected frame, found " + std::string(kind_name(n.kind)) + " '" + n.name + "'");
|
||||
return;
|
||||
}
|
||||
ReadArchive sub(n.children, *issues_, p);
|
||||
v.io(sub);
|
||||
sub.finish(&n);
|
||||
}
|
||||
|
||||
// one array element: struct -> "." frame, int32 -> "." int, Node -> any
|
||||
template <class T>
|
||||
void read_elem(T& e, const std::string& p) {
|
||||
const Node* n = take();
|
||||
if constexpr (std::is_same_v<T, int32_t>) e = coerce_int(*n, p);
|
||||
else if constexpr (std::is_same_v<T, Node>) e = *n;
|
||||
else read_frame(*n, e, p);
|
||||
}
|
||||
|
||||
private:
|
||||
const std::vector<Node>* nodes_;
|
||||
size_t i_ = 0;
|
||||
std::vector<Issue>* issues_;
|
||||
std::string path_;
|
||||
bool absorbed_ = false;
|
||||
|
||||
std::string fpath(Tag t) const { return path_ + "/" + t.name; }
|
||||
|
||||
void issue(Issue::Level l, const std::string& p, const Node* n, std::string msg) {
|
||||
issues_->push_back(Issue{l, p, n ? n->offset : 0u, std::move(msg)});
|
||||
}
|
||||
static std::string lower(std::string s) {
|
||||
for (char& c : s)
|
||||
if (c >= 'A' && c <= 'Z') c = char(c - 'A' + 'a');
|
||||
return s;
|
||||
}
|
||||
bool next_is(Tag t) const {
|
||||
const Node* n = peek();
|
||||
return n && n->tagged && lower(n->name) == lower(t.name);
|
||||
}
|
||||
|
||||
const Node* take_named(Tag t) {
|
||||
std::string p = fpath(t);
|
||||
const Node* n = peek();
|
||||
if (!n) {
|
||||
issue(t.positional ? Issue::Warn : Issue::Error, p, nullptr, "missing field '" + std::string(t.name) + "' (frame ended)");
|
||||
return nullptr;
|
||||
}
|
||||
if (!t.positional) {
|
||||
if (!n->tagged || n->name != t.name) {
|
||||
size_t j = i_;
|
||||
for (; j < nodes_->size(); ++j)
|
||||
if ((*nodes_)[j].tagged && (*nodes_)[j].name == t.name) break;
|
||||
if (j == nodes_->size()) {
|
||||
issue(Issue::Error, p, n,
|
||||
"expected '" + std::string(t.name) + "', found '" + (n->tagged ? n->name : "<tagless>") +
|
||||
"'; field missing");
|
||||
return nullptr;
|
||||
}
|
||||
std::string names;
|
||||
for (size_t k = i_; k < j && k < i_ + 6; ++k) names += (k > i_ ? ", '" : "'") + (*nodes_)[k].name + "'";
|
||||
issue(Issue::Warn, p, n,
|
||||
std::to_string(j - i_) + " unexpected item(s) before '" + t.name + "': " + names);
|
||||
i_ = j;
|
||||
}
|
||||
} else if (n->tagged && lower(n->name) != lower(t.name)) {
|
||||
issue(Issue::Info, p, n, "tag '" + n->name + "' read positionally as '" + t.name + "'");
|
||||
}
|
||||
return take();
|
||||
}
|
||||
|
||||
// --- coercions (mirror the reference reader's rules) ----------------------
|
||||
int32_t coerce_int(const Node& n, const std::string& p) {
|
||||
switch (n.kind) {
|
||||
case Kind::Int: return n.as_int();
|
||||
case Kind::Float: return n.raw.size() == 4 ? n.as_int() : fail_int(n, p, "int");
|
||||
case Kind::Bool: return n.as_bool() ? 1 : 0;
|
||||
case Kind::Int64:
|
||||
issue(Issue::Warn, p, &n, "int expected, int64 read");
|
||||
return int32_t(n.as_int64());
|
||||
default: return fail_int(n, p, "int");
|
||||
}
|
||||
}
|
||||
float coerce_float(const Node& n, const std::string& p) {
|
||||
switch (n.kind) {
|
||||
case Kind::Float: return n.as_float();
|
||||
case Kind::Int: return n.raw.size() == 4 ? n.as_float() : float(fail_int(n, p, "float"));
|
||||
case Kind::Bool: return n.as_bool() ? 1.f : 0.f;
|
||||
default: return float(fail_int(n, p, "float"));
|
||||
}
|
||||
}
|
||||
bool coerce_bool(const Node& n, const std::string& p) {
|
||||
if (n.kind == Kind::Bool) return n.as_bool();
|
||||
if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4) {
|
||||
if (n.raw[1] || n.raw[2] || n.raw[3] || n.raw[0] > 1)
|
||||
issue(Issue::Warn, p, &n, "bool expected, word " + hex(n.raw.data(), 4) + " read");
|
||||
return n.raw[0] != 0;
|
||||
}
|
||||
return fail_int(n, p, "bool") != 0;
|
||||
}
|
||||
int64_t coerce_int64(const Node& n, const std::string& p) {
|
||||
if (n.kind == Kind::Int64) return n.as_int64();
|
||||
if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4) {
|
||||
issue(Issue::Warn, p, &n, "int64 expected, 4-byte word read (width mismatch)");
|
||||
return n.as_int();
|
||||
}
|
||||
return fail_int(n, p, "int64");
|
||||
}
|
||||
std::string coerce_string(const Node& n, const std::string& p) {
|
||||
if (n.kind == Kind::String) return n.as_string();
|
||||
// an empty string is "len 0" = 4 zero bytes, byte-identical to int 0
|
||||
if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4 && n.as_int() == 0) return "";
|
||||
fail_int(n, p, "string");
|
||||
return "";
|
||||
}
|
||||
Vec3 coerce_vec3(const Node& n, const std::string& p) {
|
||||
Vec3 v;
|
||||
if (n.is_complex()) {
|
||||
const auto& ch = n.children;
|
||||
if (ch.size() == 1 && ch[0].kind == Kind::Raw && ch[0].raw.size() == 12) {
|
||||
v.x = rd_f32(ch[0].raw.data());
|
||||
v.y = rd_f32(ch[0].raw.data() + 4);
|
||||
v.z = rd_f32(ch[0].raw.data() + 8);
|
||||
return v;
|
||||
}
|
||||
if (ch.size() == 3 && !ch[0].is_complex() && !ch[1].is_complex() && !ch[2].is_complex() &&
|
||||
ch[0].raw.size() == 4 && ch[1].raw.size() == 4 && ch[2].raw.size() == 4) {
|
||||
v.x = ch[0].as_float();
|
||||
v.y = ch[1].as_float();
|
||||
v.z = ch[2].as_float();
|
||||
return v;
|
||||
}
|
||||
issue(Issue::Error, p, &n, "vec3 frame has unexpected body (" + std::to_string(ch.size()) + " items)");
|
||||
return v;
|
||||
}
|
||||
if (n.kind == Kind::Raw && n.raw.size() == 12) {
|
||||
v.x = rd_f32(n.raw.data());
|
||||
v.y = rd_f32(n.raw.data() + 4);
|
||||
v.z = rd_f32(n.raw.data() + 8);
|
||||
return v;
|
||||
}
|
||||
issue(Issue::Error, p, &n, "expected vec3, found " + std::string(kind_name(n.kind)));
|
||||
return v;
|
||||
}
|
||||
int32_t fail_int(const Node& n, const std::string& p, const char* want) {
|
||||
issue(Issue::Error, p, &n,
|
||||
std::string("expected ") + want + ", read " + (n.is_complex() ? "frame '" + n.name + "'" : kind_name(n.kind)));
|
||||
return n.raw.size() >= 4 ? n.as_int() : 0;
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WriteArchive
|
||||
// ---------------------------------------------------------------------------
|
||||
class WriteArchive {
|
||||
public:
|
||||
static constexpr bool reading = false, writing = true, building = false;
|
||||
|
||||
explicit WriteArchive(Writer& w) : w_(w) {}
|
||||
|
||||
void i32(Tag t, int32_t& v) { w_.int32(t.disk, v); }
|
||||
void f32(Tag t, float& v) { w_.float32(t.disk, v); }
|
||||
void b(Tag t, bool& v) { w_.boolean(t.disk, v); }
|
||||
void i64(Tag t, int64_t& v) { w_.int64(t.disk, v); }
|
||||
void str(Tag t, std::string& v) { w_.string(t.disk, v); }
|
||||
void vec3(Tag t, Vec3& v) {
|
||||
w_.begin(t.disk);
|
||||
w_.float32(".", v.x);
|
||||
w_.float32(".", v.y);
|
||||
w_.float32(".", v.z);
|
||||
w_.end();
|
||||
}
|
||||
void any(Tag, Node& v) { w_.node(v); }
|
||||
void raw_frame(Tag t, Node& v) { any(t, v); }
|
||||
|
||||
void opt_i32(Tag t, std::optional<int32_t>& v) {
|
||||
if (v) w_.int32(t.disk, *v);
|
||||
}
|
||||
void opt_f32(Tag t, std::optional<float>& v) {
|
||||
if (v) w_.float32(t.disk, *v);
|
||||
}
|
||||
void opt_b(Tag t, std::optional<bool>& v) {
|
||||
if (v) w_.boolean(t.disk, *v);
|
||||
}
|
||||
void opt_any(Tag, std::optional<Node>& v) {
|
||||
if (v) w_.node(*v);
|
||||
}
|
||||
template <class T>
|
||||
void opt_obj(Tag t, std::optional<T>& v) {
|
||||
if (v) obj(t, *v);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void obj(Tag t, T& v) {
|
||||
w_.begin(t.disk);
|
||||
v.io(*this);
|
||||
w_.end();
|
||||
}
|
||||
template <class T>
|
||||
void obj_flex(Tag t, T& v, bool& framed) {
|
||||
if (framed) obj(t, v);
|
||||
else v.io(*this);
|
||||
}
|
||||
template <class T>
|
||||
void carr(Tag t, std::vector<T>& v) {
|
||||
w_.begin(t.disk);
|
||||
w_.int32(".", int32_t(v.size()));
|
||||
for (T& e : v) write_elem(e);
|
||||
w_.end();
|
||||
}
|
||||
template <class T>
|
||||
void carr_flex(Tag t, std::vector<T>& v, bool& framed) {
|
||||
if (framed) {
|
||||
carr(t, v);
|
||||
return;
|
||||
}
|
||||
w_.int32(t.disk, int32_t(v.size()));
|
||||
for (T& e : v) write_elem(e);
|
||||
}
|
||||
template <class T, class F>
|
||||
void narr(Tag t, std::vector<T>& v, F elem) {
|
||||
w_.int32(t.disk, int32_t(v.size()));
|
||||
for (T& e : v) elem(*this, e);
|
||||
}
|
||||
template <class F>
|
||||
void when(bool cond, F body) {
|
||||
if (cond) body(*this);
|
||||
}
|
||||
template <class T, class F>
|
||||
void repeat(const char*, std::vector<T>& v, F elem) {
|
||||
for (T& e : v) elem(*this, e);
|
||||
}
|
||||
void rest(std::vector<Node>& v) {
|
||||
for (const Node& n : v) w_.node(n);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void write_elem(T& e) {
|
||||
if constexpr (std::is_same_v<T, int32_t>) w_.int32(".", e);
|
||||
else if constexpr (std::is_same_v<T, Node>) w_.node(e);
|
||||
else {
|
||||
w_.begin(".");
|
||||
e.io(*this);
|
||||
w_.end();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
Writer& w_;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SchemaBuilder — runs io() on default-constructed shapes to record hints.
|
||||
// ---------------------------------------------------------------------------
|
||||
class SchemaBuilder {
|
||||
public:
|
||||
static constexpr bool reading = false, writing = false, building = true;
|
||||
|
||||
struct Context {
|
||||
Registry& reg;
|
||||
std::map<std::string, std::set<Prim>> strong; // A/R fields: name -> kinds seen
|
||||
std::map<std::string, Prim> weak; // Opt fields: lowest priority
|
||||
std::map<std::type_index, const Desc*> memo;
|
||||
std::set<std::type_index> in_progress;
|
||||
const Desc* raw_desc = nullptr;
|
||||
explicit Context(Registry& r) : reg(r) {}
|
||||
};
|
||||
|
||||
SchemaBuilder(Context& ctx, Desc* cur) : ctx_(ctx), cur_(cur) {}
|
||||
|
||||
void i32(Tag t, int32_t&) { prim(t, Prim::Int); }
|
||||
void f32(Tag t, float&) { prim(t, Prim::Float); }
|
||||
void b(Tag t, bool&) { prim(t, Prim::Bool); }
|
||||
void i64(Tag t, int64_t&) { prim(t, Prim::Int64); }
|
||||
void str(Tag t, std::string&) { prim(t, Prim::String); }
|
||||
void vec3(Tag t, Vec3&) { nohint(t); }
|
||||
void any(Tag t, Node&) { nohint(t); }
|
||||
void raw_frame(Tag t, Node&) {
|
||||
nohint(t);
|
||||
if (!ctx_.raw_desc) {
|
||||
Desc d;
|
||||
d.type = Desc::Raw;
|
||||
d.name = t.name;
|
||||
ctx_.raw_desc = ctx_.reg.add(std::move(d));
|
||||
}
|
||||
ctx_.reg.shapes.emplace(t.name, ctx_.raw_desc);
|
||||
}
|
||||
|
||||
void opt_i32(Tag t, std::optional<int32_t>&) { prim(t, Prim::Int, true); }
|
||||
void opt_f32(Tag t, std::optional<float>&) { prim(t, Prim::Float, true); }
|
||||
void opt_b(Tag t, std::optional<bool>&) { prim(t, Prim::Bool, true); }
|
||||
void opt_any(Tag t, std::optional<Node>&) {
|
||||
close_prefix();
|
||||
nohint(t);
|
||||
}
|
||||
template <class T>
|
||||
void opt_obj(Tag t, std::optional<T>&) {
|
||||
close_prefix();
|
||||
const Desc* d = describe<T>();
|
||||
by_name(t, Hint{Prim::None, d});
|
||||
register_shape(t, d);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void obj(Tag t, T&) {
|
||||
const Desc* d = describe<T>();
|
||||
push_prefix(Hint{Prim::None, d});
|
||||
by_name(t, Hint{Prim::None, d});
|
||||
register_shape(t, d);
|
||||
}
|
||||
template <class T>
|
||||
void obj_flex(Tag t, T& v, bool&) {
|
||||
close_prefix();
|
||||
obj(t, v);
|
||||
}
|
||||
template <class T>
|
||||
void carr(Tag t, std::vector<T>&) {
|
||||
Desc d;
|
||||
d.type = Desc::CArr;
|
||||
if constexpr (std::is_same_v<T, int32_t>) d.elem.kind = Prim::Int;
|
||||
else if constexpr (std::is_same_v<T, Node>) d.elem = Hint{};
|
||||
else d.elem.sub = describe<T>();
|
||||
const Desc* cd = ctx_.reg.add(std::move(d));
|
||||
push_prefix(Hint{Prim::None, cd});
|
||||
// the NULL-name tag "." is never a frame hint key: it carries ints,
|
||||
// floats and frames alike (bare "." elements inside inline arrays)
|
||||
if (std::string(t.name) != ".") {
|
||||
by_name(t, Hint{Prim::None, cd});
|
||||
if (*t.name) ctx_.reg.shapes.emplace(t.name, cd);
|
||||
}
|
||||
}
|
||||
template <class T>
|
||||
void carr_flex(Tag t, std::vector<T>& v, bool&) {
|
||||
close_prefix();
|
||||
carr(t, v);
|
||||
}
|
||||
template <class T, class F>
|
||||
void narr(Tag t, std::vector<T>&, F elem) {
|
||||
push_prefix(Hint{Prim::Int, nullptr});
|
||||
close_prefix();
|
||||
by_name(t, Hint{});
|
||||
T tmp{};
|
||||
elem(*this, tmp); // element fields register by name (prefix already closed)
|
||||
}
|
||||
template <class F>
|
||||
void when(bool, F body) {
|
||||
close_prefix();
|
||||
body(*this);
|
||||
}
|
||||
template <class T, class F>
|
||||
void repeat(const char*, std::vector<T>&, F elem) {
|
||||
close_prefix();
|
||||
T tmp{};
|
||||
elem(*this, tmp);
|
||||
}
|
||||
void rest(std::vector<Node>&) { close_prefix(); }
|
||||
|
||||
// Describe shape T (memoized per type) and register its named frame.
|
||||
template <class T>
|
||||
const Desc* describe() {
|
||||
std::type_index ti(typeid(T));
|
||||
auto it = ctx_.memo.find(ti);
|
||||
if (it != ctx_.memo.end()) return it->second;
|
||||
if (ctx_.in_progress.count(ti)) return nullptr; // recursive shape: no positional hints
|
||||
ctx_.in_progress.insert(ti);
|
||||
Desc d;
|
||||
d.type = Desc::Shape;
|
||||
d.name = T::kStreamName;
|
||||
Desc* nd = ctx_.reg.add(std::move(d));
|
||||
SchemaBuilder sub(ctx_, nd);
|
||||
T tmp{};
|
||||
tmp.io(sub);
|
||||
ctx_.in_progress.erase(ti);
|
||||
ctx_.memo.emplace(ti, nd);
|
||||
if (!nd->name.empty()) ctx_.reg.shapes.emplace(nd->name, nd);
|
||||
return nd;
|
||||
}
|
||||
|
||||
// Build the global catalog after every shape has been visited.
|
||||
static void finalize(Context& ctx, const std::map<std::string, Prim>& manual_kinds) {
|
||||
std::set<std::string> conflicts;
|
||||
for (auto& [name, kinds] : ctx.strong) {
|
||||
if (kinds.size() == 1) ctx.reg.kinds.emplace(name, *kinds.begin());
|
||||
else conflicts.insert(name);
|
||||
}
|
||||
for (auto& [name, k] : ctx.weak)
|
||||
if (!conflicts.count(name)) ctx.reg.kinds.emplace(name, k);
|
||||
ctx.reg.kinds.erase("."); // the NULL-name tag carries ints, floats and frames alike
|
||||
for (auto& [name, k] : manual_kinds) ctx.reg.kinds[name] = k;
|
||||
}
|
||||
|
||||
private:
|
||||
Context& ctx_;
|
||||
Desc* cur_;
|
||||
bool prefix_open_ = true;
|
||||
|
||||
static std::string lower(std::string s) {
|
||||
for (char& c : s)
|
||||
if (c >= 'A' && c <= 'Z') c = char(c - 'A' + 'a');
|
||||
return s;
|
||||
}
|
||||
void push_prefix(Hint h) {
|
||||
if (cur_ && prefix_open_) cur_->prefix.push_back(h);
|
||||
}
|
||||
void close_prefix() { prefix_open_ = false; }
|
||||
void by_name(Tag t, Hint h) {
|
||||
if (!cur_) return;
|
||||
cur_->by_name.emplace(t.name, h);
|
||||
cur_->by_name.emplace(lower(t.name), h);
|
||||
}
|
||||
void prim(Tag t, Prim k, bool opt = false) {
|
||||
if (opt) close_prefix();
|
||||
else push_prefix(Hint{k, nullptr});
|
||||
by_name(t, Hint{k, nullptr});
|
||||
if (opt) ctx_.weak.emplace(t.name, k);
|
||||
else ctx_.strong[t.name].insert(k);
|
||||
}
|
||||
void nohint(Tag t) {
|
||||
push_prefix(Hint{});
|
||||
by_name(t, Hint{});
|
||||
}
|
||||
void register_shape(Tag t, const Desc* d) {
|
||||
if (!d) return;
|
||||
if (d->name.empty() && *t.name) ctx_.reg.shapes.emplace(t.name, d);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mars::stream
|
||||
75
src/mars/stream/bytes.h
Normal file
75
src/mars/stream/bytes.h
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
// mars::stream — byte-level helpers shared by the reader and writer.
|
||||
// Everything in the Streamable format is little-endian; these helpers do the
|
||||
// byte shuffling explicitly so the code is host-endian independent.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
using Bytes = std::vector<uint8_t>;
|
||||
|
||||
constexpr uint32_t kBeginMark = 0xBEEFBEEFu; // opens a complex (framed) value
|
||||
constexpr uint32_t kEndMark = 0x41104110u; // closes it (= ~kBeginMark)
|
||||
|
||||
inline uint32_t pad4(uint32_t n) { return (n + 3u) & ~3u; }
|
||||
|
||||
inline uint32_t rd_u32(const uint8_t* p) {
|
||||
return uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24);
|
||||
}
|
||||
inline int32_t rd_i32(const uint8_t* p) { return int32_t(rd_u32(p)); }
|
||||
inline uint64_t rd_u64(const uint8_t* p) { return uint64_t(rd_u32(p)) | (uint64_t(rd_u32(p + 4)) << 32); }
|
||||
inline int64_t rd_i64(const uint8_t* p) { return int64_t(rd_u64(p)); }
|
||||
inline float rd_f32(const uint8_t* p) {
|
||||
uint32_t u = rd_u32(p);
|
||||
float f;
|
||||
std::memcpy(&f, &u, 4);
|
||||
return f;
|
||||
}
|
||||
inline uint32_t f32_bits(float f) {
|
||||
uint32_t u;
|
||||
std::memcpy(&u, &f, 4);
|
||||
return u;
|
||||
}
|
||||
inline float bits_f32(uint32_t u) {
|
||||
float f;
|
||||
std::memcpy(&f, &u, 4);
|
||||
return f;
|
||||
}
|
||||
|
||||
inline void put_u32(Bytes& b, uint32_t v) {
|
||||
b.push_back(uint8_t(v));
|
||||
b.push_back(uint8_t(v >> 8));
|
||||
b.push_back(uint8_t(v >> 16));
|
||||
b.push_back(uint8_t(v >> 24));
|
||||
}
|
||||
inline void put_i32(Bytes& b, int32_t v) { put_u32(b, uint32_t(v)); }
|
||||
inline void put_u64(Bytes& b, uint64_t v) {
|
||||
put_u32(b, uint32_t(v));
|
||||
put_u32(b, uint32_t(v >> 32));
|
||||
}
|
||||
inline void put_f32(Bytes& b, float f) { put_u32(b, f32_bits(f)); }
|
||||
inline void put_bytes(Bytes& b, const void* p, size_t n) {
|
||||
const uint8_t* s = static_cast<const uint8_t*>(p);
|
||||
b.insert(b.end(), s, s + n);
|
||||
}
|
||||
inline void put_pad(Bytes& b, size_t to_boundary_of_item_start) {
|
||||
// append NUL bytes until b.size() - start is a multiple of 4
|
||||
while ((b.size() - to_boundary_of_item_start) & 3u) b.push_back(0);
|
||||
}
|
||||
|
||||
inline std::string hex(const uint8_t* p, size_t n) {
|
||||
static const char* d = "0123456789abcdef";
|
||||
std::string s;
|
||||
s.reserve(n * 2);
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
s.push_back(d[p[i] >> 4]);
|
||||
s.push_back(d[p[i] & 15]);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace mars::stream
|
||||
147
src/mars/stream/dump.cpp
Normal file
147
src/mars/stream/dump.cpp
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
#include "dump.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
std::string py_float_repr(double v) {
|
||||
if (std::isnan(v)) return "nan";
|
||||
if (std::isinf(v)) return v < 0 ? "-inf" : "inf";
|
||||
if (v == 0.0) return std::signbit(v) ? "-0.0" : "0.0";
|
||||
// shortest digit string that round-trips
|
||||
char buf[64];
|
||||
int prec = 1;
|
||||
for (; prec <= 17; ++prec) {
|
||||
std::snprintf(buf, sizeof buf, "%.*e", prec - 1, v);
|
||||
if (std::strtod(buf, nullptr) == v) break;
|
||||
}
|
||||
// buf: [-]d[.ddd]e[+-]XX
|
||||
std::string s(buf);
|
||||
bool neg = s[0] == '-';
|
||||
if (neg) s.erase(0, 1);
|
||||
size_t epos = s.find('e');
|
||||
int exp10 = std::atoi(s.c_str() + epos + 1);
|
||||
std::string digits;
|
||||
for (size_t i = 0; i < epos; ++i)
|
||||
if (s[i] != '.') digits.push_back(s[i]);
|
||||
while (digits.size() > 1 && digits.back() == '0') digits.pop_back();
|
||||
int decpt = exp10 + 1; // value = 0.digits * 10^decpt
|
||||
std::string out = neg ? "-" : "";
|
||||
if (decpt > -4 && decpt <= 16) {
|
||||
int nd = int(digits.size());
|
||||
if (decpt <= 0) {
|
||||
out += "0.";
|
||||
out.append(size_t(-decpt), '0');
|
||||
out += digits;
|
||||
} else if (decpt >= nd) {
|
||||
out += digits;
|
||||
out.append(size_t(decpt - nd), '0');
|
||||
out += ".0";
|
||||
} else {
|
||||
out += digits.substr(0, size_t(decpt));
|
||||
out += ".";
|
||||
out += digits.substr(size_t(decpt));
|
||||
}
|
||||
} else {
|
||||
out += digits[0];
|
||||
if (digits.size() > 1) {
|
||||
out += ".";
|
||||
out += digits.substr(1);
|
||||
}
|
||||
char e[16];
|
||||
std::snprintf(e, sizeof e, "e%c%02d", exp10 < 0 ? '-' : '+', std::abs(exp10));
|
||||
out += e;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
namespace {
|
||||
// windows-1252 0x80..0x9f -> Unicode (0 = undefined -> U+FFFD like errors="replace")
|
||||
const uint16_t kCp1252High[32] = {
|
||||
0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160,
|
||||
0x2039, 0x0152, 0, 0x017D, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022,
|
||||
0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E, 0x0178};
|
||||
} // namespace
|
||||
|
||||
std::string json_quote_cp1252(const std::string& bytes) {
|
||||
std::string o = "\"";
|
||||
char tmp[8];
|
||||
for (unsigned char c : bytes) {
|
||||
uint32_t u = c;
|
||||
if (c >= 0x80 && c < 0xa0) u = kCp1252High[c - 0x80] ? kCp1252High[c - 0x80] : 0xFFFD;
|
||||
switch (u) {
|
||||
case '"': o += "\\\""; break;
|
||||
case '\\': o += "\\\\"; break;
|
||||
case '\n': o += "\\n"; break;
|
||||
case '\r': o += "\\r"; break;
|
||||
case '\t': o += "\\t"; break;
|
||||
case '\b': o += "\\b"; break;
|
||||
case '\f': o += "\\f"; break;
|
||||
default:
|
||||
if (u < 0x20 || u >= 0x7f) {
|
||||
std::snprintf(tmp, sizeof tmp, "\\u%04x", u);
|
||||
o += tmp;
|
||||
} else {
|
||||
o.push_back(char(u));
|
||||
}
|
||||
}
|
||||
}
|
||||
o += "\"";
|
||||
return o;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
void dump_into(const Node& node, int indent, std::vector<std::string>& lines, int max_depth) {
|
||||
std::string pad(size_t(indent) * 2, ' ');
|
||||
char off[16];
|
||||
for (const Node& c : node.children) {
|
||||
std::string nm = c.tagged ? c.name : "<tagless>";
|
||||
std::snprintf(off, sizeof off, "@%08x ", c.offset);
|
||||
if (c.is_complex()) {
|
||||
lines.push_back(std::string(off) + pad + nm + " { # " + std::to_string(c.children.size()) +
|
||||
" items, " + std::to_string(c.size) + " bytes");
|
||||
if (indent < max_depth) dump_into(c, indent + 1, lines, max_depth);
|
||||
std::snprintf(off, sizeof off, "@%08x ", c.offset + c.size - 4);
|
||||
lines.push_back(std::string(off) + pad + "}");
|
||||
} else if (c.kind == Kind::Raw) {
|
||||
size_t n = c.raw.size();
|
||||
lines.push_back(std::string(off) + pad + nm + " raw[" + std::to_string(n) + "] " +
|
||||
hex(c.raw.data(), n < 32 ? n : 32) + (n > 32 ? "..." : ""));
|
||||
} else {
|
||||
std::string val, alt;
|
||||
switch (c.kind) {
|
||||
case Kind::Int:
|
||||
val = std::to_string(c.as_int());
|
||||
if (!c.hinted) {
|
||||
float f = c.as_float();
|
||||
if (std::isfinite(f)) alt = " (alt " + py_float_repr(f) + ")";
|
||||
}
|
||||
break;
|
||||
case Kind::Float:
|
||||
val = py_float_repr(c.as_float());
|
||||
if (!c.hinted) alt = " (alt " + std::to_string(c.as_int()) + ")";
|
||||
break;
|
||||
case Kind::Bool: val = c.as_bool() ? "True" : "False"; break;
|
||||
case Kind::Int64: val = std::to_string(c.as_int64()); break;
|
||||
case Kind::String: val = json_quote_cp1252(c.as_string()); break;
|
||||
default: break;
|
||||
}
|
||||
lines.push_back(std::string(off) + pad + nm + " " + kind_name(c.kind) + (c.hinted ? "" : "?") + " " +
|
||||
val + alt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::vector<std::string> dump_tree(const Node& root, int max_depth) {
|
||||
std::vector<std::string> lines;
|
||||
dump_into(root, 0, lines, max_depth);
|
||||
return lines;
|
||||
}
|
||||
|
||||
} // namespace mars::stream
|
||||
23
src/mars/stream/dump.h
Normal file
23
src/mars/stream/dump.h
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
// mars::stream — text dump of a generic tree, one line per item:
|
||||
// @<inflated offset> <indent><name> <kind>[?] <value>[ (alt <other reading>)]
|
||||
// `?` marks a guessed kind; frames print `{ ... }` with item count and size.
|
||||
// The format matches the reference reader's --dump output line for line so
|
||||
// the two can be diffed (numbers use Python's float repr, strings JSON quoting).
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "node.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
// Python-compatible shortest round-trip repr of a double ("240.0", "1e-05", "3.4028234663852886e+38").
|
||||
std::string py_float_repr(double v);
|
||||
|
||||
// JSON string literal for windows-1252 bytes (non-ASCII escaped as \uXXXX, like json.dumps).
|
||||
std::string json_quote_cp1252(const std::string& bytes);
|
||||
|
||||
std::vector<std::string> dump_tree(const Node& root, int max_depth = 999);
|
||||
|
||||
} // namespace mars::stream
|
||||
101
src/mars/stream/gzip.cpp
Normal file
101
src/mars/stream/gzip.cpp
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
#include "gzip.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include "../../../third_party/miniz/miniz.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
namespace {
|
||||
|
||||
// gzip member: 10-byte header (+ optional fields), raw deflate, CRC32, ISIZE.
|
||||
size_t gzip_header_len(const uint8_t* p, size_t n) {
|
||||
if (n < 10 || p[0] != 0x1f || p[1] != 0x8b || p[2] != 8) throw GzipError("not a gzip/deflate stream");
|
||||
uint8_t flg = p[3];
|
||||
size_t pos = 10;
|
||||
if (flg & 4) { // FEXTRA
|
||||
if (pos + 2 > n) throw GzipError("truncated gzip header");
|
||||
size_t xlen = p[pos] | (p[pos + 1] << 8);
|
||||
pos += 2 + xlen;
|
||||
}
|
||||
if (flg & 8) { // FNAME
|
||||
while (pos < n && p[pos]) ++pos;
|
||||
++pos;
|
||||
}
|
||||
if (flg & 16) { // FCOMMENT
|
||||
while (pos < n && p[pos]) ++pos;
|
||||
++pos;
|
||||
}
|
||||
if (flg & 2) pos += 2; // FHCRC
|
||||
if (pos > n) throw GzipError("truncated gzip header");
|
||||
return pos;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Bytes gunzip(const uint8_t* p, size_t n) {
|
||||
size_t hdr = gzip_header_len(p, n);
|
||||
if (n < hdr + 8) throw GzipError("truncated gzip member");
|
||||
uint32_t isize = rd_u32(p + n - 4);
|
||||
uint32_t crc_expect = rd_u32(p + n - 8);
|
||||
|
||||
Bytes out;
|
||||
out.resize(isize ? isize : 1);
|
||||
mz_stream zs;
|
||||
std::memset(&zs, 0, sizeof zs);
|
||||
if (mz_inflateInit2(&zs, -MZ_DEFAULT_WINDOW_BITS) != MZ_OK) throw GzipError("inflateInit failed");
|
||||
zs.next_in = p + hdr;
|
||||
zs.avail_in = static_cast<unsigned>(n - hdr - 8);
|
||||
size_t produced = 0;
|
||||
int rc;
|
||||
for (;;) {
|
||||
if (produced == out.size()) out.resize(out.size() * 2);
|
||||
zs.next_out = out.data() + produced;
|
||||
zs.avail_out = static_cast<unsigned>(out.size() - produced);
|
||||
rc = mz_inflate(&zs, MZ_NO_FLUSH);
|
||||
produced = out.size() - zs.avail_out;
|
||||
if (rc == MZ_STREAM_END) break;
|
||||
if (rc != MZ_OK) {
|
||||
mz_inflateEnd(&zs);
|
||||
throw GzipError(std::string("gzip container is damaged: ") + mz_error(rc));
|
||||
}
|
||||
}
|
||||
mz_inflateEnd(&zs);
|
||||
out.resize(produced);
|
||||
uint32_t crc = static_cast<uint32_t>(mz_crc32(MZ_CRC32_INIT, out.data(), out.size()));
|
||||
if (crc != crc_expect) throw GzipError("gzip CRC mismatch");
|
||||
if (isize != static_cast<uint32_t>(out.size())) throw GzipError("gzip ISIZE mismatch");
|
||||
return out;
|
||||
}
|
||||
|
||||
Bytes inflate_container(const uint8_t* p, size_t n) {
|
||||
if (is_gzip(p, n)) return gunzip(p, n);
|
||||
return Bytes(p, p + n);
|
||||
}
|
||||
|
||||
Bytes gzip(const uint8_t* p, size_t n) {
|
||||
Bytes out = {0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 0x0b}; // header, OS = NTFS (like the game's files)
|
||||
mz_stream zs;
|
||||
std::memset(&zs, 0, sizeof zs);
|
||||
if (mz_deflateInit2(&zs, 6, MZ_DEFLATED, -MZ_DEFAULT_WINDOW_BITS, 9, MZ_DEFAULT_STRATEGY) != MZ_OK)
|
||||
throw GzipError("deflateInit failed");
|
||||
size_t bound = mz_deflateBound(&zs, static_cast<mz_ulong>(n));
|
||||
size_t start = out.size();
|
||||
out.resize(start + bound);
|
||||
zs.next_in = p;
|
||||
zs.avail_in = static_cast<unsigned>(n);
|
||||
zs.next_out = out.data() + start;
|
||||
zs.avail_out = static_cast<unsigned>(bound);
|
||||
int rc = mz_deflate(&zs, MZ_FINISH);
|
||||
if (rc != MZ_STREAM_END) {
|
||||
mz_deflateEnd(&zs);
|
||||
throw GzipError("deflate failed");
|
||||
}
|
||||
out.resize(start + bound - zs.avail_out);
|
||||
mz_deflateEnd(&zs);
|
||||
put_u32(out, static_cast<uint32_t>(mz_crc32(MZ_CRC32_INIT, p, n)));
|
||||
put_u32(out, static_cast<uint32_t>(n));
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace mars::stream
|
||||
25
src/mars/stream/gzip.h
Normal file
25
src/mars/stream/gzip.h
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
// mars::stream — gzip container (a .sav is one gzip member around the stream).
|
||||
#pragma once
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
#include "bytes.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
struct GzipError : std::runtime_error {
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
inline bool is_gzip(const uint8_t* p, size_t n) { return n >= 2 && p[0] == 0x1f && p[1] == 0x8b; }
|
||||
|
||||
// Inflate one gzip member. Throws GzipError on a damaged container.
|
||||
Bytes gunzip(const uint8_t* p, size_t n);
|
||||
|
||||
// Data without the gzip magic is returned unchanged (already-inflated stream).
|
||||
Bytes inflate_container(const uint8_t* p, size_t n);
|
||||
|
||||
// Wrap `data` in a gzip member (deflate, level 6, no name/mtime).
|
||||
Bytes gzip(const uint8_t* p, size_t n);
|
||||
|
||||
} // namespace mars::stream
|
||||
120
src/mars/stream/node.h
Normal file
120
src/mars/stream/node.h
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
// mars::stream — the generic tree produced by the walker.
|
||||
//
|
||||
// A save stream is a flat sequence of *items*. Every item starts with a name
|
||||
// tag ([int32 len][bytes]; the game writes NULL names as ".") followed by the
|
||||
// value; the whole item is NUL-padded to a 4-byte boundary ("joint" padding).
|
||||
// Complex values are framed by 0xBEEFBEEF ... 0x41104110 and nest. Values
|
||||
// carry no type byte, so a Node records the kind the reader decided on, the
|
||||
// raw value bytes (always exact) and whether that kind came from a hint or a
|
||||
// guess.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "bytes.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
enum class Kind : uint8_t { Int, Float, Bool, Int64, String, Complex, Raw };
|
||||
|
||||
inline const char* kind_name(Kind k) {
|
||||
switch (k) {
|
||||
case Kind::Int: return "int";
|
||||
case Kind::Float: return "float";
|
||||
case Kind::Bool: return "bool";
|
||||
case Kind::Int64: return "int64";
|
||||
case Kind::String: return "string";
|
||||
case Kind::Complex: return "complex";
|
||||
case Kind::Raw: return "raw";
|
||||
}
|
||||
return "?";
|
||||
}
|
||||
|
||||
struct Node {
|
||||
std::string name; // tag text ("" for an empty tag)
|
||||
bool tagged = true; // false: tagless frame / unnamed raw payload
|
||||
Kind kind = Kind::Complex;
|
||||
Bytes raw; // scalar value bytes; string payload (no length); raw blob
|
||||
bool hinted = false; // kind came from the schema / catalog, not a guess
|
||||
uint32_t offset = 0; // inflated offset of the item's tag (or frame marker)
|
||||
uint32_t size = 0; // bytes consumed incl. tag, padding and frame markers
|
||||
std::vector<Node> children;
|
||||
|
||||
bool is_complex() const { return kind == Kind::Complex; }
|
||||
int32_t as_int() const { return raw.size() >= 4 ? rd_i32(raw.data()) : 0; }
|
||||
float as_float() const { return raw.size() >= 4 ? rd_f32(raw.data()) : 0.f; }
|
||||
bool as_bool() const { return !raw.empty() && raw[0] != 0; }
|
||||
int64_t as_int64() const { return raw.size() >= 8 ? rd_i64(raw.data()) : 0; }
|
||||
// string payload as stored (windows-1252 bytes, no conversion)
|
||||
std::string as_string() const { return std::string(raw.begin(), raw.end()); }
|
||||
|
||||
// --- constructors for building trees by hand -------------------------
|
||||
static Node int32(std::string name, int32_t v) {
|
||||
Node n = scalar(std::move(name), Kind::Int);
|
||||
put_i32(n.raw, v);
|
||||
return n;
|
||||
}
|
||||
static Node float32(std::string name, float v) {
|
||||
Node n = scalar(std::move(name), Kind::Float);
|
||||
put_f32(n.raw, v);
|
||||
return n;
|
||||
}
|
||||
static Node boolean(std::string name, bool v) {
|
||||
Node n = scalar(std::move(name), Kind::Bool);
|
||||
n.raw.push_back(v ? 1 : 0);
|
||||
return n;
|
||||
}
|
||||
static Node int64(std::string name, int64_t v) {
|
||||
Node n = scalar(std::move(name), Kind::Int64);
|
||||
put_u64(n.raw, uint64_t(v));
|
||||
return n;
|
||||
}
|
||||
static Node string(std::string name, const std::string& v) {
|
||||
Node n = scalar(std::move(name), Kind::String);
|
||||
n.raw.assign(v.begin(), v.end());
|
||||
return n;
|
||||
}
|
||||
static Node rawbytes(std::string name, Bytes v) {
|
||||
Node n = scalar(std::move(name), Kind::Raw);
|
||||
n.raw = std::move(v);
|
||||
return n;
|
||||
}
|
||||
static Node frame(std::string name, std::vector<Node> kids = {}) {
|
||||
Node n;
|
||||
n.name = std::move(name);
|
||||
n.kind = Kind::Complex;
|
||||
n.children = std::move(kids);
|
||||
n.hinted = true;
|
||||
return n;
|
||||
}
|
||||
|
||||
private:
|
||||
static Node scalar(std::string name, Kind k) {
|
||||
Node n;
|
||||
n.name = std::move(name);
|
||||
n.kind = k;
|
||||
n.hinted = true;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
struct Issue {
|
||||
enum Level { Info, Warn, Error };
|
||||
Level level;
|
||||
std::string path;
|
||||
uint32_t offset;
|
||||
std::string msg;
|
||||
};
|
||||
|
||||
inline const char* level_name(Issue::Level l) {
|
||||
return l == Issue::Info ? "info" : l == Issue::Warn ? "warn" : "error";
|
||||
}
|
||||
|
||||
struct Stats {
|
||||
uint32_t items = 0, frames = 0, resyncs = 0, hint_failures = 0, guessed = 0, raw_bytes = 0,
|
||||
best_effort = 0;
|
||||
};
|
||||
|
||||
} // namespace mars::stream
|
||||
432
src/mars/stream/reader.cpp
Normal file
432
src/mars/stream/reader.cpp
Normal file
|
|
@ -0,0 +1,432 @@
|
|||
#include "reader.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
namespace {
|
||||
|
||||
bool cp1252_defined(uint8_t c) {
|
||||
return !(c == 0x81 || c == 0x8d || c == 0x8f || c == 0x90 || c == 0x9d);
|
||||
}
|
||||
|
||||
// Guard used only while GUESSING that an unknown tag holds a string: every
|
||||
// byte windows-1252 defines counts as text (system names carry 0x92).
|
||||
bool text_plausible(const uint8_t* b, size_t n) {
|
||||
if (n == 0) return true;
|
||||
size_t ok = 0;
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
uint8_t c = b[i];
|
||||
if ((c >= 0x20 && c < 0x7f) || (c >= 0x80 && cp1252_defined(c)) || c == 9 || c == 10 || c == 13) ++ok;
|
||||
}
|
||||
return ok * 10 >= n * 9;
|
||||
}
|
||||
|
||||
// Unknown 4-byte word -> int or float by bit pattern.
|
||||
Prim classify_word(const uint8_t* raw) {
|
||||
int32_t i = rd_i32(raw);
|
||||
double f = rd_f32(raw);
|
||||
if (i >= -kWordIntAbs && i <= kWordIntAbs) return Prim::Int;
|
||||
if (std::isfinite(f) && std::fabs(f) >= 1e-6 && std::fabs(f) < 1e12) return Prim::Float;
|
||||
return Prim::Int;
|
||||
}
|
||||
|
||||
Kind prim_to_kind(Prim p) {
|
||||
switch (p) {
|
||||
case Prim::Int: return Kind::Int;
|
||||
case Prim::Float: return Kind::Float;
|
||||
case Prim::Bool: return Kind::Bool;
|
||||
case Prim::Int64: return Kind::Int64;
|
||||
case Prim::String: return Kind::String;
|
||||
case Prim::Raw: return Kind::Raw;
|
||||
default: return Kind::Int;
|
||||
}
|
||||
}
|
||||
|
||||
std::string lower(std::string s) {
|
||||
for (char& c : s)
|
||||
if (c >= 'A' && c <= 'Z') c = char(c - 'A' + 'a');
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Walker::Walker(const uint8_t* data, size_t n, const Registry* reg)
|
||||
: d_(data), n_(static_cast<uint32_t>(n)), reg_(reg) {}
|
||||
|
||||
void Walker::issue(Issue::Level lvl, uint32_t off, std::string msg, const std::string& path) {
|
||||
issues.push_back(Issue{lvl, path, off, std::move(msg)});
|
||||
}
|
||||
|
||||
bool Walker::zero(uint32_t a, uint32_t b) const {
|
||||
for (uint32_t i = a; i < b; ++i)
|
||||
if (d_[i]) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
Walker::TagAt Walker::tag_at(uint32_t p, bool allow_empty) const {
|
||||
TagAt t;
|
||||
if (p + 4 > n_) return t;
|
||||
int32_t ln = rd_i32(d_ + p);
|
||||
if (ln == 0) {
|
||||
if (!allow_empty) return t;
|
||||
t.ok = true;
|
||||
t.end = p + 4;
|
||||
return t;
|
||||
}
|
||||
if (ln < 0 || ln > kMaxTagLen || uint64_t(p) + 4 + uint32_t(ln) > n_) return t;
|
||||
const uint8_t* b = d_ + p + 4;
|
||||
for (int i = 0; i < ln; ++i)
|
||||
if (b[i] < 0x20 || b[i] >= 0x7f) return t;
|
||||
t.ok = true;
|
||||
t.name.assign(reinterpret_cast<const char*>(b), size_t(ln));
|
||||
t.end = p + 4 + uint32_t(ln);
|
||||
return t;
|
||||
}
|
||||
|
||||
// Read a candidate scalar at the value position (joint padding: right after the
|
||||
// name). known=true means the kind comes from the schema: a string value is
|
||||
// then accepted whatever bytes it holds (only tags must be ASCII).
|
||||
Walker::Scalar Walker::try_scalar(uint32_t tag_start, uint32_t name_end, Cand kind, bool known) {
|
||||
Scalar r;
|
||||
uint32_t vp = name_end;
|
||||
uint32_t size;
|
||||
if (kind == Cand::String) {
|
||||
if (vp + 4 > n_) {
|
||||
eof_limited_ = true;
|
||||
return r;
|
||||
}
|
||||
int32_t ln = rd_i32(d_ + vp);
|
||||
if (ln < 0 || ln > kMaxStringLen) return r;
|
||||
if (uint64_t(vp) + 4 + uint32_t(ln) > n_) {
|
||||
eof_limited_ = true;
|
||||
return r;
|
||||
}
|
||||
if (!known && !text_plausible(d_ + vp + 4, size_t(ln))) return r;
|
||||
size = 4 + uint32_t(ln);
|
||||
} else {
|
||||
size = kind == Cand::Word ? 4 : kind == Cand::Bool ? 1 : 8;
|
||||
if (uint64_t(vp) + size > n_) {
|
||||
eof_limited_ = true;
|
||||
return r;
|
||||
}
|
||||
if (kind == Cand::Bool && d_[vp] > 1) return r;
|
||||
}
|
||||
uint32_t end = tag_start + pad4(name_end - tag_start + size);
|
||||
if (end > n_) {
|
||||
eof_limited_ = true;
|
||||
return r;
|
||||
}
|
||||
if (!zero(vp + size, end)) return r;
|
||||
r.ok = true;
|
||||
r.cand = kind;
|
||||
r.vp = d_ + vp;
|
||||
r.vsize = size;
|
||||
r.end = end;
|
||||
return r;
|
||||
}
|
||||
|
||||
// Does an item plausibly start at p? Looks `depth` items ahead.
|
||||
bool Walker::plausible_item(uint32_t p, int depth, bool allow_empty) {
|
||||
if (p == n_) return true;
|
||||
if (p + 4 > n_) {
|
||||
eof_limited_ = true;
|
||||
return false;
|
||||
}
|
||||
uint32_t w = u32(p);
|
||||
if (w == kEndMark || w == kBeginMark) return true;
|
||||
TagAt t = tag_at(p, allow_empty);
|
||||
if (!t.ok) return false;
|
||||
uint32_t a = pad4(t.end);
|
||||
if (a + 4 <= n_ && u32(a) == kBeginMark && zero(t.end, a)) return true;
|
||||
if (depth <= 0) return true;
|
||||
// a tag the catalog knows to be a string is read as one without any text
|
||||
// test, so a non-ASCII value can never veto the layout of the item before it
|
||||
if (reg_ && reg_->kind(t.name) == Prim::String) {
|
||||
Scalar r = try_scalar(p, t.end, Cand::String, true);
|
||||
if (r.ok && plausible_item(r.end, depth - 1, allow_empty)) return true;
|
||||
}
|
||||
static const Cand order[] = {Cand::Word, Cand::Bool, Cand::String, Cand::Int64};
|
||||
for (Cand c : order) {
|
||||
Scalar r = try_scalar(p, t.end, c, false);
|
||||
if (!r.ok) continue;
|
||||
if (plausible_item(r.end, depth - 1, allow_empty)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Scan forward for the next END marker or plausible tag.
|
||||
std::pair<const char*, uint32_t> Walker::resync(uint32_t p, int depth) {
|
||||
uint32_t q = p + 1;
|
||||
while (q + 4 <= n_) {
|
||||
uint32_t w = u32(q);
|
||||
if (w == kEndMark && depth > 0) return {"end", q};
|
||||
if (w == kBeginMark) return {"begin", q};
|
||||
if (tag_at(q, false).ok && plausible_item(q, 2, false)) return {"tag", q};
|
||||
++q;
|
||||
}
|
||||
return {"eof", n_};
|
||||
}
|
||||
|
||||
Hint Walker::child_hint(const Desc* frame, size_t index, const std::string* name) const {
|
||||
Hint h;
|
||||
if (frame) {
|
||||
if (frame->type == Desc::Shape) {
|
||||
if (index < frame->prefix.size()) h = frame->prefix[index];
|
||||
if (h.empty() && name && !name->empty()) {
|
||||
auto it = frame->by_name.find(*name);
|
||||
if (it == frame->by_name.end()) it = frame->by_name.find(lower(*name));
|
||||
if (it != frame->by_name.end()) h = it->second;
|
||||
}
|
||||
} else if (frame->type == Desc::CArr) {
|
||||
if (index == 0) h.kind = Prim::Int;
|
||||
else h = frame->elem;
|
||||
} else if (frame->type == Desc::Raw) {
|
||||
h.kind = Prim::Raw;
|
||||
}
|
||||
}
|
||||
if (h.empty() && name && !name->empty() && reg_) {
|
||||
// global catalog: exact case only (case-folding was found to mislabel
|
||||
// unknown names, e.g. 'a' vs star-colour 'A')
|
||||
h.kind = reg_->kind(*name);
|
||||
h.sub = reg_->shape(*name);
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
Node Walker::walk() {
|
||||
uint32_t p = 0;
|
||||
Node root;
|
||||
root.tagged = false;
|
||||
root.kind = Kind::Complex;
|
||||
root.children = walk_frame(p, false, "", 0, nullptr, "");
|
||||
root.size = p;
|
||||
return root;
|
||||
}
|
||||
|
||||
// Read items until the frame's END marker (or EOF). p is left after the END.
|
||||
std::vector<Node> Walker::walk_frame(uint32_t& p, bool has_ctx, const std::string& ctx, int depth,
|
||||
const Desc* shape, const std::string& path) {
|
||||
std::vector<Node> children;
|
||||
size_t index = 0;
|
||||
if (depth > 0) ++stats.frames;
|
||||
std::string ctxr = has_ctx ? "'" + ctx + "'" : "None";
|
||||
for (;;) {
|
||||
if (p >= n_) {
|
||||
if (depth > 0) issue(Issue::Error, p, "frame " + ctxr + " not terminated before EOF", path);
|
||||
return children;
|
||||
}
|
||||
if (p + 4 > n_) {
|
||||
children.push_back(raw_node(false, "", p, n_, "trailing bytes", Issue::Warn, path));
|
||||
if (depth > 0) issue(Issue::Error, n_, "frame " + ctxr + " not terminated before EOF", path);
|
||||
p = n_;
|
||||
return children;
|
||||
}
|
||||
uint32_t w = u32(p);
|
||||
if (w == kEndMark) {
|
||||
if (depth > 0) {
|
||||
p += 4;
|
||||
return children;
|
||||
}
|
||||
issue(Issue::Warn, p, "stray END marker at top level", path);
|
||||
children.push_back(raw_node(false, "", p, p + 4, "stray END marker", Issue::Warn, path));
|
||||
p += 4;
|
||||
continue;
|
||||
}
|
||||
children.push_back(read_item(p, index, depth, shape, path));
|
||||
++index;
|
||||
}
|
||||
}
|
||||
|
||||
Node Walker::read_item(uint32_t& p, size_t index, int depth, const Desc* frame, const std::string& path) {
|
||||
++stats.items;
|
||||
uint32_t start = p;
|
||||
uint32_t w = u32(p);
|
||||
if (w == kBeginMark) { // tagless frame
|
||||
Hint h = child_hint(frame, index, nullptr);
|
||||
Node n;
|
||||
n.tagged = false;
|
||||
n.kind = Kind::Complex;
|
||||
n.offset = start;
|
||||
p += 4;
|
||||
n.children = walk_frame(p, false, "", depth + 1, h.sub, path + "/<" + std::to_string(index) + ">");
|
||||
n.size = p - start;
|
||||
return n;
|
||||
}
|
||||
TagAt t = tag_at(p, true);
|
||||
if (!t.ok) return unreadable(p, depth, path);
|
||||
const std::string& name = t.name;
|
||||
uint32_t ne = t.end;
|
||||
Hint h = child_hint(frame, index, &name);
|
||||
uint32_t a = pad4(ne);
|
||||
if (a + 4 <= n_ && u32(a) == kBeginMark && zero(ne, a)) {
|
||||
Node n;
|
||||
n.name = name;
|
||||
n.kind = Kind::Complex;
|
||||
n.offset = start;
|
||||
n.hinted = h.sub != nullptr;
|
||||
p = a + 4;
|
||||
n.children = walk_frame(p, true, name, depth + 1, h.sub, path + "/" + name);
|
||||
n.size = p - start;
|
||||
return n;
|
||||
}
|
||||
|
||||
// scalar: hinted kind first, then the guess order
|
||||
if (h.kind == Prim::Raw) {
|
||||
uint32_t vp = ne;
|
||||
uint32_t q = n_;
|
||||
if (depth > 0) {
|
||||
// the blob runs to the next END marker (any byte alignment, like bytes.find)
|
||||
const uint8_t endb[4] = {0x10, 0x41, 0x10, 0x41};
|
||||
for (uint32_t i = vp; i + 4 <= n_; ++i)
|
||||
if (std::memcmp(d_ + i, endb, 4) == 0) {
|
||||
q = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Node n;
|
||||
n.name = name;
|
||||
n.kind = Kind::Raw;
|
||||
n.offset = start;
|
||||
n.size = q - start;
|
||||
n.hinted = true;
|
||||
n.raw.assign(d_ + vp, d_ + q);
|
||||
stats.raw_bytes += q - vp;
|
||||
p = q;
|
||||
return n;
|
||||
}
|
||||
|
||||
Cand order[4];
|
||||
int norder = 0;
|
||||
switch (h.kind) {
|
||||
case Prim::Int:
|
||||
case Prim::Float: order[norder++] = Cand::Word; break;
|
||||
case Prim::Bool: order[norder++] = Cand::Bool; break;
|
||||
case Prim::String: order[norder++] = Cand::String; break;
|
||||
case Prim::Int64: order[norder++] = Cand::Int64; break;
|
||||
default: break;
|
||||
}
|
||||
for (Cand c : {Cand::Word, Cand::Bool, Cand::String, Cand::Int64}) {
|
||||
bool have = false;
|
||||
for (int i = 0; i < norder; ++i)
|
||||
if (order[i] == c) have = true;
|
||||
if (!have) order[norder++] = c;
|
||||
}
|
||||
bool hinted_kind = h.kind != Prim::None;
|
||||
|
||||
// pass 1: continuation must be a non-empty tag / marker; pass 2 tolerates
|
||||
// empty ("") tags; pass 3 (only when lookahead was defeated by EOF) takes
|
||||
// whatever fits the remaining bytes.
|
||||
eof_limited_ = false;
|
||||
Scalar chosen;
|
||||
int chosen_i = -1;
|
||||
int chosen_mode = -1;
|
||||
for (int mode = 0; mode < 3 && chosen_i < 0; ++mode) {
|
||||
if (mode == 2 && !eof_limited_) break;
|
||||
for (int i = 0; i < norder; ++i) {
|
||||
Scalar r = try_scalar(start, ne, order[i], i == 0 && hinted_kind);
|
||||
if (!r.ok) continue;
|
||||
if (mode == 2 || plausible_item(r.end, 2, mode == 1)) {
|
||||
chosen = r;
|
||||
chosen_i = i;
|
||||
chosen_mode = mode;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (chosen_i >= 0) {
|
||||
Node n;
|
||||
n.name = name;
|
||||
n.offset = start;
|
||||
n.size = chosen.end - start;
|
||||
Prim ck;
|
||||
switch (chosen.cand) {
|
||||
case Cand::Word:
|
||||
ck = (h.kind == Prim::Int || h.kind == Prim::Float) ? h.kind : classify_word(chosen.vp);
|
||||
break;
|
||||
case Cand::Bool: ck = Prim::Bool; break;
|
||||
case Cand::String: ck = Prim::String; break;
|
||||
default: ck = Prim::Int64; break;
|
||||
}
|
||||
n.kind = prim_to_kind(ck);
|
||||
if (chosen.cand == Cand::String) n.raw.assign(chosen.vp + 4, chosen.vp + chosen.vsize);
|
||||
else n.raw.assign(chosen.vp, chosen.vp + chosen.vsize);
|
||||
n.hinted = hinted_kind && chosen_i == 0;
|
||||
if (hinted_kind && chosen_i != 0) {
|
||||
++stats.hint_failures;
|
||||
issue(Issue::Warn, start,
|
||||
"'" + name + "': hinted type " + prim_name(h.kind) + " not plausible, read as " + prim_name(ck),
|
||||
path);
|
||||
} else if (!hinted_kind) {
|
||||
++stats.guessed;
|
||||
}
|
||||
if (chosen_mode == 2) {
|
||||
++stats.best_effort;
|
||||
issue(Issue::Warn, start,
|
||||
"'" + name + "': stream ends before layout can be confirmed; read as " + prim_name(ck), path);
|
||||
}
|
||||
p = chosen.end;
|
||||
return n;
|
||||
}
|
||||
|
||||
// tag looked fine but no value layout fits: skip to the next sync point
|
||||
auto [what, q] = resync(ne, depth);
|
||||
++stats.resyncs;
|
||||
issue(Issue::Warn, start,
|
||||
"'" + name + "': no value layout fits; skipped " + std::to_string(q - ne) + " bytes to " + what, path);
|
||||
Node n;
|
||||
n.name = name;
|
||||
n.kind = Kind::Raw;
|
||||
n.offset = start;
|
||||
n.size = q - start;
|
||||
n.raw.assign(d_ + ne, d_ + q);
|
||||
stats.raw_bytes += q - ne;
|
||||
p = q;
|
||||
return n;
|
||||
}
|
||||
|
||||
// No tag at p. A small unnamed payload before an END is normal (info);
|
||||
// anything longer is a resync event.
|
||||
Node Walker::unreadable(uint32_t& p, int depth, const std::string& path) {
|
||||
uint32_t start = p;
|
||||
if (depth > 0) {
|
||||
uint32_t q = start;
|
||||
while (q + 4 <= n_ && q - start <= uint32_t(kSmallRaw)) {
|
||||
if (u32(q) == kEndMark) {
|
||||
p = q;
|
||||
return raw_node(false, "", start, q, "unnamed payload", Issue::Info, path);
|
||||
}
|
||||
q += 4;
|
||||
}
|
||||
}
|
||||
auto [what, q] = resync(start, depth);
|
||||
++stats.resyncs;
|
||||
p = q;
|
||||
return raw_node(false, "", start, q, (std::string("unreadable; resynced to ") + what).c_str(), Issue::Warn,
|
||||
path);
|
||||
}
|
||||
|
||||
Node Walker::raw_node(bool tagged, const std::string& name, uint32_t p, uint32_t q, const char* why,
|
||||
Issue::Level lvl, const std::string& path) {
|
||||
issue(lvl, p, std::string(why) + ": " + std::to_string(q - p) + " raw bytes", path);
|
||||
stats.raw_bytes += q - p;
|
||||
Node n;
|
||||
n.tagged = tagged;
|
||||
n.name = name;
|
||||
n.kind = Kind::Raw;
|
||||
n.offset = p;
|
||||
n.size = q - p;
|
||||
n.raw.assign(d_ + p, d_ + q);
|
||||
return n;
|
||||
}
|
||||
|
||||
Node read_tree(const Bytes& inflated, std::vector<Issue>* issues, Stats* stats, const Registry* reg) {
|
||||
Walker w(inflated.data(), inflated.size(), reg);
|
||||
Node root = w.walk();
|
||||
if (issues) *issues = std::move(w.issues);
|
||||
if (stats) *stats = w.stats;
|
||||
return root;
|
||||
}
|
||||
|
||||
} // namespace mars::stream
|
||||
78
src/mars/stream/reader.h
Normal file
78
src/mars/stream/reader.h
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
// mars::stream — the generic walker.
|
||||
//
|
||||
// Tokenizes an inflated stream into a Node tree without needing a schema.
|
||||
// Frames (BEEFBEEF / 41104110) are hard synchronisation points; scalar layout
|
||||
// is chosen by a hint (registry) or by trying [word, bool, string, int64] and
|
||||
// keeping the first layout after which another plausible item (or a marker,
|
||||
// or EOF) follows. Unreadable stretches become raw nodes and the walk resumes
|
||||
// at the next marker or plausible tag, so a damaged or unknown region never
|
||||
// derails the rest of the file.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "node.h"
|
||||
#include "schema.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
constexpr int kMaxTagLen = 64; // field names are short identifiers
|
||||
constexpr int kMaxStringLen = 1 << 20; // sanity bound for string payloads
|
||||
constexpr int kSmallRaw = 64; // unnamed payload <= this before an END is info, not warn
|
||||
constexpr int kWordIntAbs = 100000; // |int| <= this -> classified as int, not float
|
||||
|
||||
class Walker {
|
||||
public:
|
||||
// `reg` may be null: everything is then guessed.
|
||||
Walker(const uint8_t* data, size_t n, const Registry* reg);
|
||||
|
||||
Node walk();
|
||||
|
||||
std::vector<Issue> issues;
|
||||
Stats stats;
|
||||
|
||||
private:
|
||||
enum class Cand { Word, Bool, String, Int64 };
|
||||
struct Scalar {
|
||||
bool ok = false;
|
||||
Cand cand = Cand::Word;
|
||||
Prim prim = Prim::None; // resolved kind for the node
|
||||
const uint8_t* vp = nullptr;
|
||||
uint32_t vsize = 0;
|
||||
uint32_t end = 0;
|
||||
};
|
||||
struct TagAt {
|
||||
bool ok = false;
|
||||
std::string name;
|
||||
uint32_t end = 0;
|
||||
};
|
||||
|
||||
const uint8_t* d_;
|
||||
uint32_t n_;
|
||||
const Registry* reg_;
|
||||
bool eof_limited_ = false;
|
||||
|
||||
void issue(Issue::Level lvl, uint32_t off, std::string msg, const std::string& path);
|
||||
uint32_t u32(uint32_t p) const { return rd_u32(d_ + p); }
|
||||
bool zero(uint32_t a, uint32_t b) const;
|
||||
TagAt tag_at(uint32_t p, bool allow_empty) const;
|
||||
Scalar try_scalar(uint32_t tag_start, uint32_t name_end, Cand kind, bool known);
|
||||
bool plausible_item(uint32_t p, int depth, bool allow_empty);
|
||||
std::pair<const char*, uint32_t> resync(uint32_t p, int depth);
|
||||
Hint child_hint(const Desc* frame, size_t index, const std::string* name) const;
|
||||
std::vector<Node> walk_frame(uint32_t& p, bool has_ctx, const std::string& ctx, int depth, const Desc* shape,
|
||||
const std::string& path);
|
||||
Node read_item(uint32_t& p, size_t index, int depth, const Desc* frame, const std::string& path);
|
||||
Node unreadable(uint32_t& p, int depth, const std::string& path);
|
||||
Node raw_node(bool tagged, const std::string& name, uint32_t p, uint32_t q, const char* why, Issue::Level lvl,
|
||||
const std::string& path);
|
||||
};
|
||||
|
||||
// Convenience: walk an inflated stream with the SotS save registry.
|
||||
Node read_tree(const Bytes& inflated, std::vector<Issue>* issues = nullptr, Stats* stats = nullptr,
|
||||
const Registry* reg = &save_registry());
|
||||
|
||||
} // namespace mars::stream
|
||||
98
src/mars/stream/save.cpp
Normal file
98
src/mars/stream/save.cpp
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
#include "save.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
#include <iterator>
|
||||
|
||||
#include "gzip.h"
|
||||
#include "reader.h"
|
||||
#include "writer.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
namespace {
|
||||
|
||||
// Kinds for tags that only occur inside opaque ("any") regions; they let the
|
||||
// walker type those bodies without a shape (tech tree, designs, objectives,
|
||||
// comms, research, encounters).
|
||||
const std::map<std::string, Prim>& manual_kinds() {
|
||||
static const std::map<std::string, Prim> k = {
|
||||
{"TNm", Prim::String}, {"tfc", Prim::Bool}, {"tResCost", Prim::Int}, {"tResDone", Prim::Int},
|
||||
{"tAcq", Prim::Int}, {"tiAcq", Prim::Int}, {"tUnlck", Prim::Int}, {"dName", Prim::String},
|
||||
{"wfn", Prim::String}, {"bId", Prim::Bool}, {"faiDes", Prim::Bool}, {"dHide", Prim::Bool},
|
||||
{"cmp", Prim::Bool}, {"dsc", Prim::String}, {"xcsn", Prim::String}, {"nm", Prim::String},
|
||||
{"ntg", Prim::String}, {"wep", Prim::String}, {"gmch", Prim::Int}, {"drad", Prim::Float},
|
||||
{"cst", Prim::Float}, {"aOdd", Prim::Float}, {"aInc", Prim::Float}, {"rMd", Prim::Float},
|
||||
{"sctSize", Prim::Float}, {"smx", Prim::Float}, {"nPrvVa", Prim::Float}, {"crPce", Prim::Bool},
|
||||
{"maintHf", Prim::Bool}, {"caps2", Prim::Int64}, {"tRsld", Prim::Bool}, {"tRsldd", Prim::Bool},
|
||||
{"tRsldc", Prim::Bool}, {"tRsldi", Prim::Bool}, {"tRsldr", Prim::Bool}, {"prm", Prim::Float},
|
||||
};
|
||||
return k;
|
||||
}
|
||||
|
||||
Registry build_registry() {
|
||||
Registry reg;
|
||||
SchemaBuilder::Context ctx(reg);
|
||||
SchemaBuilder root(ctx, nullptr); // the file root is an unframed sequence: no positional hints
|
||||
shapes::SaveGame g;
|
||||
g.io(root);
|
||||
SchemaBuilder::finalize(ctx, manual_kinds());
|
||||
return reg;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
const Registry& save_registry() {
|
||||
static const Registry reg = build_registry();
|
||||
return reg;
|
||||
}
|
||||
|
||||
shapes::SaveGame apply_shapes(const Node& root, std::vector<Issue>& issues) {
|
||||
shapes::SaveGame game;
|
||||
ReadArchive ar(root.children, issues, "");
|
||||
game.io(ar);
|
||||
if (!ar.done()) {
|
||||
issues.push_back(Issue{Issue::Warn, "", ar.peek()->offset,
|
||||
std::to_string(ar.remaining()) + " unexpected trailing items"});
|
||||
}
|
||||
return game;
|
||||
}
|
||||
|
||||
SaveDocument read_save_bytes(const uint8_t* p, size_t n) {
|
||||
SaveDocument doc;
|
||||
doc.inflated = inflate_container(p, n);
|
||||
Walker w(doc.inflated.data(), doc.inflated.size(), &save_registry());
|
||||
doc.tree = w.walk();
|
||||
doc.issues = std::move(w.issues);
|
||||
doc.stats = w.stats;
|
||||
doc.game = apply_shapes(doc.tree, doc.issues);
|
||||
return doc;
|
||||
}
|
||||
|
||||
SaveDocument read_save_file(const std::string& path) {
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
if (!f) throw GzipError("cannot open " + path);
|
||||
Bytes data((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
|
||||
return read_save_bytes(data.data(), data.size());
|
||||
}
|
||||
|
||||
Bytes write_save(shapes::SaveGame& game) {
|
||||
Writer w;
|
||||
WriteArchive ar(w);
|
||||
game.io(ar);
|
||||
return w.take();
|
||||
}
|
||||
|
||||
Bytes write_tree(const Node& root) {
|
||||
Writer w;
|
||||
for (const Node& c : root.children) w.node(c);
|
||||
return w.take();
|
||||
}
|
||||
|
||||
std::string format_issue(const Issue& i) {
|
||||
char off[24];
|
||||
std::snprintf(off, sizeof off, "0x%x", i.offset);
|
||||
return std::string("[") + level_name(i.level) + "] @" + off + " " + i.path + ": " + i.msg;
|
||||
}
|
||||
|
||||
} // namespace mars::stream
|
||||
47
src/mars/stream/save.h
Normal file
47
src/mars/stream/save.h
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
// mars::stream — top-level save-file API.
|
||||
//
|
||||
// SaveDocument doc = read_save_file(path); // or read_save_bytes(...)
|
||||
// doc.tree generic Node tree (always complete: unknown regions are raw)
|
||||
// doc.game typed shapes applied on top of the tree
|
||||
// doc.issues walker + shape deviations (error / warn / info)
|
||||
// write_save(doc.game) -> inflated bytes; gzip() them for a .sav
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "node.h"
|
||||
#include "shapes.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
struct SaveDocument {
|
||||
Bytes inflated;
|
||||
Node tree;
|
||||
shapes::SaveGame game;
|
||||
std::vector<Issue> issues; // walker issues first, then shape issues
|
||||
Stats stats;
|
||||
|
||||
size_t count(Issue::Level l) const {
|
||||
size_t n = 0;
|
||||
for (const Issue& i : issues) n += i.level == l;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
// Parse a .sav (gzip) or an already-inflated stream. Throws GzipError on a damaged container.
|
||||
SaveDocument read_save_bytes(const uint8_t* p, size_t n);
|
||||
SaveDocument read_save_file(const std::string& path);
|
||||
|
||||
// Apply the typed shapes to a generic tree (root children = file items).
|
||||
shapes::SaveGame apply_shapes(const Node& root, std::vector<Issue>& issues);
|
||||
|
||||
// Serialize typed shapes back to an inflated stream.
|
||||
Bytes write_save(shapes::SaveGame& game);
|
||||
|
||||
// Re-emit a generic tree (root children) as bytes.
|
||||
Bytes write_tree(const Node& root);
|
||||
|
||||
std::string format_issue(const Issue& i);
|
||||
|
||||
} // namespace mars::stream
|
||||
116
src/mars/stream/savedump_main.cpp
Normal file
116
src/mars/stream/savedump_main.cpp
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
// sots_savedump — CLI over mars::stream: dump / summarize / round-trip a save.
|
||||
//
|
||||
// sots_savedump SAVE [--dump] [--strict] [--roundtrip] [--issues N] [--max-depth D]
|
||||
// [--inflate FILE] [--rewrite FILE]
|
||||
//
|
||||
// --dump generic tree, one item per line (same format as the reference reader)
|
||||
// --strict exit 2 on any error or warning
|
||||
// --roundtrip re-emit the tree and the typed shapes and compare with the input
|
||||
// --rewrite write the typed shapes back out as a gzip .sav
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
|
||||
#include "dump.h"
|
||||
#include "gzip.h"
|
||||
#include "save.h"
|
||||
|
||||
using namespace mars::stream;
|
||||
|
||||
static void summary(const SaveDocument& d, const char* path) {
|
||||
const auto& s = d.game.summary;
|
||||
const auto& sim = d.game.sim;
|
||||
std::printf("file: %s inflated: %zu bytes\n", path, d.inflated.size());
|
||||
std::printf("items: %u frames: %u guessed: %u hint-failures: %u resyncs: %u raw-bytes: %u\n",
|
||||
d.stats.items, d.stats.frames, d.stats.guessed, d.stats.hint_failures, d.stats.resyncs,
|
||||
d.stats.raw_bytes);
|
||||
std::printf("issues: %zu error, %zu warn, %zu info\n", d.count(Issue::Error), d.count(Issue::Warn),
|
||||
d.count(Issue::Info));
|
||||
std::printf("summary: game='%s' turn=%d numSys=%d players=%zu scenario='%s'\n", s.gameName.c_str(), s.turn,
|
||||
s.numSys, s.players.size(), s.scenario.c_str());
|
||||
std::printf("sim: gameName='%s' players=%zu systems=%zu fleets=%zu\n", sim.gameName.c_str(),
|
||||
sim.players.size(), sim.systems.size(), sim.fleets.size());
|
||||
for (const auto& p : sim.players)
|
||||
std::printf(" player %d: '%s' species=%d home=%d sav=%d designs=%zu\n", p.playerID,
|
||||
p.player.plryName.c_str(), p.player.species, p.player.homeSys, p.player.sav,
|
||||
p.player.designs.size());
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc < 2) {
|
||||
std::fprintf(stderr, "usage: sots_savedump SAVE [--dump] [--strict] [--roundtrip] [--issues N] "
|
||||
"[--max-depth D] [--inflate FILE] [--rewrite FILE]\n");
|
||||
return 2;
|
||||
}
|
||||
const char* path = argv[1];
|
||||
bool dump = false, strict = false, roundtrip = false;
|
||||
int max_issues = 30, max_depth = 999;
|
||||
const char* inflate_to = nullptr;
|
||||
const char* rewrite_to = nullptr;
|
||||
for (int i = 2; i < argc; ++i) {
|
||||
std::string a = argv[i];
|
||||
if (a == "--dump") dump = true;
|
||||
else if (a == "--strict") strict = true;
|
||||
else if (a == "--roundtrip") roundtrip = true;
|
||||
else if (a == "--issues" && i + 1 < argc) max_issues = std::atoi(argv[++i]);
|
||||
else if (a == "--max-depth" && i + 1 < argc) max_depth = std::atoi(argv[++i]);
|
||||
else if (a == "--inflate" && i + 1 < argc) inflate_to = argv[++i];
|
||||
else if (a == "--rewrite" && i + 1 < argc) rewrite_to = argv[++i];
|
||||
else {
|
||||
std::fprintf(stderr, "unknown option %s\n", argv[i]);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
SaveDocument doc;
|
||||
try {
|
||||
doc = read_save_file(path);
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "error: %s\n", e.what());
|
||||
return 2;
|
||||
}
|
||||
if (inflate_to) {
|
||||
std::ofstream f(inflate_to, std::ios::binary);
|
||||
f.write(reinterpret_cast<const char*>(doc.inflated.data()), std::streamsize(doc.inflated.size()));
|
||||
}
|
||||
if (dump) {
|
||||
for (const std::string& l : dump_tree(doc.tree, max_depth)) std::puts(l.c_str());
|
||||
} else {
|
||||
summary(doc, path);
|
||||
int shown = 0;
|
||||
for (const Issue& i : doc.issues) {
|
||||
if (i.level == Issue::Info) continue;
|
||||
if (shown == 0) std::printf("issues (errors/warnings, first %d):\n", max_issues);
|
||||
if (shown++ >= max_issues) break;
|
||||
std::printf(" %s\n", format_issue(i).c_str());
|
||||
}
|
||||
}
|
||||
int rc = doc.count(Issue::Error) ? 1 : 0;
|
||||
if (roundtrip) {
|
||||
Bytes tree_bytes = write_tree(doc.tree);
|
||||
bool tree_ok = tree_bytes == doc.inflated;
|
||||
Bytes typed_bytes = write_save(doc.game);
|
||||
bool typed_ok = typed_bytes == doc.inflated;
|
||||
size_t first_diff = 0;
|
||||
if (!typed_ok) {
|
||||
size_t n = std::min(typed_bytes.size(), doc.inflated.size());
|
||||
while (first_diff < n && typed_bytes[first_diff] == doc.inflated[first_diff]) ++first_diff;
|
||||
}
|
||||
char where[48] = "";
|
||||
if (!typed_ok) std::snprintf(where, sizeof where, ", first difference at 0x%zx", first_diff);
|
||||
std::printf("roundtrip: tree %s (%zu bytes), typed %s (%zu bytes%s)\n", tree_ok ? "identical" : "DIFFERS",
|
||||
tree_bytes.size(), typed_ok ? "identical" : "DIFFERS", typed_bytes.size(), where);
|
||||
if (!tree_ok || !typed_ok) rc = rc ? rc : 1;
|
||||
}
|
||||
if (rewrite_to) {
|
||||
Bytes out = write_save(doc.game);
|
||||
Bytes gz = gzip(out.data(), out.size());
|
||||
std::ofstream f(rewrite_to, std::ios::binary);
|
||||
f.write(reinterpret_cast<const char*>(gz.data()), std::streamsize(gz.size()));
|
||||
}
|
||||
if (strict && (doc.count(Issue::Error) || doc.count(Issue::Warn))) {
|
||||
std::fprintf(stderr, "strict: %zu error(s), %zu warning(s)\n", doc.count(Issue::Error), doc.count(Issue::Warn));
|
||||
return 2;
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
75
src/mars/stream/schema.h
Normal file
75
src/mars/stream/schema.h
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
// mars::stream — type hints the walker consults while tokenizing.
|
||||
//
|
||||
// The stream carries no type bytes, so the generic walker decides scalar kinds
|
||||
// from (a) a positional / by-name description of the frame it is inside
|
||||
// (`Desc`), (b) a global (tag -> kind) catalog, and (c) a lookahead
|
||||
// plausibility test. The descriptors are built once from the typed shapes in
|
||||
// shapes.h by running their io() under a SchemaBuilder archive (archive.h);
|
||||
// nothing here is required for the walk to succeed — hints only make the
|
||||
// output better typed.
|
||||
#pragma once
|
||||
|
||||
#include <deque>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
// Scalar kinds a hint can name. Hint::None means "no scalar hint".
|
||||
enum class Prim : uint8_t { None, Int, Float, Bool, Int64, String, Raw };
|
||||
|
||||
struct Desc;
|
||||
|
||||
struct Hint {
|
||||
Prim kind = Prim::None;
|
||||
const Desc* sub = nullptr; // descriptor for a framed child (Shape / CArr / Raw)
|
||||
bool empty() const { return kind == Prim::None && sub == nullptr; }
|
||||
};
|
||||
|
||||
struct Desc {
|
||||
enum Type { Shape, CArr, Raw };
|
||||
Type type = Shape;
|
||||
std::string name; // Shape: on-disk frame tag ("" when unnamed)
|
||||
std::vector<Hint> prefix; // Shape: positional hints up to the first variable-length field
|
||||
std::map<std::string, Hint> by_name; // Shape: fallback by child tag (exact, then lower-cased)
|
||||
Hint elem; // CArr: hint for every element after the "." count
|
||||
};
|
||||
|
||||
struct Registry {
|
||||
std::deque<Desc> owned; // stable storage for descriptors
|
||||
std::map<std::string, Prim> kinds; // global tag -> scalar kind
|
||||
std::map<std::string, const Desc*> shapes; // global tag -> frame descriptor
|
||||
std::map<std::string, Prim> string_tags; // tags known to be strings (never text-tested)
|
||||
|
||||
Desc* add(Desc d) {
|
||||
owned.push_back(std::move(d));
|
||||
return &owned.back();
|
||||
}
|
||||
const Desc* shape(const std::string& n) const {
|
||||
auto it = shapes.find(n);
|
||||
return it == shapes.end() ? nullptr : it->second;
|
||||
}
|
||||
Prim kind(const std::string& n) const {
|
||||
auto it = kinds.find(n);
|
||||
return it == kinds.end() ? Prim::None : it->second;
|
||||
}
|
||||
};
|
||||
|
||||
inline const char* prim_name(Prim p) {
|
||||
switch (p) {
|
||||
case Prim::None: return "-";
|
||||
case Prim::Int: return "int";
|
||||
case Prim::Float: return "float";
|
||||
case Prim::Bool: return "bool";
|
||||
case Prim::Int64: return "int64";
|
||||
case Prim::String: return "string";
|
||||
case Prim::Raw: return "raw";
|
||||
}
|
||||
return "?";
|
||||
}
|
||||
|
||||
// The registry describing the SotS save layout (built from shapes.h).
|
||||
const Registry& save_registry();
|
||||
|
||||
} // namespace mars::stream
|
||||
1585
src/mars/stream/shapes.h
Normal file
1585
src/mars/stream/shapes.h
Normal file
File diff suppressed because it is too large
Load diff
31
src/mars/stream/writer.cpp
Normal file
31
src/mars/stream/writer.cpp
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
#include "writer.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
void Writer::node(const Node& n) {
|
||||
switch (n.kind) {
|
||||
case Kind::Complex:
|
||||
if (n.tagged) begin(n.name);
|
||||
else begin_tagless();
|
||||
for (const Node& c : n.children) node(c);
|
||||
end();
|
||||
return;
|
||||
case Kind::String:
|
||||
string(n.name, n.as_string());
|
||||
return;
|
||||
case Kind::Raw:
|
||||
// raw payloads keep whatever the reader captured, including any
|
||||
// padding bytes that preceded the END marker
|
||||
if (n.tagged) raw(n.name, n.raw);
|
||||
else bare(n.raw.data(), n.raw.size());
|
||||
return;
|
||||
default:
|
||||
// int / float / bool / int64: the raw bytes are the value
|
||||
begin_item(n.name);
|
||||
put_bytes(buf_, n.raw.data(), n.raw.size());
|
||||
end_item();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mars::stream
|
||||
66
src/mars/stream/writer.h
Normal file
66
src/mars/stream/writer.h
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
// mars::stream — emits the Streamable byte format (joint padding, framed
|
||||
// complex values). Mirrors what the reader accepts so read -> write is
|
||||
// byte-identical.
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "bytes.h"
|
||||
#include "node.h"
|
||||
|
||||
namespace mars::stream {
|
||||
|
||||
class Writer {
|
||||
public:
|
||||
// --- scalars: [len][name][value][pad to 4] ------------------------------
|
||||
void int32(const std::string& name, int32_t v) { begin_item(name); put_i32(buf_, v); end_item(); }
|
||||
void float32(const std::string& name, float v) { begin_item(name); put_f32(buf_, v); end_item(); }
|
||||
void boolean(const std::string& name, bool v) { begin_item(name); buf_.push_back(v ? 1 : 0); end_item(); }
|
||||
void int64(const std::string& name, int64_t v) { begin_item(name); put_u64(buf_, uint64_t(v)); end_item(); }
|
||||
// string value = [int32 len][cp1252 bytes]; an empty string is 4 zero bytes
|
||||
void string(const std::string& name, const std::string& v) {
|
||||
begin_item(name);
|
||||
put_i32(buf_, int32_t(v.size()));
|
||||
put_bytes(buf_, v.data(), v.size());
|
||||
end_item();
|
||||
}
|
||||
// opaque payload whose length only the writer knows (RNG state)
|
||||
void raw(const std::string& name, const uint8_t* p, size_t n) {
|
||||
begin_item(name);
|
||||
put_bytes(buf_, p, n);
|
||||
end_item();
|
||||
}
|
||||
void raw(const std::string& name, const Bytes& b) { raw(name, b.data(), b.size()); }
|
||||
|
||||
// --- frames: [len][name][pad] BEEFBEEF ... 41104110 -----------------------
|
||||
void begin(const std::string& name) {
|
||||
begin_item(name);
|
||||
end_item(); // the tag alone is padded to 4
|
||||
put_u32(buf_, kBeginMark);
|
||||
}
|
||||
void begin_tagless() { put_u32(buf_, kBeginMark); }
|
||||
void end() { put_u32(buf_, kEndMark); }
|
||||
|
||||
// unnamed bytes (unnamed raw payload before an END marker)
|
||||
void bare(const uint8_t* p, size_t n) { put_bytes(buf_, p, n); }
|
||||
|
||||
// re-emit a generic tree node (and its subtree) exactly
|
||||
void node(const Node& n);
|
||||
|
||||
const Bytes& bytes() const { return buf_; }
|
||||
Bytes take() { return std::move(buf_); }
|
||||
size_t size() const { return buf_.size(); }
|
||||
|
||||
private:
|
||||
Bytes buf_;
|
||||
size_t item_start_ = 0;
|
||||
|
||||
void begin_item(const std::string& name) {
|
||||
item_start_ = buf_.size();
|
||||
put_i32(buf_, int32_t(name.size()));
|
||||
put_bytes(buf_, name.data(), name.size());
|
||||
}
|
||||
void end_item() { put_pad(buf_, item_start_); }
|
||||
};
|
||||
|
||||
} // namespace mars::stream
|
||||
53
tests/mars_stream/build_and_run.sh
Executable file
53
tests/mars_stream/build_and_run.sh
Executable file
|
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/env bash
|
||||
# Self-contained build + test for src/mars/stream and src/mars/rng (plain g++, no CMake).
|
||||
#
|
||||
# tests/mars_stream/build_and_run.sh [BUILD_DIR]
|
||||
#
|
||||
# Environment (all optional):
|
||||
# SOTS_SAVES_DIR directory with the owner's *.sav files -> real-save test runs
|
||||
# (skipped cleanly when unset; saves never enter the repo)
|
||||
# SOTS_SAVE_READER path to the reference save_reader.py -> oracle dump comparison
|
||||
# PYTHON interpreter for the oracle script (default: python3)
|
||||
set -euo pipefail
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
root="$(cd "$here/../.." && pwd)"
|
||||
out="${1:-$root/build-mars-stream}"
|
||||
mkdir -p "$out"
|
||||
|
||||
CXX="${CXX:-g++}"
|
||||
CC="${CC:-gcc}"
|
||||
CXXFLAGS="-std=c++17 -O2 -Wall -Wextra -Wpedantic -I$root/src"
|
||||
MINIZ_DEFS="-DMINIZ_NO_STDIO -DMINIZ_NO_ARCHIVE_APIS -DMINIZ_NO_TIME"
|
||||
|
||||
echo "== build ($out)"
|
||||
$CC -O2 -w $MINIZ_DEFS -c "$root/third_party/miniz/miniz.c" -o "$out/miniz.o"
|
||||
objs=()
|
||||
for src in gzip reader writer dump save; do
|
||||
$CXX $CXXFLAGS $MINIZ_DEFS -c "$root/src/mars/stream/$src.cpp" -o "$out/stream_$src.o"
|
||||
objs+=("$out/stream_$src.o")
|
||||
done
|
||||
$CXX $CXXFLAGS -c "$root/src/mars/rng/mt19937.cpp" -o "$out/rng_mt19937.o"
|
||||
$CXX $CXXFLAGS -o "$out/sots_savedump" "$root/src/mars/stream/savedump_main.cpp" "${objs[@]}" "$out/miniz.o"
|
||||
$CXX $CXXFLAGS -o "$out/test_rng" "$here/test_rng.cpp" "$out/rng_mt19937.o"
|
||||
$CXX $CXXFLAGS -o "$out/test_stream" "$here/test_stream.cpp" "${objs[@]}" "$out/miniz.o"
|
||||
$CXX $CXXFLAGS -o "$out/test_save" "$here/test_save.cpp" "${objs[@]}" "$out/miniz.o" "$out/rng_mt19937.o"
|
||||
|
||||
echo "== test_rng"
|
||||
"$out/test_rng"
|
||||
echo "== test_stream"
|
||||
"$out/test_stream"
|
||||
echo "== test_save"
|
||||
if [ -n "${SOTS_SAVES_DIR:-}" ]; then
|
||||
mkdir -p "$out/dumps"
|
||||
SOTS_DUMP_DIR="$out/dumps" "$out/test_save"
|
||||
if [ -n "${SOTS_SAVE_READER:-}" ]; then
|
||||
echo "== oracle comparison"
|
||||
${PYTHON:-python3} "$here/oracle/compare.py" --reader "$SOTS_SAVE_READER" --saves "$SOTS_SAVES_DIR" \
|
||||
--dumps "$out/dumps" --python "${PYTHON:-python3}"
|
||||
else
|
||||
echo "oracle comparison skipped (SOTS_SAVE_READER not set)"
|
||||
fi
|
||||
else
|
||||
"$out/test_save"
|
||||
fi
|
||||
echo "== all ok"
|
||||
102
tests/mars_stream/oracle/compare.py
Executable file
102
tests/mars_stream/oracle/compare.py
Executable file
|
|
@ -0,0 +1,102 @@
|
|||
#!/usr/bin/env python3
|
||||
"""compare.py -- oracle check of the C++ walker against the reference reader.
|
||||
|
||||
usage: compare.py --reader SAVE_READER.py --saves DIR --dumps DIR [--python PY]
|
||||
|
||||
For every DIR/*.sav it runs `save_reader.py SAVE --dump --padding joint` and
|
||||
reads the C++ dump written by test_save (`<name>.cpp.dump` in --dumps), then
|
||||
reports two agreement figures per save:
|
||||
|
||||
exact lines identical (including the `?` guess marks and `(alt ...)`)
|
||||
canonical lines identical after normalising what is presentation only:
|
||||
guess marks and alt readings dropped, 4-byte words compared by
|
||||
their raw bytes (an int reading and a float reading of the same
|
||||
word are the same datum), strings compared by content.
|
||||
|
||||
The typed summary (game name, turn, numSys, players) is compared as well.
|
||||
Exit status 1 when any canonical line differs or a summary disagrees.
|
||||
"""
|
||||
import argparse
|
||||
import ast
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
LINE = re.compile(r"^@([0-9a-f]{8}) (\s*)(\S+|) (\S+?)(\??) ?(.*)$")
|
||||
|
||||
|
||||
def canon(line: str) -> str:
|
||||
m = LINE.match(line)
|
||||
if not m:
|
||||
return line
|
||||
off, indent, name, kind, _guess, rest = m.groups()
|
||||
rest = re.sub(r"\s+\(alt .*\)$", "", rest)
|
||||
if kind in ("int", "float"):
|
||||
try:
|
||||
raw = struct.pack("<i", int(rest)) if kind == "int" else struct.pack("<f", float(rest))
|
||||
return f"@{off} {indent}{name} word {raw.hex()}"
|
||||
except (ValueError, struct.error):
|
||||
pass
|
||||
return f"@{off} {indent}{name} {kind} {rest}"
|
||||
|
||||
|
||||
def summary_of_reader(py, reader, save):
|
||||
out = subprocess.run([*shlex.split(py), reader, save, "--padding", "joint"], capture_output=True, text=True, check=True).stdout
|
||||
m = re.search(r"summary: game=(.*?) turn=(\d+) numSys=(\d+) players=(\d+)", out)
|
||||
return (ast.literal_eval(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)))
|
||||
|
||||
|
||||
def summary_of_cpp(path):
|
||||
txt = open(path, encoding="utf-8").read()
|
||||
m = re.search(r"summary: game=(\".*?\") turn=(\d+) numSys=(\d+) players=(\d+)", txt)
|
||||
return (json.loads(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--reader", required=True)
|
||||
ap.add_argument("--saves", required=True)
|
||||
ap.add_argument("--dumps", required=True)
|
||||
ap.add_argument("--python", default=sys.executable)
|
||||
a = ap.parse_args()
|
||||
bad = 0
|
||||
saves = sorted(glob.glob(os.path.join(a.saves, "*.sav")))
|
||||
if not saves:
|
||||
print("compare: no saves")
|
||||
return 0
|
||||
for save in saves:
|
||||
base = os.path.basename(save)
|
||||
cpp_dump = os.path.join(a.dumps, base + ".cpp.dump")
|
||||
if not os.path.exists(cpp_dump):
|
||||
print(f"{base}: no C++ dump at {cpp_dump}")
|
||||
bad += 1
|
||||
continue
|
||||
ref = subprocess.run([*shlex.split(a.python), a.reader, save, "--dump", "--padding", "joint"],
|
||||
capture_output=True, text=True, check=True).stdout.splitlines()
|
||||
cpp = open(cpp_dump, encoding="utf-8").read().splitlines()
|
||||
n = max(len(ref), len(cpp))
|
||||
exact = sum(1 for x, y in zip(ref, cpp) if x == y)
|
||||
cref, ccpp = [canon(l) for l in ref], [canon(l) for l in cpp]
|
||||
canonical = sum(1 for x, y in zip(cref, ccpp) if x == y)
|
||||
first = next((i for i, (x, y) in enumerate(zip(cref, ccpp)) if x != y), None)
|
||||
rs = summary_of_reader(a.python, a.reader, save)
|
||||
cs = summary_of_cpp(os.path.join(a.dumps, base + ".cpp.summary"))
|
||||
ok = canonical == n and len(ref) == len(cpp) and rs == cs
|
||||
print(f"{base}: lines ref={len(ref)} cpp={len(cpp)} exact {exact}/{n} ({100.0 * exact / n:.2f}%) "
|
||||
f"canonical {canonical}/{n} ({100.0 * canonical / n:.2f}%) summary {'agree' if rs == cs else 'DIFFER'}"
|
||||
f" {cs}")
|
||||
if first is not None:
|
||||
print(f" first canonical difference at line {first + 1}:\n ref: {ref[first]}\n cpp: {cpp[first]}")
|
||||
if not ok:
|
||||
bad += 1
|
||||
print("compare:", "ok" if not bad else f"{bad} save(s) disagree")
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
99
tests/mars_stream/test_rng.cpp
Normal file
99
tests/mars_stream/test_rng.cpp
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
// mars::rng tests — reference vectors + state (de)serialization.
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
|
||||
#include "mars/rng/mt19937.h"
|
||||
|
||||
using mars::rng::MT19937;
|
||||
|
||||
static int fails = 0;
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
++fails; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
int main() {
|
||||
// --- standard MT19937 outputs for init_genrand(5489) ------------------------
|
||||
{
|
||||
MT19937 r(5489u);
|
||||
static const uint32_t expect[10] = {3499211612u, 581869302u, 3890346734u, 3586334585u, 545404204u,
|
||||
4161255391u, 3922919429u, 949333985u, 2715962298u, 1323567403u};
|
||||
for (uint32_t e : expect) CHECK(r.next_u32() == e);
|
||||
// the 10000th output of the 5489 stream (well-known check value)
|
||||
MT19937 r2(5489u);
|
||||
uint32_t v = 0;
|
||||
for (int i = 0; i < 10000; ++i) v = r2.next_u32();
|
||||
CHECK(v == 4123659995u);
|
||||
}
|
||||
// --- a fresh generator has twisted once: left == N ------------------------
|
||||
{
|
||||
MT19937 r(1u);
|
||||
CHECK(r.left() == MT19937::N);
|
||||
CHECK(r.index() == 0);
|
||||
r.next_u32();
|
||||
CHECK(r.left() == MT19937::N - 1);
|
||||
for (int i = 1; i < MT19937::N; ++i) r.next_u32();
|
||||
CHECK(r.left() == 0); // block exhausted; the next draw twists
|
||||
r.next_u32();
|
||||
CHECK(r.left() == MT19937::N - 1);
|
||||
}
|
||||
// --- float mapping: y * 2^-32 narrowed to float -----------------------------
|
||||
{
|
||||
MT19937 r(5489u);
|
||||
float f = r.next_float();
|
||||
float expect = static_cast<float>(3499211612.0 / 4294967296.0);
|
||||
CHECK(f == expect);
|
||||
CHECK(f >= 0.f && f <= 1.f);
|
||||
}
|
||||
// --- next_int: in range, and consumes exactly one word when mask == n-1 ----
|
||||
{
|
||||
MT19937 r(7u);
|
||||
for (int i = 0; i < 1000; ++i) CHECK(r.next_int(10) < 10);
|
||||
MT19937 a(9u), b(9u);
|
||||
uint32_t x = a.next_int(256);
|
||||
CHECK(x == (b.next_u32() & 255u));
|
||||
}
|
||||
// --- save_state / load_state round trip, blob layout mt[624] + left --------
|
||||
{
|
||||
MT19937 r(123456u);
|
||||
for (int i = 0; i < 700; ++i) r.next_u32(); // past one twist
|
||||
uint8_t blob[MT19937::kStateBytes];
|
||||
r.save_state(blob);
|
||||
CHECK(MT19937::kStateBytes == 0x9c4);
|
||||
uint32_t left_in_blob = uint32_t(blob[2496]) | (uint32_t(blob[2497]) << 8) | (uint32_t(blob[2498]) << 16) |
|
||||
(uint32_t(blob[2499]) << 24);
|
||||
CHECK(int(left_in_blob) == r.left());
|
||||
CHECK(std::memcmp(blob, r.state(), 4) == 0 || true); // first word is mt[0] little-endian
|
||||
uint32_t w0 = uint32_t(blob[0]) | (uint32_t(blob[1]) << 8) | (uint32_t(blob[2]) << 16) | (uint32_t(blob[3]) << 24);
|
||||
CHECK(w0 == r.state()[0]);
|
||||
|
||||
MT19937 s(1u);
|
||||
CHECK(s.load_state(blob, sizeof blob));
|
||||
CHECK(s.left() == r.left());
|
||||
for (int i = 0; i < 2000; ++i) CHECK(s.next_u32() == r.next_u32());
|
||||
|
||||
// truncated / out-of-range blobs are rejected
|
||||
CHECK(!s.load_state(blob, sizeof blob - 1));
|
||||
uint8_t bad[MT19937::kStateBytes];
|
||||
std::memcpy(bad, blob, sizeof bad);
|
||||
bad[2496] = 0x71; // left = 625 > N
|
||||
bad[2497] = 0x02;
|
||||
CHECK(!s.load_state(bad, sizeof bad));
|
||||
}
|
||||
// --- load_state(mt, left) positions the next word at mt[N - left] -----------
|
||||
{
|
||||
MT19937 r(42u);
|
||||
uint32_t st[MT19937::N];
|
||||
std::memcpy(st, r.state(), sizeof st);
|
||||
MT19937 s(0u);
|
||||
s.load_state(st, 5);
|
||||
for (int i = 0; i < MT19937::N - 5; ++i) r.next_u32();
|
||||
for (int i = 0; i < 100; ++i) CHECK(s.next_u32() == r.next_u32());
|
||||
}
|
||||
std::printf("test_rng: %s\n", fails ? "FAILED" : "ok");
|
||||
return fails ? 1 : 0;
|
||||
}
|
||||
156
tests/mars_stream/test_save.cpp
Normal file
156
tests/mars_stream/test_save.cpp
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
// Real-save test: reads every *.sav in $SOTS_SAVES_DIR (owner's data, never
|
||||
// in the repo) and checks that the walker parses clean, the typed shapes
|
||||
// load, both round trips are byte-identical and the RNG blob is a valid
|
||||
// MT19937 state. Skips (exit 0) when the variable is unset.
|
||||
//
|
||||
// With SOTS_DUMP_DIR set, writes <name>.cpp.dump / <name>.cpp.summary there
|
||||
// for tests/mars_stream/oracle/compare.py.
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <dirent.h>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mars/rng/mt19937.h"
|
||||
#include "mars/stream/dump.h"
|
||||
#include "mars/stream/save.h"
|
||||
|
||||
using namespace mars::stream;
|
||||
|
||||
static int fails = 0;
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
++fails; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
static std::vector<std::string> list_saves(const std::string& dir) {
|
||||
std::vector<std::string> out;
|
||||
DIR* d = opendir(dir.c_str());
|
||||
if (!d) return out;
|
||||
while (dirent* e = readdir(d)) {
|
||||
std::string n = e->d_name;
|
||||
if (n.size() > 4 && n.compare(n.size() - 4, 4, ".sav") == 0) out.push_back(dir + "/" + n);
|
||||
}
|
||||
closedir(d);
|
||||
std::sort(out.begin(), out.end());
|
||||
return out;
|
||||
}
|
||||
|
||||
static void check_save(const std::string& path, const char* dump_dir) {
|
||||
std::printf("== %s\n", path.c_str());
|
||||
SaveDocument doc = read_save_file(path);
|
||||
std::printf(" inflated %zu bytes, items %u, frames %u, resyncs %u, hint-failures %u, raw-bytes %u\n",
|
||||
doc.inflated.size(), doc.stats.items, doc.stats.frames, doc.stats.resyncs, doc.stats.hint_failures,
|
||||
doc.stats.raw_bytes);
|
||||
std::printf(" issues: %zu error, %zu warn, %zu info\n", doc.count(Issue::Error), doc.count(Issue::Warn),
|
||||
doc.count(Issue::Info));
|
||||
for (const Issue& i : doc.issues)
|
||||
if (i.level != Issue::Info) std::printf(" %s\n", format_issue(i).c_str());
|
||||
|
||||
// --- the confirmed format parses clean --------------------------------------
|
||||
CHECK(doc.stats.resyncs == 0);
|
||||
CHECK(doc.stats.hint_failures == 0);
|
||||
CHECK(doc.count(Issue::Error) == 0);
|
||||
CHECK(doc.count(Issue::Warn) == 0);
|
||||
CHECK(doc.stats.raw_bytes == 2503); // only the opaque RNG blob
|
||||
|
||||
// --- typed shapes -------------------------------------------------------------
|
||||
const auto& s = doc.game.summary;
|
||||
const auto& sim = doc.game.sim;
|
||||
std::printf(" summary: game='%s' turn=%d numSys=%d players=%zu\n", s.gameName.c_str(), s.turn, s.numSys,
|
||||
s.players.size());
|
||||
CHECK(!s.gameName.empty());
|
||||
CHECK(s.turn >= 1);
|
||||
CHECK(s.numSys > 0);
|
||||
CHECK(!s.players.empty());
|
||||
CHECK(sim.gameName == s.gameName);
|
||||
CHECK(int(sim.systems.size()) == s.numSys);
|
||||
CHECK(sim.systems.size() == sim.systemIds.size());
|
||||
CHECK(sim.players.size() == sim.playerIds.size());
|
||||
CHECK(sim.fleets.size() == sim.fleetIds.size());
|
||||
CHECK(sim.species.size() == 7);
|
||||
CHECK(!sim.players.empty() && !sim.players[0].player.plryName.empty());
|
||||
CHECK(doc.game.createParams.name == s.gameName);
|
||||
CHECK(doc.game.createParams.nSys == s.numSys);
|
||||
CHECK(doc.game.createParams.mapP.planets.size() == size_t(s.numSys));
|
||||
for (const auto& se : sim.systems) CHECK(!se.sys.name.empty());
|
||||
for (const auto& fe : sim.fleets) CHECK(fe.flt.ships.size() >= 1);
|
||||
CHECK(!doc.game.cdTable.ids.empty());
|
||||
|
||||
// --- RNG blob: mt[624] + left, produced by our MT19937 from RSeed --------------
|
||||
CHECK(sim.rng.is_complex() && sim.rng.children.size() == 1);
|
||||
const Node& blob = sim.rng.children[0];
|
||||
CHECK(blob.kind == Kind::Raw && blob.raw.size() == 2503);
|
||||
mars::rng::MT19937 saved(1u);
|
||||
CHECK(saved.load_state(blob.raw.data(), blob.raw.size()));
|
||||
std::printf(" rng: left=%d (index %d), RSeed=%d\n", saved.left(), saved.index(), doc.game.createParams.rseed);
|
||||
CHECK(saved.left() >= 0 && saved.left() <= mars::rng::MT19937::N);
|
||||
{
|
||||
// the saved block must be reachable from seed(RSeed) by whole twists
|
||||
mars::rng::MT19937 gen(uint32_t(doc.game.createParams.rseed));
|
||||
int twists = -1;
|
||||
for (int k = 0; k < 16 && twists < 0; ++k) {
|
||||
if (std::memcmp(gen.state(), saved.state(), sizeof(uint32_t) * mars::rng::MT19937::N) == 0) twists = k;
|
||||
else
|
||||
for (int i = 0; i < mars::rng::MT19937::N; ++i) gen.next_u32(); // consume a block -> next twist
|
||||
}
|
||||
std::printf(" rng: state == seed(RSeed) after %d twist(s)\n", twists);
|
||||
CHECK(twists >= 0);
|
||||
}
|
||||
|
||||
// --- round trips ------------------------------------------------------------------
|
||||
Bytes tree_bytes = write_tree(doc.tree);
|
||||
CHECK(tree_bytes == doc.inflated);
|
||||
Bytes typed_bytes = write_save(doc.game);
|
||||
if (typed_bytes != doc.inflated) {
|
||||
size_t i = 0, n = std::min(typed_bytes.size(), doc.inflated.size());
|
||||
while (i < n && typed_bytes[i] == doc.inflated[i]) ++i;
|
||||
std::printf(" typed round trip differs at 0x%zx (sizes %zu vs %zu)\n", i, typed_bytes.size(),
|
||||
doc.inflated.size());
|
||||
}
|
||||
CHECK(typed_bytes == doc.inflated);
|
||||
std::printf(" round trip: tree %s, typed %s\n", tree_bytes == doc.inflated ? "identical" : "DIFFERS",
|
||||
typed_bytes == doc.inflated ? "identical" : "DIFFERS");
|
||||
|
||||
// --- optional dump for the oracle comparison -------------------------------------
|
||||
if (dump_dir) {
|
||||
std::string base = path.substr(path.find_last_of('/') + 1);
|
||||
std::ofstream d(std::string(dump_dir) + "/" + base + ".cpp.dump");
|
||||
for (const std::string& l : dump_tree(doc.tree)) d << l << '\n';
|
||||
std::ofstream sm(std::string(dump_dir) + "/" + base + ".cpp.summary");
|
||||
sm << "summary: game=" << json_quote_cp1252(s.gameName) << " turn=" << s.turn << " numSys=" << s.numSys
|
||||
<< " players=" << s.players.size() << '\n';
|
||||
sm << "sim: players=" << sim.players.size() << " systems=" << sim.systems.size()
|
||||
<< " fleets=" << sim.fleets.size() << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
const char* dir = std::getenv("SOTS_SAVES_DIR");
|
||||
if (!dir || !*dir) {
|
||||
std::printf("test_save: SKIPPED (SOTS_SAVES_DIR not set)\n");
|
||||
return 0;
|
||||
}
|
||||
std::vector<std::string> saves = list_saves(dir);
|
||||
if (saves.empty()) {
|
||||
std::printf("test_save: SKIPPED (no *.sav in %s)\n", dir);
|
||||
return 0;
|
||||
}
|
||||
const char* dump_dir = std::getenv("SOTS_DUMP_DIR");
|
||||
for (const std::string& p : saves) {
|
||||
try {
|
||||
check_save(p, dump_dir && *dump_dir ? dump_dir : nullptr);
|
||||
} catch (const std::exception& e) {
|
||||
std::printf("FAIL %s: %s\n", p.c_str(), e.what());
|
||||
++fails;
|
||||
}
|
||||
}
|
||||
std::printf("test_save: %s (%zu save(s), %d failures)\n", fails ? "FAILED" : "ok", saves.size(), fails);
|
||||
return fails ? 1 : 0;
|
||||
}
|
||||
530
tests/mars_stream/test_stream.cpp
Normal file
530
tests/mars_stream/test_stream.cpp
Normal file
|
|
@ -0,0 +1,530 @@
|
|||
// mars::stream unit tests on hand-built byte fixtures (no game data).
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mars/stream/dump.h"
|
||||
#include "mars/stream/gzip.h"
|
||||
#include "mars/stream/reader.h"
|
||||
#include "mars/stream/save.h"
|
||||
#include "mars/stream/writer.h"
|
||||
|
||||
using namespace mars::stream;
|
||||
|
||||
static int fails = 0;
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
++fails; \
|
||||
} \
|
||||
} while (0)
|
||||
#define CHECK_EQ(a, b) \
|
||||
do { \
|
||||
auto _a = (a); \
|
||||
auto _b = (b); \
|
||||
if (!(_a == _b)) { \
|
||||
std::printf("FAIL %s:%d: %s == %s\n", __FILE__, __LINE__, #a, #b); \
|
||||
++fails; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
static Bytes B(std::initializer_list<int> v) {
|
||||
Bytes b;
|
||||
for (int x : v) b.push_back(uint8_t(x));
|
||||
return b;
|
||||
}
|
||||
static Bytes cat(std::initializer_list<Bytes> parts) {
|
||||
Bytes b;
|
||||
for (const Bytes& p : parts) b.insert(b.end(), p.begin(), p.end());
|
||||
return b;
|
||||
}
|
||||
static size_t count(const std::vector<Issue>& v, Issue::Level l) {
|
||||
size_t n = 0;
|
||||
for (const Issue& i : v) n += i.level == l;
|
||||
return n;
|
||||
}
|
||||
|
||||
// --- 1. primitive encodings and joint padding (bytes written by hand) -----------
|
||||
static void test_primitives_bytes() {
|
||||
// [len][name][value][pad to 4], padding computed over the whole item
|
||||
const Bytes turn = B({4, 0, 0, 0, 'T', 'u', 'r', 'n', 1, 0, 0, 0}); // 12: no pad
|
||||
const Bytes haltv = B({5, 0, 0, 0, 'h', 'a', 'l', 't', 'v', 1, 0, 0}); // 10 -> 12
|
||||
const Bytes vnh = B({3, 0, 0, 0, 'v', 'n', 'h', 0}); // 8: no pad
|
||||
const Bytes name = B({4, 0, 0, 0, 'N', 'a', 'm', 'e', 3, 0, 0, 0, 'S', 'o', 'l', 0}); // 15 -> 16
|
||||
const Bytes key = B({3, 0, 0, 0, 'K', 'e', 'y', 0, 0, 0, 0, 0}); // empty string = 4 zero bytes
|
||||
const Bytes incmod = B({6, 0, 0, 0, 'I', 'n', 'c', 'M', 'o', 'd', 0, 0, 0x80, 0x3f, 0, 0}); // float 1.0, 14 -> 16
|
||||
const Bytes bats2 = B({5, 0, 0, 0, 'B', 'a', 't', 's', '2', 0x2a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); // int64 42: 17 -> 20
|
||||
const Bytes dot = B({1, 0, 0, 0, '.', 7, 0, 0, 0, 0, 0, 0}); // "." int 7: 9 -> 12
|
||||
|
||||
Writer w;
|
||||
w.int32("Turn", 1);
|
||||
w.boolean("haltv", true);
|
||||
w.boolean("vnh", false);
|
||||
w.string("Name", "Sol");
|
||||
w.string("Key", "");
|
||||
w.float32("IncMod", 1.0f);
|
||||
w.int64("Bats2", 42);
|
||||
w.int32(".", 7);
|
||||
CHECK(w.bytes() == cat({turn, haltv, vnh, name, key, incmod, bats2, dot}));
|
||||
|
||||
std::vector<Issue> issues;
|
||||
Stats st;
|
||||
Node root = read_tree(w.bytes(), &issues, &st); // with the save registry: all tags hinted
|
||||
CHECK_EQ(root.children.size(), size_t(8));
|
||||
const auto& c = root.children;
|
||||
CHECK(c[0].name == "Turn" && c[0].kind == Kind::Int && c[0].as_int() == 1 && c[0].hinted);
|
||||
CHECK(c[0].offset == 0 && c[0].size == 12);
|
||||
CHECK(c[1].name == "haltv" && c[1].kind == Kind::Bool && c[1].as_bool() && c[1].size == 12);
|
||||
CHECK(c[2].name == "vnh" && c[2].kind == Kind::Bool && !c[2].as_bool() && c[2].size == 8);
|
||||
CHECK(c[3].name == "Name" && c[3].kind == Kind::String && c[3].as_string() == "Sol" && c[3].size == 16);
|
||||
CHECK(c[4].name == "Key" && c[4].kind == Kind::String && c[4].as_string().empty() && c[4].size == 12);
|
||||
CHECK(c[5].name == "IncMod" && c[5].kind == Kind::Float && c[5].as_float() == 1.0f && c[5].size == 16);
|
||||
CHECK(c[6].name == "Bats2" && c[6].kind == Kind::Int64 && c[6].as_int64() == 42 && c[6].size == 20);
|
||||
CHECK(c[7].name == "." && c[7].kind == Kind::Int && c[7].as_int() == 7 && !c[7].hinted);
|
||||
CHECK_EQ(st.resyncs, 0u);
|
||||
CHECK_EQ(count(issues, Issue::Warn), size_t(0));
|
||||
CHECK_EQ(count(issues, Issue::Error), size_t(0));
|
||||
|
||||
// without any registry everything is guessed by layout + bit pattern
|
||||
Node g = read_tree(w.bytes(), nullptr, nullptr, nullptr);
|
||||
CHECK(g.children[0].kind == Kind::Int && !g.children[0].hinted);
|
||||
CHECK(g.children[1].kind == Kind::Bool);
|
||||
CHECK(g.children[3].kind == Kind::String);
|
||||
CHECK(g.children[4].kind == Kind::Int && g.children[4].as_int() == 0); // empty string looks like int 0
|
||||
CHECK(g.children[5].kind == Kind::Float); // 0x3f800000 is not a small int
|
||||
CHECK(g.children[6].kind == Kind::Int64);
|
||||
|
||||
// round trip: the tree re-emits byte-identically
|
||||
CHECK(write_tree(root) == w.bytes());
|
||||
CHECK(write_tree(g) == w.bytes());
|
||||
}
|
||||
|
||||
// --- 2. frames: nesting, tagless frame, empty frame, "." arrays ------------------
|
||||
static void test_frames() {
|
||||
Writer w;
|
||||
w.begin("Summary");
|
||||
w.string("GameName", "x");
|
||||
w.begin("Players"); // VectorHelper: "." count + "." elements
|
||||
w.int32(".", 2);
|
||||
w.begin(".");
|
||||
w.int32("Rank", 1);
|
||||
w.end();
|
||||
w.begin(".");
|
||||
w.int32("Rank", 2);
|
||||
w.end();
|
||||
w.end();
|
||||
w.begin("Empty");
|
||||
w.end();
|
||||
w.begin_tagless();
|
||||
w.float32(".", 2.5f);
|
||||
w.end();
|
||||
w.end();
|
||||
// expected framing bytes for the head: [7]"Summary"[pad] BEEFBEEF
|
||||
const Bytes head = B({7, 0, 0, 0, 'S', 'u', 'm', 'm', 'a', 'r', 'y', 0, 0xef, 0xbe, 0xef, 0xbe});
|
||||
CHECK(std::equal(head.begin(), head.end(), w.bytes().begin()));
|
||||
const Bytes tail = B({0x10, 0x41, 0x10, 0x41});
|
||||
CHECK(std::equal(tail.begin(), tail.end(), w.bytes().end() - 4));
|
||||
|
||||
std::vector<Issue> issues;
|
||||
Stats st;
|
||||
Node root = read_tree(w.bytes(), &issues, &st);
|
||||
CHECK_EQ(root.children.size(), size_t(1));
|
||||
const Node& s = root.children[0];
|
||||
CHECK(s.is_complex() && s.name == "Summary" && s.offset == 0 && s.size == w.size());
|
||||
CHECK_EQ(s.children.size(), size_t(4));
|
||||
const Node& players = s.children[1];
|
||||
CHECK(players.is_complex() && players.children.size() == 3);
|
||||
CHECK(players.children[0].name == "." && players.children[0].as_int() == 2);
|
||||
CHECK(players.children[1].is_complex() && players.children[1].name == ".");
|
||||
CHECK(players.children[2].children[0].name == "Rank" && players.children[2].children[0].as_int() == 2);
|
||||
CHECK(s.children[2].is_complex() && s.children[2].children.empty());
|
||||
CHECK(s.children[3].is_complex() && !s.children[3].tagged && s.children[3].children.size() == 1);
|
||||
CHECK(s.children[3].children[0].as_float() == 2.5f);
|
||||
CHECK_EQ(st.frames, 6u);
|
||||
CHECK_EQ(st.resyncs, 0u);
|
||||
CHECK(write_tree(root) == w.bytes());
|
||||
|
||||
// the dump prints frames with item count and size, tagless as <tagless>
|
||||
auto lines = dump_tree(root);
|
||||
CHECK(lines.size() == 17);
|
||||
CHECK(lines[0] == "@00000000 Summary { # 4 items, " + std::to_string(w.size()) + " bytes");
|
||||
CHECK(lines[1] == "@00000010 GameName string \"x\"");
|
||||
CHECK(lines[13].find("<tagless> {") != std::string::npos);
|
||||
}
|
||||
|
||||
// --- 3. resync: garbage inside a frame becomes raw; the walk continues ----------
|
||||
// (both fixtures were checked against the reference reader: same tree, same
|
||||
// stats, one warning each)
|
||||
static void test_resync() {
|
||||
Bytes tail;
|
||||
{
|
||||
Writer t;
|
||||
t.begin("B");
|
||||
t.int32("z", 2);
|
||||
t.end();
|
||||
tail = t.take();
|
||||
}
|
||||
// case A: no tag at all after a nested frame -> unnamed raw, resync to the next plausible tag
|
||||
{
|
||||
Writer w;
|
||||
w.begin("A");
|
||||
w.begin("C");
|
||||
w.end();
|
||||
Bytes s = w.take();
|
||||
s.insert(s.end(), 12, 0xff);
|
||||
Writer y;
|
||||
y.string("y", std::string(80, 'y'));
|
||||
y.end();
|
||||
Bytes yb = y.take();
|
||||
s = cat({s, yb, tail});
|
||||
std::vector<Issue> issues;
|
||||
Stats st;
|
||||
Node root = read_tree(s, &issues, &st);
|
||||
CHECK_EQ(root.children.size(), size_t(2));
|
||||
const Node& a = root.children[0];
|
||||
CHECK(a.name == "A" && a.children.size() == 3 && a.size == 136);
|
||||
CHECK(a.children[0].name == "C" && a.children[0].children.empty() && a.children[0].size == 16);
|
||||
const Node& r = a.children[1];
|
||||
CHECK(r.kind == Kind::Raw && !r.tagged && r.raw.size() == 12 && r.offset == 0x1c);
|
||||
CHECK(a.children[2].name == "y" && a.children[2].kind == Kind::String && a.children[2].as_string().size() == 80);
|
||||
CHECK(root.children[1].name == "B" && root.children[1].children[0].as_int() == 2);
|
||||
CHECK_EQ(st.resyncs, 1u);
|
||||
CHECK_EQ(st.raw_bytes, 12u);
|
||||
CHECK_EQ(count(issues, Issue::Warn), size_t(1));
|
||||
CHECK(write_tree(root) == s); // raw nodes re-emit exactly
|
||||
auto lines = dump_tree(root);
|
||||
CHECK(lines[3] == "@0000001c <tagless> raw[12] ffffffffffffffffffffffff");
|
||||
}
|
||||
// case B: a fine-looking tag whose value fits no layout -> tagged raw up to the next tag
|
||||
{
|
||||
Bytes s = B({3, 0, 0, 0, 'a', 'b', 'c', 5});
|
||||
s.insert(s.end(), 12, 0xff);
|
||||
Writer w;
|
||||
w.int32("y", 2);
|
||||
w.end();
|
||||
Bytes head;
|
||||
{
|
||||
Writer h;
|
||||
h.begin("A");
|
||||
head = h.take();
|
||||
}
|
||||
s = cat({head, s, w.take(), tail});
|
||||
std::vector<Issue> issues;
|
||||
Stats st;
|
||||
Node root = read_tree(s, &issues, &st);
|
||||
const Node& a = root.children[0];
|
||||
CHECK(a.children.size() == 2 && a.size == 48);
|
||||
CHECK(a.children[0].name == "abc" && a.children[0].kind == Kind::Raw && a.children[0].tagged);
|
||||
CHECK(a.children[0].raw.size() == 13 && a.children[0].raw[0] == 5);
|
||||
CHECK(a.children[1].name == "y" && a.children[1].as_int() == 2 && a.children[1].offset == 0x20);
|
||||
CHECK_EQ(st.resyncs, 1u);
|
||||
CHECK_EQ(st.raw_bytes, 13u);
|
||||
CHECK_EQ(count(issues, Issue::Warn), size_t(1));
|
||||
CHECK(issues[0].msg.find("no value layout fits; skipped 13 bytes to tag") != std::string::npos);
|
||||
CHECK(write_tree(root) == s);
|
||||
}
|
||||
|
||||
// unnamed 12-byte payload right before END is only an info (Vector3 fallback)
|
||||
Bytes v3;
|
||||
{
|
||||
Writer t;
|
||||
t.begin("Pos");
|
||||
t.end();
|
||||
v3 = t.take();
|
||||
Bytes body = B({0, 0, 0x80, 0x3f, 0, 0, 0, 0x40, 0, 0, 0x40, 0x40});
|
||||
v3.insert(v3.end() - 4, body.begin(), body.end());
|
||||
}
|
||||
std::vector<Issue> issues;
|
||||
Stats st;
|
||||
Node r3 = read_tree(v3, &issues, &st);
|
||||
CHECK(r3.children[0].children.size() == 1 && r3.children[0].children[0].kind == Kind::Raw);
|
||||
CHECK_EQ(count(issues, Issue::Info), size_t(1));
|
||||
CHECK_EQ(count(issues, Issue::Warn), size_t(0));
|
||||
// and the shape layer reads it as a vec3
|
||||
std::vector<Issue> si;
|
||||
ReadArchive ar(r3.children, si, "");
|
||||
Vec3 v;
|
||||
ar.vec3(A("Pos"), v);
|
||||
CHECK(v.x == 1.f && v.y == 2.f && v.z == 3.f);
|
||||
|
||||
// a stray END at top level and trailing bytes are reported, not fatal
|
||||
Bytes stray = B({0x10, 0x41, 0x10, 0x41, 1, 2});
|
||||
issues.clear();
|
||||
Node r4 = read_tree(stray, &issues, &st);
|
||||
CHECK(r4.children.size() == 2);
|
||||
CHECK(count(issues, Issue::Warn) >= 2);
|
||||
CHECK(write_tree(r4) == stray);
|
||||
}
|
||||
|
||||
// --- 4. cp1252 string values never veto layout when the tag is known ------------
|
||||
static void test_cp1252() {
|
||||
Writer w;
|
||||
w.boolean("haltv", true);
|
||||
w.string("Name", std::string("Kor\x92Voth")); // 0x92 = right single quote
|
||||
w.int32("VFlags", 3);
|
||||
std::vector<Issue> issues;
|
||||
Stats st;
|
||||
Node root = read_tree(w.bytes(), &issues, &st);
|
||||
CHECK_EQ(root.children.size(), size_t(3));
|
||||
CHECK(root.children[0].kind == Kind::Bool && root.children[0].hinted);
|
||||
CHECK(root.children[1].kind == Kind::String && root.children[1].as_string() == "Kor\x92Voth");
|
||||
CHECK(root.children[2].kind == Kind::Int && root.children[2].as_int() == 3);
|
||||
CHECK_EQ(st.hint_failures, 0u);
|
||||
auto lines = dump_tree(root);
|
||||
CHECK(lines[1] == "@0000000c Name string \"Kor\\u2019Voth\"");
|
||||
// guessing an unknown tag still accepts every cp1252-defined byte
|
||||
Writer g;
|
||||
g.string("zz", std::string("caf\xe9"));
|
||||
g.int32("q", 1);
|
||||
Node r2 = read_tree(g.bytes(), nullptr, nullptr, nullptr);
|
||||
CHECK(r2.children[0].kind == Kind::String);
|
||||
}
|
||||
|
||||
// --- 5. hints: registry types Summary children positionally and by name ---------
|
||||
static void test_hints() {
|
||||
Writer w;
|
||||
w.begin("Summary");
|
||||
w.string("GameName", "g");
|
||||
w.int32("Turn", 0); // 0 either way
|
||||
w.int32("NumSys", 0);
|
||||
w.int32("Checksum", 0);
|
||||
w.begin("Players");
|
||||
w.int32(".", 0);
|
||||
w.end();
|
||||
w.begin("Session");
|
||||
w.begin("TMRS");
|
||||
w.float32("TSTL", 0.0f); // 0.0f: guessed would be int 0
|
||||
w.end();
|
||||
w.end();
|
||||
w.int32("MapShape", 0);
|
||||
w.float32("IncMod", 0.0f);
|
||||
w.end();
|
||||
Node root = read_tree(w.bytes());
|
||||
const Node& s = root.children[0];
|
||||
CHECK(s.hinted);
|
||||
CHECK(s.children[0].kind == Kind::String && s.children[0].hinted);
|
||||
CHECK(s.children[4].hinted); // Players CArr
|
||||
CHECK(s.children[4].children[0].kind == Kind::Int && s.children[4].children[0].hinted);
|
||||
const Node& tstl = s.children[5].children[0].children[0];
|
||||
CHECK(tstl.kind == Kind::Float && tstl.hinted && tstl.as_float() == 0.f);
|
||||
CHECK(s.children[7].kind == Kind::Float && s.children[7].hinted); // IncMod by name
|
||||
// the registry knows the top-level shapes and the RNG raw frame
|
||||
const Registry& reg = save_registry();
|
||||
CHECK(reg.shape("Summary") && reg.shape("Sim") && reg.shape("Sys") && reg.shape("Player"));
|
||||
CHECK(reg.shape("RNG") && reg.shape("RNG")->type == Desc::Raw);
|
||||
CHECK(reg.kind("GameName") == Prim::String && reg.kind("Bats2") == Prim::Int64 && reg.kind("haltv") == Prim::Bool);
|
||||
CHECK(reg.kind(".") == Prim::None);
|
||||
CHECK(reg.kind("Team") == Prim::Int); // int in Slot and Player; the frame is a separate shape entry
|
||||
CHECK(reg.shape("Team") != nullptr);
|
||||
CHECK(reg.kind("pop") == Prim::Int64 && reg.shape("pop") != nullptr); // int64 in stats, Population frame in Ship
|
||||
}
|
||||
|
||||
// --- 6. RNG raw frame: body read straight to the END marker ---------------------
|
||||
static void test_raw_frame() {
|
||||
Bytes blob(2500, 0xAB);
|
||||
Writer w;
|
||||
w.begin("RNG");
|
||||
w.raw(".", blob);
|
||||
w.end();
|
||||
w.int32("Map", 1);
|
||||
std::vector<Issue> issues;
|
||||
Stats st;
|
||||
Node root = read_tree(w.bytes(), &issues, &st);
|
||||
CHECK(root.children[0].children.size() == 1);
|
||||
const Node& r = root.children[0].children[0];
|
||||
CHECK(r.kind == Kind::Raw && r.name == "." && r.raw.size() == 2503); // 2500 + 3 joint-padding bytes
|
||||
CHECK_EQ(st.raw_bytes, 2503u);
|
||||
CHECK(root.children[1].as_int() == 1);
|
||||
CHECK(write_tree(root) == w.bytes());
|
||||
}
|
||||
|
||||
// --- 7. typed shapes: Summary write -> bytes -> read; and hand-checked bytes ------
|
||||
static void test_typed_summary() {
|
||||
shapes::Summary s;
|
||||
s.gameName = "Test";
|
||||
s.turn = 7;
|
||||
s.numSys = 3;
|
||||
s.checksum = 99;
|
||||
shapes::PlayerInfo p;
|
||||
p.slot.isPlay = true;
|
||||
p.slot.fxNm = "re";
|
||||
p.slot.fxCrID.idx = -1;
|
||||
p.slot.fxCrID.r = 10;
|
||||
p.slot.fxCrID.g = 20;
|
||||
p.slot.fxCrID.b = 30;
|
||||
p.slot.tag = 1466349286;
|
||||
p.slot.team = -1;
|
||||
p.slot.settings.treasury = 50000;
|
||||
p.rank = 2;
|
||||
s.players.push_back(p);
|
||||
s.session.tmrs.tctl = 240.f;
|
||||
s.incMod = 1.f;
|
||||
s.resMod = 1.f;
|
||||
s.alliances = true;
|
||||
s.encounters = true;
|
||||
|
||||
Writer w;
|
||||
WriteArchive wa(w);
|
||||
wa.obj(A("Summary"), s);
|
||||
Bytes bytes = w.take();
|
||||
|
||||
// spot-check the head bytes by hand: frame tag, GameName, Turn
|
||||
Bytes head = B({7, 0, 0, 0, 'S', 'u', 'm', 'm', 'a', 'r', 'y', 0, 0xef, 0xbe, 0xef, 0xbe,
|
||||
8, 0, 0, 0, 'G', 'a', 'm', 'e', 'N', 'a', 'm', 'e', 4, 0, 0, 0, 'T', 'e', 's', 't',
|
||||
4, 0, 0, 0, 'T', 'u', 'r', 'n', 7, 0, 0, 0});
|
||||
CHECK(bytes.size() > head.size() && std::equal(head.begin(), head.end(), bytes.begin()));
|
||||
|
||||
std::vector<Issue> issues;
|
||||
Stats st;
|
||||
Node root = read_tree(bytes, &issues, &st);
|
||||
CHECK_EQ(st.resyncs, 0u);
|
||||
CHECK_EQ(st.hint_failures, 0u);
|
||||
shapes::Summary back;
|
||||
ReadArchive ra(root.children, issues, "");
|
||||
ra.obj(A("Summary"), back);
|
||||
CHECK_EQ(count(issues, Issue::Error), size_t(0));
|
||||
CHECK_EQ(count(issues, Issue::Warn), size_t(0));
|
||||
CHECK(back.gameName == "Test" && back.turn == 7 && back.numSys == 3 && back.checksum == 99);
|
||||
CHECK(back.players.size() == 1 && back.players[0].rank == 2);
|
||||
CHECK(back.players[0].slot.fxNm == "re" && back.players[0].slot.tag == 1466349286);
|
||||
CHECK(back.players[0].slot.fxCrID.idx == -1 && back.players[0].slot.fxCrID.b == 30);
|
||||
CHECK(back.players[0].slot.settings.treasury == 50000);
|
||||
CHECK(back.session.tmrs.tctl == 240.f && back.incMod == 1.f && back.alliances && !back.teams);
|
||||
CHECK(back.scenario.empty());
|
||||
|
||||
// and back out: byte-identical
|
||||
Writer w2;
|
||||
WriteArchive wa2(w2);
|
||||
wa2.obj(A("Summary"), back);
|
||||
CHECK(w2.bytes() == bytes);
|
||||
|
||||
// the "." positional items are reported as info, never warn
|
||||
CHECK(count(issues, Issue::Info) >= 4); // FxCrID idx/r/g/b, Settings x4, Players count/elements
|
||||
|
||||
// a missing confirmed field is an error; an unexpected item before it a warning
|
||||
Writer w3;
|
||||
w3.begin("Summary");
|
||||
w3.string("GameName", "g");
|
||||
w3.int32("Bogus", 1);
|
||||
w3.int32("Turn", 2);
|
||||
w3.end();
|
||||
Node r3 = read_tree(w3.bytes());
|
||||
std::vector<Issue> i3;
|
||||
shapes::Summary s3;
|
||||
ReadArchive ra3(r3.children, i3, "");
|
||||
ra3.obj(A("Summary"), s3);
|
||||
CHECK(s3.turn == 2);
|
||||
CHECK(count(i3, Issue::Warn) >= 1);
|
||||
CHECK(count(i3, Issue::Error) >= 1); // NumSys and the rest are missing
|
||||
}
|
||||
|
||||
// --- 8. conditionals, optionals and inline arrays through the archives ------------
|
||||
static void test_typed_conditionals() {
|
||||
shapes::Ship ship;
|
||||
ship.desID = 5;
|
||||
ship.hbq = true;
|
||||
shapes::BuildOrder o;
|
||||
o.desID = 9;
|
||||
ship.bq2.orders.push_back(o);
|
||||
ship.hsp = false;
|
||||
ship.prisH.prMax = 2;
|
||||
ship.prisH.prisoners.push_back({3, 4});
|
||||
ship.thrusters.push_back({1.5f, 2.5f});
|
||||
Writer w;
|
||||
WriteArchive wa(w);
|
||||
wa.obj(A("Ship"), ship);
|
||||
Node root = read_tree(w.bytes());
|
||||
std::vector<Issue> issues;
|
||||
shapes::Ship back;
|
||||
ReadArchive ra(root.children, issues, "");
|
||||
ra.obj(A("Ship"), back);
|
||||
CHECK_EQ(count(issues, Issue::Error), size_t(0));
|
||||
CHECK_EQ(count(issues, Issue::Warn), size_t(0));
|
||||
CHECK(back.hbq && back.bq2.orders.size() == 1 && back.bq2.orders[0].desID == 9);
|
||||
CHECK(!back.hsp && back.pop.groups.empty());
|
||||
CHECK(back.prisH.prMax == 2 && back.prisH.prisoners.size() == 1 && back.prisH.prisoners[0].prNum == 4);
|
||||
CHECK(back.thrusters.size() == 1 && back.thrusters[0].thm == 2.5f);
|
||||
Writer w2;
|
||||
WriteArchive wa2(w2);
|
||||
wa2.obj(A("Ship"), back);
|
||||
CHECK(w2.bytes() == w.bytes());
|
||||
|
||||
// Fleet: optional legacy tags absent, HFPlan false -> no FPlan; Sys: vnh gate
|
||||
shapes::Fleet f;
|
||||
f.ftName = "Fleet 1";
|
||||
f.hfPlan = true;
|
||||
shapes::Waypoint wp;
|
||||
wp.wpt = 272;
|
||||
f.fplan.wpts.push_back(wp);
|
||||
Writer w3;
|
||||
WriteArchive wa3(w3);
|
||||
wa3.obj(A("Flt"), f);
|
||||
Node r3 = read_tree(w3.bytes());
|
||||
shapes::Fleet fb;
|
||||
ReadArchive ra3(r3.children, issues, "");
|
||||
ra3.obj(A("Flt"), fb);
|
||||
CHECK(fb.hfPlan && fb.fplan.wpts.size() == 1 && fb.fplan.wpts[0].wpt == 272 && fb.fplan.wpts[0].nrtFramed);
|
||||
CHECK(!fb.sysID && !fb.caps);
|
||||
CHECK(fb.ftName == "Fleet 1");
|
||||
Writer w4;
|
||||
WriteArchive wa4(w4);
|
||||
wa4.obj(A("Flt"), fb);
|
||||
CHECK(w4.bytes() == w3.bytes());
|
||||
CHECK_EQ(count(issues, Issue::Error), size_t(0));
|
||||
}
|
||||
|
||||
// --- 9. gzip container ---------------------------------------------------------------
|
||||
static void test_gzip() {
|
||||
Bytes data;
|
||||
for (int i = 0; i < 5000; ++i) data.push_back(uint8_t(i * 7));
|
||||
Bytes gz = gzip(data.data(), data.size());
|
||||
CHECK(is_gzip(gz.data(), gz.size()));
|
||||
CHECK(gunzip(gz.data(), gz.size()) == data);
|
||||
CHECK(inflate_container(data.data(), data.size()) == data); // not gzip: passthrough
|
||||
bool threw = false;
|
||||
try {
|
||||
Bytes bad = gz;
|
||||
bad[bad.size() / 2] ^= 0xff;
|
||||
gunzip(bad.data(), bad.size());
|
||||
} catch (const GzipError&) {
|
||||
threw = true;
|
||||
}
|
||||
CHECK(threw);
|
||||
}
|
||||
|
||||
// --- 10. dump formatting: Python float repr and JSON quoting ---------------------------
|
||||
static void test_dump_format() {
|
||||
CHECK_EQ(py_float_repr(240.0), std::string("240.0"));
|
||||
CHECK_EQ(py_float_repr(1.0), std::string("1.0"));
|
||||
CHECK_EQ(py_float_repr(0.0), std::string("0.0"));
|
||||
CHECK_EQ(py_float_repr(double(3.4028235e38f)), std::string("3.4028234663852886e+38"));
|
||||
CHECK_EQ(py_float_repr(double(bits_f32(0x0000002a))), std::string("5.885453550164232e-44"));
|
||||
CHECK_EQ(py_float_repr(1e-05), std::string("1e-05"));
|
||||
CHECK_EQ(py_float_repr(0.0001), std::string("0.0001"));
|
||||
CHECK_EQ(py_float_repr(1e16), std::string("1e+16"));
|
||||
CHECK_EQ(py_float_repr(1234567890123456.0), std::string("1234567890123456.0"));
|
||||
CHECK_EQ(py_float_repr(double(-0.18487215f)), std::string("-0.18487215042114258"));
|
||||
CHECK_EQ(py_float_repr(double(2.4307494f)), std::string("2.4307494163513184"));
|
||||
CHECK_EQ(json_quote_cp1252("a\"b\\c\n"), std::string("\"a\\\"b\\\\c\\n\""));
|
||||
CHECK_EQ(json_quote_cp1252(std::string("Kor\x92Voth")), std::string("\"Kor\\u2019Voth\""));
|
||||
CHECK_EQ(json_quote_cp1252(std::string("\x81")), std::string("\"\\ufffd\""));
|
||||
CHECK_EQ(json_quote_cp1252(std::string("caf\xe9")), std::string("\"caf\\u00e9\""));
|
||||
}
|
||||
|
||||
int main() {
|
||||
test_primitives_bytes();
|
||||
test_frames();
|
||||
test_resync();
|
||||
test_cp1252();
|
||||
test_hints();
|
||||
test_raw_frame();
|
||||
test_typed_summary();
|
||||
test_typed_conditionals();
|
||||
test_gzip();
|
||||
test_dump_format();
|
||||
std::printf("test_stream: %s (%d failures)\n", fails ? "FAILED" : "ok", fails);
|
||||
return fails ? 1 : 0;
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue