The serializer recovery reaches this repo as a generated wire schema
(include/generated/sots_stream_schema.h, 386 classes / 2042 items): for each
class, the ordered sequence of items its Write puts on the stream. Facts only —
no field offsets, no sizeof, no strides. This engine reads and writes the on-disk
format; it does not inherit the original's memory layout.
The table is a specification, not a program: the recovery is a linear pass over
Write, so it cannot see Write's branches (StarShip's BQ2 is gated on hbq but
listed unconditionally) and it flattens container loops. A codec driven off it
would desynchronise. The hand-written io() shapes stay the codec; SchemaProbe
(probe.h) walks them with every branch taken and test_wire_schema LCS-aligns
that against the table — 56 shapes bound, 657 items matched, 0 mismatches.
Four defects the check found, all invisible to a round-trip test:
- SystemParams field 1 is a string, not an int. It is the empty string in
every save, and an empty string is four zero bytes — byte-identical to the
int 0, so it round-tripped by luck. A named planet would have desynced.
- ObservedTech/ObservedWeapon odet is a bool, not an int. Byte-safe only
because a 4-char tag makes a bool item and an int item both 12 bytes.
- SpeciesRatios nv and ShipRecords srbd are counts, not fields.
CoverageArchive separates items a field names from items a Node merely carries,
because a byte-identical round trip is not a coverage claim. Typed coverage of a
real save goes 37.9% -> 97.1% (97.2/97.2/97.6 on the others) with the round trip
still byte-identical, by typing TechTree (both NumTechs sections), Events,
ShipRecs, sprjs, civr, comms, spy2, spymgr, aid, Ojvs, AIEnf, FNG, trdmgr and
the Des section/gun-bank tree. Ratchet at 95%.
trdmgr resolves a recorded trap: ServerTradeManager's Read/Write really are the
inherited no-op, but the call is virtual and ServerTradeManagerImpl has the real
serializer. Same shape resolves IServerSpyManager -> ServerSpyManager.
ctest 34/34, clean_room_check OK, test_save skips cleanly with SOTS_SAVES_DIR unset.
270 lines
9.8 KiB
C++
270 lines
9.8 KiB
C++
// mars::stream — SchemaProbe: a fourth archive that records what a shape's
|
|
// io() puts on the stream, so the shape can be checked against the wire schema
|
|
// recovered from the game's own serializers (include/generated/sots_stream_schema.h).
|
|
//
|
|
// The three working archives read, write and build hints. This one executes
|
|
// nothing: it walks io() and appends one Item per stream item, in order. Two
|
|
// choices make its output directly comparable to the generated table:
|
|
//
|
|
// * `when(cond, body)` runs the body **unconditionally**. The recovery is a
|
|
// linear pass over the game's Write and cannot see Write's branches either,
|
|
// so both sides list every branch. Comparing "all branches taken" against
|
|
// "all branches taken" is the only alignment that means anything.
|
|
// * `narr` emits the count item and then the element fields as *siblings*
|
|
// (member == false), which is exactly how the flattened recovery presents a
|
|
// container loop.
|
|
//
|
|
// `any()` / `raw_frame()` — the escape hatch shapes.h uses for bodies it does
|
|
// not model — record Opaque. Opaque items are the coverage debt: the shape
|
|
// round-trips them byte-for-byte by carrying the Node, but it does not
|
|
// understand them. Counting Opaque against the wire schema is how this
|
|
// codebase measures how much of the save it actually reads.
|
|
#pragma once
|
|
|
|
#include <cstdint>
|
|
#include <map>
|
|
#include <optional>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "archive.h"
|
|
|
|
namespace mars::stream {
|
|
|
|
class SchemaProbe {
|
|
public:
|
|
static constexpr bool reading = false, writing = false, building = false;
|
|
|
|
enum class P : uint8_t { Unknown, I32, I64, F32, Bool, Str, Frame, Raw };
|
|
enum class S : uint8_t { Scalar, Frame, CArr, NArr, Raw };
|
|
|
|
struct Item {
|
|
std::string tag; // on-disk tag ("." when the game writes a NULL name)
|
|
P prim = P::Unknown;
|
|
S shape = S::Scalar;
|
|
bool member = true; // false: a container element
|
|
bool optional = false; // the shape consumes it only if the tag matches
|
|
bool opaque = false; // carried as a Node: round-tripped, not understood
|
|
};
|
|
|
|
std::vector<Item> items;
|
|
bool has_rest = false; // the shape absorbs a trailing tail into `extra`
|
|
|
|
// --- scalars ------------------------------------------------------------
|
|
void i32(Tag t, int32_t&) { add(t, P::I32, S::Scalar); }
|
|
void f32(Tag t, float&) { add(t, P::F32, S::Scalar); }
|
|
void b(Tag t, bool&) { add(t, P::Bool, S::Scalar); }
|
|
void i64(Tag t, int64_t&) { add(t, P::I64, S::Scalar); }
|
|
void str(Tag t, std::string&) { add(t, P::Str, S::Scalar); }
|
|
void vec3(Tag t, Vec3&) { add(t, P::Frame, S::Frame); } // written as a Vector3 frame
|
|
|
|
void any(Tag t, Node&) { add(t, P::Unknown, S::Frame).opaque = true; }
|
|
// raw_frame is a *framed* item (the RNG block) whose body is one opaque
|
|
// payload, so on the wire it is a frame like any other nested object.
|
|
void raw_frame(Tag t, Node&) { add(t, P::Frame, S::Frame).opaque = true; }
|
|
|
|
// --- optional named items ------------------------------------------------
|
|
void opt_i32(Tag t, std::optional<int32_t>&) { add(t, P::I32, S::Scalar).optional = true; }
|
|
void opt_f32(Tag t, std::optional<float>&) { add(t, P::F32, S::Scalar).optional = true; }
|
|
void opt_b(Tag t, std::optional<bool>&) { add(t, P::Bool, S::Scalar).optional = true; }
|
|
void opt_any(Tag t, std::optional<Node>&) {
|
|
Item& i = add(t, P::Unknown, S::Frame);
|
|
i.optional = true;
|
|
i.opaque = true;
|
|
}
|
|
template <class T>
|
|
void opt_obj(Tag t, std::optional<T>&) {
|
|
add(t, P::Frame, S::Frame).optional = true;
|
|
}
|
|
|
|
// --- framed struct / arrays ------------------------------------------------
|
|
// A nested shape is one item here; the nested class is its own wire entry.
|
|
template <class T>
|
|
void obj(Tag t, T&) {
|
|
add(t, P::Frame, S::Frame);
|
|
}
|
|
template <class T>
|
|
void obj_flex(Tag t, T& v, bool&) {
|
|
obj(t, v);
|
|
}
|
|
template <class T>
|
|
void carr(Tag t, std::vector<T>&) {
|
|
add(t, P::Frame, S::CArr);
|
|
}
|
|
template <class T>
|
|
void carr_flex(Tag t, std::vector<T>& v, bool&) {
|
|
carr(t, v);
|
|
}
|
|
template <class T, class F>
|
|
void narr(Tag t, std::vector<T>&, F elem) {
|
|
add(t, P::I32, S::NArr);
|
|
T tmp{};
|
|
size_t first = items.size();
|
|
elem(*this, tmp);
|
|
for (size_t i = first; i < items.size(); ++i) items[i].member = false;
|
|
}
|
|
|
|
// --- control flow: both sides list every branch ------------------------------
|
|
template <class F>
|
|
void when(bool, F body) {
|
|
body(*this);
|
|
}
|
|
template <class T, class F>
|
|
void repeat(const char*, std::vector<T>&, F elem) {
|
|
T tmp{};
|
|
size_t first = items.size();
|
|
elem(*this, tmp);
|
|
for (size_t i = first; i < items.size(); ++i) items[i].member = false;
|
|
}
|
|
void rest(std::vector<Node>&) { has_rest = true; }
|
|
|
|
// Run one shape's io() and return its item sequence.
|
|
template <class T>
|
|
static SchemaProbe of() {
|
|
SchemaProbe p;
|
|
T tmp{};
|
|
tmp.io(p);
|
|
return p;
|
|
}
|
|
|
|
private:
|
|
Item& add(Tag t, P prim, S shape) {
|
|
Item i;
|
|
i.tag = t.disk; // what actually goes on the wire, not the schema name
|
|
i.prim = prim;
|
|
i.shape = shape;
|
|
items.push_back(std::move(i));
|
|
return items.back();
|
|
}
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// CoverageArchive — how much of a real save the shapes actually understand.
|
|
//
|
|
// A byte-identical round trip is not a coverage claim. shapes.h reaches it
|
|
// partly by typing fields and partly by carrying whole bodies as generic Nodes
|
|
// (`ar.any`, `ar.raw_frame`, and the `ar.rest` tail): a Node round-trips
|
|
// trivially because it is copied verbatim. This archive runs a *populated*
|
|
// shape's io() and separates the two — every item a field names is `typed`,
|
|
// every item that only survives because a Node carried it is `opaque`.
|
|
//
|
|
// Unlike SchemaProbe this one honours `when()`, because it walks real data.
|
|
class CoverageArchive {
|
|
public:
|
|
static constexpr bool reading = false, writing = false, building = false;
|
|
|
|
size_t typed = 0, opaque = 0;
|
|
std::map<std::string, size_t> opaque_by_tag; // where the untyped items are
|
|
|
|
static size_t count(const Node& n) {
|
|
size_t c = 1;
|
|
for (const Node& k : n.children) c += count(k);
|
|
return c;
|
|
}
|
|
|
|
void i32(Tag, int32_t&) { ++typed; }
|
|
void f32(Tag, float&) { ++typed; }
|
|
void b(Tag, bool&) { ++typed; }
|
|
void i64(Tag, int64_t&) { ++typed; }
|
|
void str(Tag, std::string&) { ++typed; }
|
|
void vec3(Tag, Vec3&) { typed += 4; } // the frame plus three floats
|
|
|
|
void any(Tag t, Node& v) { charge(t.name, count(v)); }
|
|
void raw_frame(Tag t, Node& v) { charge(t.name, count(v)); }
|
|
|
|
void opt_i32(Tag, std::optional<int32_t>& v) { typed += v.has_value(); }
|
|
void opt_f32(Tag, std::optional<float>& v) { typed += v.has_value(); }
|
|
void opt_b(Tag, std::optional<bool>& v) { typed += v.has_value(); }
|
|
void opt_any(Tag t, std::optional<Node>& v) {
|
|
if (v) charge(t.name, count(*v));
|
|
}
|
|
template <class T>
|
|
void opt_obj(Tag t, std::optional<T>& v) {
|
|
if (v) obj(t, *v);
|
|
}
|
|
|
|
template <class T>
|
|
void obj(Tag t, T& v) {
|
|
++typed; // the frame item itself
|
|
const char* save = cur_;
|
|
cur_ = *t.name ? t.name : (*T::kStreamName ? T::kStreamName : cur_);
|
|
v.io(*this);
|
|
cur_ = save;
|
|
}
|
|
template <class T>
|
|
void obj_flex(Tag t, T& v, bool&) {
|
|
obj(t, v);
|
|
}
|
|
template <class T>
|
|
void carr(Tag t, std::vector<T>& v) {
|
|
typed += 2; // the frame item and its "." count
|
|
for (T& e : v) elem_of(t.name, e);
|
|
}
|
|
template <class T>
|
|
void carr_flex(Tag t, std::vector<T>& v, bool&) {
|
|
carr(t, v);
|
|
}
|
|
template <class T, class F>
|
|
void narr(Tag, std::vector<T>& v, F fn) {
|
|
++typed; // the count item
|
|
for (T& e : v) fn(*this, e);
|
|
}
|
|
// A tag for the tail is not available here; `rest` is charged to "<rest>".
|
|
template <class F>
|
|
void when(bool cond, F body) {
|
|
if (cond) body(*this);
|
|
}
|
|
template <class T, class F>
|
|
void repeat(const char*, std::vector<T>& v, F fn) {
|
|
for (T& e : v) fn(*this, e);
|
|
}
|
|
void rest(std::vector<Node>& v) {
|
|
for (const Node& n : v) charge(std::string("<rest:") + cur_ + ">", count(n));
|
|
}
|
|
|
|
private:
|
|
const char* cur_ = "root";
|
|
void charge(const std::string& tag, size_t n) {
|
|
opaque += n;
|
|
if (n) opaque_by_tag[tag] += n;
|
|
}
|
|
void elem_of(const char* tag, Node& n) { charge(tag, count(n)); }
|
|
void elem_of(const char*, int32_t&) { ++typed; }
|
|
void elem_of(const char*, float&) { ++typed; }
|
|
void elem_of(const char*, std::string&) { ++typed; }
|
|
template <class T>
|
|
void elem_of(const char* tag, T& v) {
|
|
++typed; // the element's own frame
|
|
const char* save = cur_;
|
|
cur_ = *tag ? tag : cur_;
|
|
v.io(*this);
|
|
cur_ = save;
|
|
}
|
|
};
|
|
|
|
inline const char* probe_prim_name(SchemaProbe::P p) {
|
|
switch (p) {
|
|
case SchemaProbe::P::Unknown: return "?";
|
|
case SchemaProbe::P::I32: return "i32";
|
|
case SchemaProbe::P::I64: return "i64";
|
|
case SchemaProbe::P::F32: return "f32";
|
|
case SchemaProbe::P::Bool: return "bool";
|
|
case SchemaProbe::P::Str: return "str";
|
|
case SchemaProbe::P::Frame: return "frame";
|
|
case SchemaProbe::P::Raw: return "raw";
|
|
}
|
|
return "?";
|
|
}
|
|
|
|
inline const char* probe_shape_name(SchemaProbe::S s) {
|
|
switch (s) {
|
|
case SchemaProbe::S::Scalar: return "scalar";
|
|
case SchemaProbe::S::Frame: return "frame";
|
|
case SchemaProbe::S::CArr: return "carr";
|
|
case SchemaProbe::S::NArr: return "narr";
|
|
case SchemaProbe::S::Raw: return "raw";
|
|
}
|
|
return "?";
|
|
}
|
|
|
|
} // namespace mars::stream
|