// mars::stream — archives that drive a typed shape's io() field list. // // A shape is a plain struct with // static constexpr const char* kStreamName = "Sys"; // "" when the frame is unnamed ("." elements) // template void io(Ar& ar) { ar.i32(A("Idx"), idx); ... } // listing its fields in on-disk order. The same io() is driven by three // archives: // ReadArchive fills the struct from a generic Node tree (walker output), // matching tags by name (A) or position (R), coercing kinds // and reporting deviations as Issues; // WriteArchive emits the struct through a Writer in the same order; // SchemaBuilder records the field list as a Desc so the walker can type the // frame's children before any shape is applied. // // Tag helpers: A("Turn") = on-disk tag confirmed, matched by name. // R("treasury") = reference/positional field, written as "." (or // the given disk tag); matched by position. #pragma once #include #include #include #include #include #include #include #include #include #include "node.h" #include "schema.h" #include "writer.h" namespace mars::stream { struct Tag { const char* name; // schema name const char* disk; // tag emitted when writing bool positional; }; constexpr Tag A(const char* n) { return Tag{n, n, false}; } constexpr Tag R(const char* n, const char* disk = ".") { return Tag{n, disk, true}; } struct Vec3 { float x = 0, y = 0, z = 0; bool operator==(const Vec3& o) const { return x == o.x && y == o.y && z == o.z; } }; template struct has_io : std::false_type {}; template struct has_io> : std::true_type {}; // --------------------------------------------------------------------------- // ReadArchive // --------------------------------------------------------------------------- class ReadArchive { public: static constexpr bool reading = true, writing = false, building = false; ReadArchive(const std::vector& nodes, std::vector& issues, std::string path) : nodes_(&nodes), issues_(&issues), path_(std::move(path)) {} // --- scalars ------------------------------------------------------------ void i32(Tag t, int32_t& v) { if (const Node* n = take_named(t)) v = coerce_int(*n, fpath(t)); } void f32(Tag t, float& v) { if (const Node* n = take_named(t)) v = coerce_float(*n, fpath(t)); } void b(Tag t, bool& v) { if (const Node* n = take_named(t)) v = coerce_bool(*n, fpath(t)); } void i64(Tag t, int64_t& v) { if (const Node* n = take_named(t)) v = coerce_int64(*n, fpath(t)); } void str(Tag t, std::string& v) { if (const Node* n = take_named(t)) v = coerce_string(*n, fpath(t)); } void vec3(Tag t, Vec3& v) { if (const Node* n = take_named(t)) v = coerce_vec3(*n, fpath(t)); } // generic item kept as a node copy ("any") void any(Tag t, Node& v) { if (const Node* n = take_named(t)) v = *n; } void raw_frame(Tag t, Node& v) { any(t, v); } // --- optional named items (consumed only when the next tag matches) ------ void opt_i32(Tag t, std::optional& v) { if (next_is(t)) v = coerce_int(*take(), fpath(t)); } void opt_f32(Tag t, std::optional& v) { if (next_is(t)) v = coerce_float(*take(), fpath(t)); } void opt_b(Tag t, std::optional& v) { if (next_is(t)) v = coerce_bool(*take(), fpath(t)); } void opt_any(Tag t, std::optional& v) { if (next_is(t)) v = *take(); } template void opt_obj(Tag t, std::optional& v) { if (next_is(t)) { v.emplace(); obj(t, *v); } } // --- framed struct --------------------------------------------------------- template void obj(Tag t, T& v) { const Node* n = take_named(t); if (!n) return; read_frame(*n, v, fpath(t)); } // struct that may also appear inline (unframed); `framed` records which template void obj_flex(Tag t, T& v, bool& framed) { const Node* n = peek(); if (n && !n->is_complex()) { framed = false; v.io(*this); return; } framed = true; obj(t, v); } // --- framed array: "." count + n x "." element ----------------------------- template void carr(Tag t, std::vector& v) { const Node* n = take_named(t); if (!n) return; std::string p = fpath(t); if (!n->is_complex()) { issue(Issue::Error, p, n, "expected framed array, found " + std::string(kind_name(n->kind)) + " '" + n->name + "'"); return; } ReadArchive sub(n->children, *issues_, p); v.clear(); if (sub.done()) return; int32_t count = coerce_int(*sub.take(), p); size_t remaining = sub.remaining(); if (count < 0 || size_t(count) > remaining) { issue(Issue::Error, p, n, "framed-array count " + std::to_string(count) + " exceeds " + std::to_string(remaining) + " item(s)"); count = int32_t(std::min(size_t(std::max(count, 0)), remaining)); } for (int32_t i = 0; i < count; ++i) { if (sub.done()) { issue(Issue::Error, p, n, "framed array truncated: " + std::to_string(i) + " of " + std::to_string(count)); break; } v.emplace_back(); sub.read_elem(v.back(), p + "[" + std::to_string(i) + "]"); } if (!sub.done()) issue(Issue::Warn, p, sub.peek(), std::to_string(sub.remaining()) + " item(s) after framed array elements"); } template void carr_flex(Tag t, std::vector& v, bool& framed) { const Node* n = peek(); if (n && !n->is_complex()) { framed = false; narr(t, v, [](ReadArchive& a, T& e) { a.read_elem(e, a.path_); }); return; } framed = true; carr(t, v); } // --- inline array: named count + n x element ------------------------------- template void narr(Tag t, std::vector& v, F elem) { const Node* cn = take_named(t); if (!cn) return; std::string p = fpath(t); int32_t count = coerce_int(*cn, p); size_t rem = remaining(); v.clear(); if (count < 0 || size_t(count) > rem) { issue(Issue::Error, p, cn, "array count " + std::to_string(count) + " exceeds the " + std::to_string(rem) + " item(s) left in the frame"); count = int32_t(std::min(size_t(std::max(count, 0)), rem)); } for (int32_t i = 0; i < count; ++i) { if (done()) { issue(Issue::Error, p, nullptr, "array truncated: " + std::to_string(i) + " of " + std::to_string(count) + " elements present"); break; } size_t before = i_; v.emplace_back(); std::string save = path_; path_ = p + "[" + std::to_string(i) + "]"; elem(*this, v.back()); path_ = save; if (i_ == before) { issue(Issue::Error, p, peek(), "array element " + std::to_string(i) + " consumed nothing; stopping"); v.pop_back(); break; } } } // --- conditional group ----------------------------------------------------- template void when(bool cond, F body) { if (cond) body(*this); } // --- uncounted repetition while the next tag is `lead` --------------------- template void repeat(const char* lead, std::vector& v, F elem) { v.clear(); while (!done() && peek()->tagged && peek()->name == lead) { size_t before = i_; v.emplace_back(); std::string save = path_; path_ = path_ + "/" + lead + "[" + std::to_string(v.size() - 1) + "]"; elem(*this, v.back()); path_ = save; if (i_ == before) { v.pop_back(); break; } } } // --- everything left in the frame, kept generic ---------------------------- void rest(std::vector& v) { v.assign(nodes_->begin() + long(i_), nodes_->end()); i_ = nodes_->size(); absorbed_ = true; } // --- cursor ---------------------------------------------------------------- bool done() const { return i_ >= nodes_->size(); } size_t remaining() const { return nodes_->size() - i_; } const Node* peek() const { return done() ? nullptr : &(*nodes_)[i_]; } const Node* take() { return &(*nodes_)[i_++]; } // After io(): leftover items are unexpected unless rest() absorbed them. void finish(const Node* frame) { if (!done() && !absorbed_) { issue(Issue::Warn, path_, peek(), std::to_string(remaining()) + " unexpected item(s) at end of frame"); (void)frame; i_ = nodes_->size(); } } template void read_frame(const Node& n, T& v, const std::string& p) { if (!n.is_complex()) { issue(Issue::Error, p, &n, "expected frame, found " + std::string(kind_name(n.kind)) + " '" + n.name + "'"); return; } ReadArchive sub(n.children, *issues_, p); v.io(sub); sub.finish(&n); } // one array element: struct -> "." frame, int32 -> "." int, Node -> any template void read_elem(T& e, const std::string& p) { const Node* n = take(); if constexpr (std::is_same_v) e = coerce_int(*n, p); else if constexpr (std::is_same_v) e = *n; else read_frame(*n, e, p); } private: const std::vector* nodes_; size_t i_ = 0; std::vector* issues_; std::string path_; bool absorbed_ = false; std::string fpath(Tag t) const { return path_ + "/" + t.name; } void issue(Issue::Level l, const std::string& p, const Node* n, std::string msg) { issues_->push_back(Issue{l, p, n ? n->offset : 0u, std::move(msg)}); } static std::string lower(std::string s) { for (char& c : s) if (c >= 'A' && c <= 'Z') c = char(c - 'A' + 'a'); return s; } bool next_is(Tag t) const { const Node* n = peek(); return n && n->tagged && lower(n->name) == lower(t.name); } const Node* take_named(Tag t) { std::string p = fpath(t); const Node* n = peek(); if (!n) { issue(t.positional ? Issue::Warn : Issue::Error, p, nullptr, "missing field '" + std::string(t.name) + "' (frame ended)"); return nullptr; } if (!t.positional) { if (!n->tagged || n->name != t.name) { size_t j = i_; for (; j < nodes_->size(); ++j) if ((*nodes_)[j].tagged && (*nodes_)[j].name == t.name) break; if (j == nodes_->size()) { issue(Issue::Error, p, n, "expected '" + std::string(t.name) + "', found '" + (n->tagged ? n->name : "") + "'; field missing"); return nullptr; } std::string names; for (size_t k = i_; k < j && k < i_ + 6; ++k) names += (k > i_ ? ", '" : "'") + (*nodes_)[k].name + "'"; issue(Issue::Warn, p, n, std::to_string(j - i_) + " unexpected item(s) before '" + t.name + "': " + names); i_ = j; } } else if (n->tagged && lower(n->name) != lower(t.name)) { issue(Issue::Info, p, n, "tag '" + n->name + "' read positionally as '" + t.name + "'"); } return take(); } // --- coercions (mirror the reference reader's rules) ---------------------- int32_t coerce_int(const Node& n, const std::string& p) { switch (n.kind) { case Kind::Int: return n.as_int(); case Kind::Float: return n.raw.size() == 4 ? n.as_int() : fail_int(n, p, "int"); case Kind::Bool: return n.as_bool() ? 1 : 0; case Kind::Int64: issue(Issue::Warn, p, &n, "int expected, int64 read"); return int32_t(n.as_int64()); default: return fail_int(n, p, "int"); } } float coerce_float(const Node& n, const std::string& p) { switch (n.kind) { case Kind::Float: return n.as_float(); case Kind::Int: return n.raw.size() == 4 ? n.as_float() : float(fail_int(n, p, "float")); case Kind::Bool: return n.as_bool() ? 1.f : 0.f; default: return float(fail_int(n, p, "float")); } } bool coerce_bool(const Node& n, const std::string& p) { if (n.kind == Kind::Bool) return n.as_bool(); if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4) { if (n.raw[1] || n.raw[2] || n.raw[3] || n.raw[0] > 1) issue(Issue::Warn, p, &n, "bool expected, word " + hex(n.raw.data(), 4) + " read"); return n.raw[0] != 0; } return fail_int(n, p, "bool") != 0; } int64_t coerce_int64(const Node& n, const std::string& p) { if (n.kind == Kind::Int64) return n.as_int64(); if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4) { issue(Issue::Warn, p, &n, "int64 expected, 4-byte word read (width mismatch)"); return n.as_int(); } return fail_int(n, p, "int64"); } std::string coerce_string(const Node& n, const std::string& p) { if (n.kind == Kind::String) return n.as_string(); // an empty string is "len 0" = 4 zero bytes, byte-identical to int 0 if ((n.kind == Kind::Int || n.kind == Kind::Float) && n.raw.size() == 4 && n.as_int() == 0) return ""; fail_int(n, p, "string"); return ""; } Vec3 coerce_vec3(const Node& n, const std::string& p) { Vec3 v; if (n.is_complex()) { const auto& ch = n.children; if (ch.size() == 1 && ch[0].kind == Kind::Raw && ch[0].raw.size() == 12) { v.x = rd_f32(ch[0].raw.data()); v.y = rd_f32(ch[0].raw.data() + 4); v.z = rd_f32(ch[0].raw.data() + 8); return v; } if (ch.size() == 3 && !ch[0].is_complex() && !ch[1].is_complex() && !ch[2].is_complex() && ch[0].raw.size() == 4 && ch[1].raw.size() == 4 && ch[2].raw.size() == 4) { v.x = ch[0].as_float(); v.y = ch[1].as_float(); v.z = ch[2].as_float(); return v; } issue(Issue::Error, p, &n, "vec3 frame has unexpected body (" + std::to_string(ch.size()) + " items)"); return v; } if (n.kind == Kind::Raw && n.raw.size() == 12) { v.x = rd_f32(n.raw.data()); v.y = rd_f32(n.raw.data() + 4); v.z = rd_f32(n.raw.data() + 8); return v; } issue(Issue::Error, p, &n, "expected vec3, found " + std::string(kind_name(n.kind))); return v; } int32_t fail_int(const Node& n, const std::string& p, const char* want) { issue(Issue::Error, p, &n, std::string("expected ") + want + ", read " + (n.is_complex() ? "frame '" + n.name + "'" : kind_name(n.kind))); return n.raw.size() >= 4 ? n.as_int() : 0; } }; // --------------------------------------------------------------------------- // WriteArchive // --------------------------------------------------------------------------- class WriteArchive { public: static constexpr bool reading = false, writing = true, building = false; explicit WriteArchive(Writer& w) : w_(w) {} void i32(Tag t, int32_t& v) { w_.int32(t.disk, v); } void f32(Tag t, float& v) { w_.float32(t.disk, v); } void b(Tag t, bool& v) { w_.boolean(t.disk, v); } void i64(Tag t, int64_t& v) { w_.int64(t.disk, v); } void str(Tag t, std::string& v) { w_.string(t.disk, v); } void vec3(Tag t, Vec3& v) { w_.begin(t.disk); w_.float32(".", v.x); w_.float32(".", v.y); w_.float32(".", v.z); w_.end(); } void any(Tag, Node& v) { w_.node(v); } void raw_frame(Tag t, Node& v) { any(t, v); } void opt_i32(Tag t, std::optional& v) { if (v) w_.int32(t.disk, *v); } void opt_f32(Tag t, std::optional& v) { if (v) w_.float32(t.disk, *v); } void opt_b(Tag t, std::optional& v) { if (v) w_.boolean(t.disk, *v); } void opt_any(Tag, std::optional& v) { if (v) w_.node(*v); } template void opt_obj(Tag t, std::optional& v) { if (v) obj(t, *v); } template void obj(Tag t, T& v) { w_.begin(t.disk); v.io(*this); w_.end(); } template void obj_flex(Tag t, T& v, bool& framed) { if (framed) obj(t, v); else v.io(*this); } template void carr(Tag t, std::vector& v) { w_.begin(t.disk); w_.int32(".", int32_t(v.size())); for (T& e : v) write_elem(e); w_.end(); } template void carr_flex(Tag t, std::vector& v, bool& framed) { if (framed) { carr(t, v); return; } w_.int32(t.disk, int32_t(v.size())); for (T& e : v) write_elem(e); } template void narr(Tag t, std::vector& v, F elem) { w_.int32(t.disk, int32_t(v.size())); for (T& e : v) elem(*this, e); } template void when(bool cond, F body) { if (cond) body(*this); } template void repeat(const char*, std::vector& v, F elem) { for (T& e : v) elem(*this, e); } void rest(std::vector& v) { for (const Node& n : v) w_.node(n); } template void write_elem(T& e) { if constexpr (std::is_same_v) w_.int32(".", e); else if constexpr (std::is_same_v) w_.node(e); else { w_.begin("."); e.io(*this); w_.end(); } } private: Writer& w_; }; // --------------------------------------------------------------------------- // SchemaBuilder — runs io() on default-constructed shapes to record hints. // --------------------------------------------------------------------------- class SchemaBuilder { public: static constexpr bool reading = false, writing = false, building = true; struct Context { Registry& reg; std::map> strong; // A/R fields: name -> kinds seen std::map weak; // Opt fields: lowest priority std::map memo; std::set in_progress; const Desc* raw_desc = nullptr; explicit Context(Registry& r) : reg(r) {} }; SchemaBuilder(Context& ctx, Desc* cur) : ctx_(ctx), cur_(cur) {} void i32(Tag t, int32_t&) { prim(t, Prim::Int); } void f32(Tag t, float&) { prim(t, Prim::Float); } void b(Tag t, bool&) { prim(t, Prim::Bool); } void i64(Tag t, int64_t&) { prim(t, Prim::Int64); } void str(Tag t, std::string&) { prim(t, Prim::String); } void vec3(Tag t, Vec3&) { nohint(t); } void any(Tag t, Node&) { nohint(t); } void raw_frame(Tag t, Node&) { nohint(t); if (!ctx_.raw_desc) { Desc d; d.type = Desc::Raw; d.name = t.name; ctx_.raw_desc = ctx_.reg.add(std::move(d)); } ctx_.reg.shapes.emplace(t.name, ctx_.raw_desc); } void opt_i32(Tag t, std::optional&) { prim(t, Prim::Int, true); } void opt_f32(Tag t, std::optional&) { prim(t, Prim::Float, true); } void opt_b(Tag t, std::optional&) { prim(t, Prim::Bool, true); } void opt_any(Tag t, std::optional&) { close_prefix(); nohint(t); } template void opt_obj(Tag t, std::optional&) { close_prefix(); const Desc* d = describe(); by_name(t, Hint{Prim::None, d}); register_shape(t, d); } template void obj(Tag t, T&) { const Desc* d = describe(); push_prefix(Hint{Prim::None, d}); by_name(t, Hint{Prim::None, d}); register_shape(t, d); } template void obj_flex(Tag t, T& v, bool&) { close_prefix(); obj(t, v); } template void carr(Tag t, std::vector&) { Desc d; d.type = Desc::CArr; if constexpr (std::is_same_v) d.elem.kind = Prim::Int; else if constexpr (std::is_same_v) d.elem = Hint{}; else d.elem.sub = describe(); const Desc* cd = ctx_.reg.add(std::move(d)); push_prefix(Hint{Prim::None, cd}); // the NULL-name tag "." is never a frame hint key: it carries ints, // floats and frames alike (bare "." elements inside inline arrays) if (std::string(t.name) != ".") { by_name(t, Hint{Prim::None, cd}); if (*t.name) ctx_.reg.shapes.emplace(t.name, cd); } } template void carr_flex(Tag t, std::vector& v, bool&) { close_prefix(); carr(t, v); } template void narr(Tag t, std::vector&, F elem) { push_prefix(Hint{Prim::Int, nullptr}); close_prefix(); by_name(t, Hint{}); T tmp{}; elem(*this, tmp); // element fields register by name (prefix already closed) } template void when(bool, F body) { close_prefix(); body(*this); } template void repeat(const char*, std::vector&, F elem) { close_prefix(); T tmp{}; elem(*this, tmp); } void rest(std::vector&) { close_prefix(); } // Describe shape T (memoized per type) and register its named frame. template const Desc* describe() { std::type_index ti(typeid(T)); auto it = ctx_.memo.find(ti); if (it != ctx_.memo.end()) return it->second; if (ctx_.in_progress.count(ti)) return nullptr; // recursive shape: no positional hints ctx_.in_progress.insert(ti); Desc d; d.type = Desc::Shape; d.name = T::kStreamName; Desc* nd = ctx_.reg.add(std::move(d)); SchemaBuilder sub(ctx_, nd); T tmp{}; tmp.io(sub); ctx_.in_progress.erase(ti); ctx_.memo.emplace(ti, nd); if (!nd->name.empty()) ctx_.reg.shapes.emplace(nd->name, nd); return nd; } // Build the global catalog after every shape has been visited. static void finalize(Context& ctx, const std::map& manual_kinds) { std::set conflicts; for (auto& [name, kinds] : ctx.strong) { if (kinds.size() == 1) ctx.reg.kinds.emplace(name, *kinds.begin()); else conflicts.insert(name); } for (auto& [name, k] : ctx.weak) if (!conflicts.count(name)) ctx.reg.kinds.emplace(name, k); ctx.reg.kinds.erase("."); // the NULL-name tag carries ints, floats and frames alike for (auto& [name, k] : manual_kinds) ctx.reg.kinds[name] = k; } private: Context& ctx_; Desc* cur_; bool prefix_open_ = true; static std::string lower(std::string s) { for (char& c : s) if (c >= 'A' && c <= 'Z') c = char(c - 'A' + 'a'); return s; } void push_prefix(Hint h) { if (cur_ && prefix_open_) cur_->prefix.push_back(h); } void close_prefix() { prefix_open_ = false; } void by_name(Tag t, Hint h) { if (!cur_) return; cur_->by_name.emplace(t.name, h); cur_->by_name.emplace(lower(t.name), h); } void prim(Tag t, Prim k, bool opt = false) { if (opt) close_prefix(); else push_prefix(Hint{k, nullptr}); by_name(t, Hint{k, nullptr}); if (opt) ctx_.weak.emplace(t.name, k); else ctx_.strong[t.name].insert(k); } void nohint(Tag t) { push_prefix(Hint{}); by_name(t, Hint{}); } void register_shape(Tag t, const Desc* d) { if (!d) return; if (d->name.empty() && *t.name) ctx_.reg.shapes.emplace(t.name, d); } }; } // namespace mars::stream