merge lane E2: ship census into BuildTurnRecord; phase-granular --commit-blocked; T36 blocked on budget + ship construction

src/app/CMakeLists.txt

@@ -11,7 +11,11 @@ add_library(sots_app STATIC
   turn.cpp
   report.cpp)
 target_include_directories(sots_app PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/..)
-target_link_libraries(sots_app PUBLIC mars_stream mars_rng sots_game_sim)
+# game_design (and game_data under it) is here for one phase only: the tail's turn record
+# needs a design's hull size and defence-platform flag, which are recomputed from the section
+# catalog and are nowhere on the wire. No game data is embedded; the root is supplied at run
+# time and its absence costs exactly six fields.
+target_link_libraries(sots_app PUBLIC mars_stream mars_rng sots_game_sim game_design)
 target_compile_features(sots_app PUBLIC cxx_std_17)
 if(NOT MSVC)
   target_compile_options(sots_app PRIVATE -Wall -Wextra -Werror)

src/app/main.cpp

@@ -5,15 +5,21 @@
 // published phase order of all three turn drivers, runs the phases we hold, prints every
 // phase it did NOT run, and writes the result back through the engine's own writer.
 //
-// It reads no game data of its own. The save path comes from the command line or, for the
+// It embeds no game data. The save path comes from the command line or, for the test, from
-// test, from $SOTS_SAVES_DIR; nothing is embedded.
+// $SOTS_SAVES_DIR. One phase -- the tail's turn record -- needs the game's own section
+// catalog, because a design's hull size and its defence-platform flag are recomputed from it
+// and are nowhere on the wire; that root is named by --data or $SOTS_DATA_DIR, is never
+// embedded, and its absence costs exactly those six fields and nothing else.
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <fstream>
 #include <string>
+#include <vector>
 
 #include "app/report.h"
 #include "app/turn.h"
+#include "game/data/catalog.h"
 #include "mars/stream/gzip.h"
 #include "mars/stream/save.h"
 
@@ -21,26 +27,53 @@ namespace {
 
 int Usage() {
     std::fprintf(stderr,
-                 "usage: sots_turn SAVE [--out FILE] [--metric FILE] [--phases] [--verbose]\n"
+                 "usage: sots_turn SAVE [--out FILE] [--metric FILE] [--data DIR] [--phases]\n"
-                 "                      [--commit-blocked] [--commit-rng] [--roundtrip]\n"
+                 "                      [--verbose] [--commit-blocked[=IDS]] [--commit-rng]\n"
+                 "                      [--commit-blocked-except=IDS] [--roundtrip]\n"
                  "\n"
                  "  SAVE              a .sav to load (gzip or already inflated)\n"
                  "  --out FILE        write the post-turn save here\n"
                  "  --metric FILE     write the completion metric as JSON here\n"
+                 "  --data DIR        the game's data root (default $SOTS_DATA_DIR); without\n"
+                 "                    it the tail's ship census is not modelled\n"
                  "  --phases          print the whole phase table with this run's numbers\n"
                  "  --verbose         add each phase's standing note to the listing\n"
                  "  --commit-blocked  write results of phases whose inputs are unmodelled\n"
+                 "  --commit-blocked=IDS         only these blocked phases commit, by phase\n"
+                 "                               id, comma-separated (e.g. T36)\n"
+                 "  --commit-blocked-except=IDS  these blocked phases never commit\n"
                  "  --commit-rng      write the advanced generator state back\n"
-                 "  --roundtrip       re-serialise the UNTOUCHED save and check byte identity\n");
+                 "  --roundtrip       re-serialise the UNTOUCHED save and check byte identity\n"
+                 "\n"
+                 "The two id lists exist so a blocked phase's own closed and regressed counts\n"
+                 "can be measured apart from every other blocked phase's, rather than netted\n"
+                 "into one number for all of them.\n");
     return 2;
 }
 
+// "T36,P02" -> {"T36", "P02"}. Empty entries are dropped rather than matched against nothing.
+std::vector<std::string> SplitIds(const std::string& s) {
+    std::vector<std::string> out;
+    std::string cur;
+    for (char c : s) {
+        if (c == ',') {
+            if (!cur.empty()) out.push_back(cur);
+            cur.clear();
+        } else {
+            cur += c;
+        }
+    }
+    if (!cur.empty()) out.push_back(cur);
+    return out;
+}
+
 }  // namespace
 
 int main(int argc, char** argv) {
-    std::string in, out, metric;
+    std::string in, out, metric, dataDir;
     bool phases = false, verbose = false, roundtrip = false;
     sots::app::TurnOptions opt;
+    if (const char* env = std::getenv("SOTS_DATA_DIR")) dataDir = env;
 
     for (int i = 1; i < argc; ++i) {
         const std::string a = argv[i];
@@ -53,12 +86,21 @@ int main(int argc, char** argv) {
             if (!next(out)) return Usage();
         } else if (a == "--metric") {
             if (!next(metric)) return Usage();
+        } else if (a == "--data") {
+            if (!next(dataDir)) return Usage();
         } else if (a == "--phases") {
             phases = true;
         } else if (a == "--verbose") {
             phases = verbose = true;
         } else if (a == "--commit-blocked") {
             opt.commitBlocked = true;
+        } else if (a.rfind("--commit-blocked=", 0) == 0) {
+            opt.commitBlocked = true;
+            opt.commitOnly = SplitIds(a.substr(std::strlen("--commit-blocked=")));
+            if (opt.commitOnly.empty()) return Usage();
+        } else if (a.rfind("--commit-blocked-except=", 0) == 0) {
+            opt.commitExcept = SplitIds(a.substr(std::strlen("--commit-blocked-except=")));
+            if (opt.commitExcept.empty()) return Usage();
         } else if (a == "--commit-rng") {
             opt.commitRng = true;
         } else if (a == "--roundtrip") {
@@ -107,6 +149,26 @@ int main(int argc, char** argv) {
         if (!ok) return 1;
     }
 
+    // The one place the standalone needs the game's own data. Loading is total -- unreadable
+    // files become problems and the rest still loads -- so the catalog is accepted only when
+    // it actually holds sections, and a root that yields none is reported rather than used.
+    game::data::Catalog catalog;
+    if (!dataDir.empty()) {
+        catalog = game::data::load_catalog(dataDir);
+        if (catalog.sections.empty()) {
+            std::printf("data: %s holds no ship sections; the ship census stays unmodelled\n",
+                        dataDir.c_str());
+        } else {
+            opt.catalog = &catalog;
+            std::printf("data: %s -- %zu section(s) over %zu race(s), %zu load problem(s)\n",
+                        dataDir.c_str(), catalog.sections.size(), catalog.races.size(),
+                        catalog.problems.size());
+        }
+    } else {
+        std::printf("data: no root given (--data DIR / $SOTS_DATA_DIR); the tail's ship census "
+                    "stays unmodelled\n");
+    }
+
     const sots::app::TurnResult r = sots::app::RunStrategicTurn(doc.game, opt);
 
     if (phases) sots::app::PrintPhaseLog(stdout, r, verbose);

src/app/phase_catalog.cpp

@@ -217,12 +217,22 @@ constexpr PhaseDesc kTail[] = {
     {Driver::Tail, 34, "T34", "RecordObservedDesigns", PhaseStatus::Stub, ""},
     {Driver::Tail, 35, "T35", "RebuildPlayerReports", PhaseStatus::Stub, ""},
     {Driver::Tail, 36, "T36", "FinalizeTurnRecords", PhaseStatus::Blocked,
-     "fills every player's turn record and archives it by turn; must stay last. Seven of its "
+     "fills every player's turn record and archives it by turn; must stay last. Thirteen of "
-     "fields are recoverable from the wire and are reproduced -- turn, colony count, savings, "
+     "its fields are modelled and self-checked every run against the record the input save "
+     "already carries for its own turn: seven from the wire -- turn, colony count, savings, "
      "the savings delta, the completed-tech count, the summed population and the alliance mask "
-     "phase S04 rebuilt -- and the model is self-checked every run against the record the input "
+     "phase S04 rebuilt -- and, when a data root is given, the six per-hull-class ship counts, "
-     "save already carries for its own turn. Not committed: four further fields of the same "
+     "which need each design's hull size and defence-platform flag from the section catalog "
-     "record are unmodelled, and savings for the NEW turn comes from a blocked phase"},
+     "because neither is on the wire. Only 32 of the 480 census leaves the corpus archives are "
+     "nonzero, so cls1 (cruisers) and cls2's platform count are UNEXERCISED, not verified. "
+     "Still not committed, and now blocked on exactly two named things, neither of them here: "
+     "savings and the income derived from it come from P01/P02, which are blocked on the "
+     "per-system money output; and a ship the turn BUILDS never enters our fleet list, so the "
+     "census carries the pre-construction count. Measured with --commit-blocked=T36 on "
+     "turn1->turn2: 29 leaves closed, 7 regressed -- sav x3, inc x3 and one shpt[0] short by "
+     "exactly the one destroyer that turn completes. The archived record is one struct on the "
+     "wire, so those words cannot be omitted while the rest is written: committing is "
+     "all-or-nothing at the record"},
 };
 
 PhaseTally Tally(const PhaseDesc* p, std::size_t n) {

src/app/turn.cpp

@@ -17,6 +17,18 @@
 #include "game/sim/tuning.h"
 
 namespace sots::app {
+
+bool TurnOptions::CommitBlocked(const char* phaseId) const {
+    if (!commitBlocked) return false;
+    const std::string id(phaseId);
+    for (const auto& e : commitExcept)
+        if (e == id) return false;
+    if (commitOnly.empty()) return true;
+    for (const auto& o : commitOnly)
+        if (o == id) return true;
+    return false;
+}
+
 namespace {
 
 using mars::stream::Node;
@@ -114,7 +126,7 @@ void RunPlayerDriver(Player& p, const TurnOptions& opt, CountingRandom* rng,
         const int wouldBe = sim::SaturatingAdd(p.sav, b.net);
         ++t.fired[1];
         if (wouldBe != p.sav) ++t.wouldWrite[2];
-        if (opt.commitBlocked) {
+        if (opt.CommitBlocked("P02")) {
             p.sav = wouldBe;
             ++t.writes[2];
         }
@@ -271,10 +283,12 @@ struct TurnRecordAudit {
     int dangling = 0;            // owned-system ids the save's system table does not carry
     std::vector<std::string> firstMismatches;
     int missingArchive = 0;      // players with no archive element for the input turn
+    bool censusModelled = false; // a catalog was supplied, so six more fields are compared
 };
 
-TurnRecordAudit AuditTurnRecordsAgainstSave(const SaveGame& game) {
+TurnRecordAudit AuditTurnRecordsAgainstSave(const SaveGame& game, const ShipCensusIndex* census) {
     TurnRecordAudit a;
+    a.censusModelled = census && census->modelled();
     const std::int32_t inputTurn = game.sim.frame;
     for (std::size_t i = 0; i < game.sim.players.size(); ++i) {
         if (i >= game.sim.turnstats.players.size()) break;
@@ -290,7 +304,7 @@ TurnRecordAudit AuditTurnRecordsAgainstSave(const SaveGame& game) {
         // the one the load path wrote, so its alliance mask is predicted to be zero.
         const bool spineRan = inputTurn > EarliestArchivedTurn(hist);
         const TurnRecord built = BuildTurnRecord(game.sim.players[i].player, game.sim.systems,
-                                                 inputTurn, i, spineRan, &dangling);
+                                                 inputTurn, i, spineRan, &dangling, census);
         a.dangling += dangling;
         const TurnRecordDiff d = CompareTurnRecord(built, *stored);
         ++a.playersChecked;
@@ -307,25 +321,38 @@ void RunFinalizeTurnRecords(SaveGame& game, const TurnOptions& opt, PhaseRecord&
                             const TurnRecordAudit& audit,
                             const std::vector<std::int32_t>& allianceMasks) {
     rec.invocations = static_cast<int>(game.sim.players.size());
-    // Six fields per player would be written, plus a new archive element per player. Nothing
+    const bool commits = opt.CommitBlocked("T36");
-    // is committed by default: five further fields of the same element are unmodelled, and
+    // Rebuilt here rather than reused from the load-time audit: the record this phase archives
-    // the two the model DOES hold for the new turn -- savings and the income derived from it
+    // is the state at the END of the turn, and a census taken before the phases ran would be
-    // -- are downstream of a blocked phase, so every element written would be wrong in a way
+    // the wrong one the moment a phase adds or removes a ship. Nothing in the standalone does
-    // the untouched save is not. `--commit-blocked` writes them anyway, so the claim that
+    // that today, which makes the two identical today and not by construction.
-    // committing makes things worse is a measurement rather than an argument.
+    ShipCensusIndex censusNow;
+    if (opt.catalog) censusNow = ShipCensusIndex(game, *opt.catalog);
+    const ShipCensusIndex* census = opt.catalog ? &censusNow : nullptr;
+    const bool haveCensus = census && census->modelled();
+    const int perPlayerFields = haveCensus ? 13 : 7;
+    // Thirteen fields per player would be written when a data root is present -- seven from
+    // the wire and the six census counters -- plus a new archive element per player. Nothing
+    // is committed by default, and the reason is now exactly two fields wide: savings and the
+    // income derived from it are downstream of P01/P02, which are blocked on the per-system
+    // money output, so every element written carries two confidently-wrong words. The record
+    // is one struct on the wire, so there is no way to archive the eleven right fields and
+    // leave those two out -- the format has no hole. `--commit-blocked` writes them anyway,
+    // so the claim that committing makes things worse stays a measurement, not an argument.
     int archived = 0;
     for (std::size_t i = 0; i < game.sim.players.size(); ++i) {
         if (i >= game.sim.turnstats.players.size()) break;
         auto& hist = game.sim.turnstats.players[i].hist;
         if (FindArchivedRecord(hist, game.sim.frame)) continue;  // the key is the turn
-        if (!opt.commitBlocked) {
+        if (!commits) {
             ++archived;
             continue;
         }
         // The record being archived is this turn's, and this turn ran the spine, so the
         // alliance mask is the one phase S04 rebuilt rather than a zero from the load path.
         const TurnRecord built = BuildTurnRecord(game.sim.players[i].player, game.sim.systems,
-                                                 game.sim.frame, i, /*spineRan=*/true);
+                                                 game.sim.frame, i, /*spineRan=*/true,
+                                                 /*danglingOwnedSystems=*/nullptr, census);
         mars::stream::shapes::PlayerTurnStats s;
         s.trn = built.turn;
         // The mask the spine's phase 4 rebuilt earlier in this same turn, not a value
@@ -336,39 +363,81 @@ void RunFinalizeTurnRecords(SaveGame& game, const TurnOptions& opt, PhaseRecord&
         s.sav = built.savings;
         s.inc = built.income;
         s.tch = built.completedTech;
-        // The census is three hull classes wide whether or not the player owns a ship; the
+        // The census is three hull classes wide whether or not the player owns a ship, so the
-        // counts are the unmodelled part, the shape is not.
+        // three groups are written either way; with a data root the ship and platform totals
+        // are filled from the fleet walk, without one they stay zero and say so. The four
+        // other words of each group -- losses and kills -- have no model behind them.
         for (std::int32_t c = 0; c < 3; ++c) {
             mars::stream::shapes::ClassStats cs;
             cs.cls = c;
+            if (built.censusModelled) {
+                cs.shpt = built.ships[static_cast<std::size_t>(c)];
+                cs.satt = built.platforms[static_cast<std::size_t>(c)];
+            }
             s.classes.push_back(cs);
         }
         hist.stats.push_back(s);
         ++archived;
-        rec.leafWrites += 7;
+        rec.leafWrites += perPlayerFields;
     }
     rec.committed = rec.leafWrites > 0;
-    if (!opt.commitBlocked) rec.wouldWrite = archived * 7;
+    if (!commits) rec.wouldWrite = archived * perPlayerFields;
-    rec.notes.push_back(fmt("%s %d record(s) for turn %d; 7 modelled field(s) each",
+    rec.notes.push_back(fmt("%s %d record(s) for turn %d; %d modelled field(s) each",
-                            opt.commitBlocked ? "ARCHIVED" : "would archive", archived,
+                            commits ? "ARCHIVED" : "would archive", archived, game.sim.frame,
-                            game.sim.frame));
+                            perPlayerFields));
     if (audit.playersChecked)
         rec.notes.push_back(fmt("SELF-CHECK on the input turn, where the save carries the "
-                                "answer: %d field(s) over %d player(s), %d mismatch(es)%s",
+                                "answer: %d field(s) over %d player(s), %d mismatch(es)%s "
+                                "(census %s the comparison)",
                                 audit.fieldsCompared, audit.playersChecked, audit.mismatches,
                                 audit.dangling ? " (owned-system ids missing from the table!)"
-                                               : ""));
+                                               : "",
+                                audit.censusModelled ? "IN" : "not in"));
     for (const auto& m : audit.firstMismatches) rec.notes.push_back(m);
     if (audit.missingArchive)
         rec.notes.push_back(fmt("%d player(s) carry no archive element for the input turn",
                                 audit.missingArchive));
+    if (haveCensus) {
+        rec.notes.push_back(fmt("ship census MODELLED from the section catalog: %d design(s) "
+                                "classified, %d unclassifiable, %d ship(s) whose design is "
+                                "unknown, %d fleet(s) owned by no player in the vector",
+                                census->designsSeen, census->designsUnclassified,
+                                census->shipsWithoutDesign, census->fleetsWithoutOwner));
+        rec.notes.push_back("census COVERAGE (rule 6): only 32 of the 480 census leaves the "
+                            "11-save corpus archives are nonzero anywhere -- per leaf "
+                            "(cls0 shpt/satt, cls1 shpt/satt, cls2 shpt/satt) = 18/3, 0/0, "
+                            "11/0. cls1 entirely (cruisers, ships and platforms) and satt for "
+                            "cls2 have NEVER been observed nonzero: those three counters are "
+                            "unexercised hypotheses, not verified. A zero leaf agrees for free");
+        rec.notes.push_back("census is of the fleet list AS IT STANDS, which is the input "
+                            "save's: no phase we run creates a ship. MEASURED on both corpus "
+                            "pairs -- the archived count is exactly one destroyer higher than "
+                            "ours for the one player whose build queue completes that turn, "
+                            "while the self-check on the input turn is exact. So this leaf is "
+                            "short by the turn's construction, not wrong about classification");
+        rec.notes.push_back("census HYPOTHESIS: hull size is an assignment in slot order, and "
+                            "the slots are visited in the original's in-memory order (mission, "
+                            "command, engine), not the wire's. Design rule A6 makes every "
+                            "shipped design class-homogeneous, so no save in the corpus can "
+                            "tell the two orders apart");
+    } else {
+        rec.notes.push_back("NOT modelled: the per-hull-class ship census -- it needs each "
+                            "design's hull size and defence-platform flag, neither of which is "
+                            "on the wire; both come from the section catalog. Pass --data DIR "
+                            "(or set SOTS_DATA_DIR) and the six counters are filled and "
+                            "self-checked; without it they are written as zeros with nothing "
+                            "behind them");
+    }
     std::size_t nUnmodelled = 0;
     const char* const* un = TurnRecord::Unmodelled(nUnmodelled);
     for (std::size_t i = 0; i < nUnmodelled; ++i) rec.notes.push_back(fmt("NOT modelled: %s", un[i]));
-    rec.notes.push_back("and the two fields the model does hold for the NEW turn -- savings and "
+    rec.notes.push_back("BLOCKED ON TWO NAMED THINGS, neither of them in this phase: savings "
-                        "the income derived from it -- come from a blocked phase, so committing "
+                        "and the income derived from it come from P01/P02, blocked on the "
-                        "this would replace one container-shaped divergence per player with "
+                        "per-system money output; and one ship count is short by the ships the "
-                        "several wrong leaves");
+                        "turn builds, because no phase we run creates one. The archived record "
+                        "is one struct on the wire, so those words cannot be left out while the "
+                        "rest is written -- committing is all-or-nothing at the record, and "
+                        "there is no field-granular knob that could change that");
 }
 
 }  // namespace
@@ -416,9 +485,12 @@ TurnResult RunStrategicTurn(SaveGame& game, const TurnOptions& opt) {
     TurnResult r;
 
     // Taken before any phase runs: the turn-record model is checked against the record the
-    // INPUT save already carries for its own turn. None of the six fields it reads is written
+    // INPUT save already carries for its own turn. None of the fields it reads is written by
-    // by a phase below, but the check is taken first so that stays true by construction.
+    // a phase below, but the check is taken first so that stays true by construction.
-    const TurnRecordAudit recordAudit = AuditTurnRecordsAgainstSave(game);
+    ShipCensusIndex inputCensus;
+    if (opt.catalog) inputCensus = ShipCensusIndex(game, *opt.catalog);
+    const TurnRecordAudit recordAudit =
+        AuditTurnRecordsAgainstSave(game, opt.catalog ? &inputCensus : nullptr);
 
     mars::rng::MT19937 gen(1u);
     r.rngLoaded = LoadGenerator(game, gen);
@@ -555,11 +627,12 @@ TurnResult RunStrategicTurn(SaveGame& game, const TurnOptions& opt) {
                     if (e.player.npc) continue;
                     if (e.player.status == 1) continue;
                     ++n;
-                    if (opt.commitBlocked) e.player.status = 1;
+                    if (opt.CommitBlocked("S31")) e.player.status = 1;
                 }
+                const bool s31Commits = opt.CommitBlocked("S31");
                 rec.invocations = static_cast<int>(game.sim.players.size());
-                rec.leafWrites = opt.commitBlocked ? n : 0;
+                rec.leafWrites = s31Commits ? n : 0;
-                rec.wouldWrite = opt.commitBlocked ? 0 : n;
+                rec.wouldWrite = s31Commits ? 0 : n;
                 rec.committed = rec.leafWrites > 0;
                 rec.notes.push_back(fmt("%d player status word(s) would be restored to 1", n));
                 rec.notes.push_back("MEASURED: the phase writes 1, the post-turn file carries "

src/app/turn.h

@@ -10,6 +10,7 @@
 #include <vector>
 
 #include "app/phase_catalog.h"
+#include "game/data/catalog.h"
 #include "mars/rng/mt19937.h"
 #include "mars/stream/shapes.h"
 
@@ -20,6 +21,16 @@ struct TurnOptions {
     // a blocked phase writing a confidently-wrong value is worse than a phase that does not
     // write at all, because it replaces a leaf that may currently agree by construction.
     bool commitBlocked = false;
+    // Which blocked phases that switch applies to, by phase id ("P02", "S31", "T36").
+    //
+    // `commitBlocked` alone is all-or-nothing, and all-or-nothing hides which phase paid for
+    // which leaf: a run that commits three blocked phases reports one closed count and one
+    // regressed count for all three together, which is the netting the campaign does not do.
+    // These two lists narrow it. `commitOnly` non-empty means only those ids commit;
+    // `commitExcept` names ids that never do. A phase must pass both filters.
+    std::vector<std::string> commitOnly;
+    std::vector<std::string> commitExcept;
+    bool CommitBlocked(const char* phaseId) const;
     // Write the advanced generator state back into the save. Off by default: the turn
     // consumes roughly eighteen to twenty words that nothing here models, so an advanced
     // state is wrong in a different way than an untouched one, and an untouched one at
@@ -27,6 +38,12 @@ struct TurnOptions {
     bool commitRng = false;
     // Tuning constants were loaded, so formulas that read them may run.
     bool haveTuning = false;
+    // The game's data root, when the operator supplied one. The standalone reads no game data
+    // of its own and needs none for any other phase; the tail's turn record is the one place
+    // where a save is not enough, because a design's hull size and its defence-platform flag
+    // are recomputed from the section catalog and never written down. Null means the ship
+    // census is not modelled, and the phase says so rather than writing six zeros.
+    const ::game::data::Catalog* catalog = nullptr;
 };
 
 // One line of the run log.

src/app/turn_record.cpp

@@ -2,6 +2,7 @@
 
 #include <cstdio>
 #include <limits>
+#include <map>
 
 #include "app/alliance.h"
 
@@ -13,9 +14,47 @@ constexpr const char* kUnmodelled[] = {
     "battles fought -- written by the tail's battle-tally phase, which is a stub",
     "systems acquired / lost this turn -- the two counted lists are never non-empty in the "
     "corpus, so their element meaning is unobserved",
-    "the per-hull-class ship census -- needs each design's hull size and its class flag, "
+    "the four per-class ship LOSS and KILL counters (shpl/shpk/satl/satk) -- zero in every "
-    "neither of which is on the wire; both come from the game data",
+    "save in the corpus, so nothing has ever exercised them and no model stands behind the "
+    "zeros this phase writes",
 };
+
+// The order the original's stats pass visits a design's slots in, expressed as indices into
+// the WIRE's section array. The wire writes command, mission, engine; the in-memory array the
+// stats pass walks holds mission, command, engine. Hull size is an ASSIGNMENT -- the last
+// resolved section wins -- so the order is load-bearing in principle. It is not observable in
+// this corpus: design rule A6 makes every shipped design class-homogeneous, so first-wins and
+// last-wins agree on all 503 design records. This is therefore a hypothesis carried from the
+// original's slot order, not a fact the saves can confirm.
+constexpr int kWireSlotVisitOrder[3] = {1, 0, 2};
+
+// One save `Des` record, classified. The record names each section as (species index, id in
+// that race's manifest), which is what the catalog's id lookup takes.
+game::design::HullClass ClassifySavedDesign(const game::data::Catalog& cat,
+                                            const mars::stream::shapes::Design& d) {
+    std::vector<const game::data::ShipSectionDef*> found;
+    int unresolved = 0;
+    for (int wire : kWireSlotVisitOrder) {
+        if (static_cast<std::size_t>(wire) >= d.sections.size()) continue;
+        const auto& s = d.sections[static_cast<std::size_t>(wire)];
+        const int species = s.sec.a, id = s.sec.b;
+        if (species == 0 && id == 0) continue;  // an empty slot, not a bad reference
+        if (species < 0 || species >= game::data::kSpeciesCount) {
+            ++unresolved;
+            continue;
+        }
+        const game::data::ShipSectionDef* def = cat.section_by_id(
+            game::data::species_name(static_cast<game::data::Species>(species)), id);
+        if (!def) {
+            ++unresolved;
+            continue;
+        }
+        found.push_back(def);
+    }
+    game::design::HullClass h = game::design::classify_sections(found);
+    h.unresolved_sections = unresolved;
+    return h;
+}
 }  // namespace
 
 const char* const* TurnRecord::Unmodelled(std::size_t& count) {
@@ -23,10 +62,58 @@ const char* const* TurnRecord::Unmodelled(std::size_t& count) {
     return kUnmodelled;
 }
 
+ShipCensusIndex::ShipCensusIndex(const mars::stream::shapes::SaveGame& game,
+                                 const game::data::Catalog& cat) {
+    modelled_ = true;
+    byPlayer_.resize(game.sim.players.size());
+
+    // Design ids are unique across the whole game -- the legacy (rider) designs share the id
+    // space -- so one map over every player's lists serves every fleet.
+    std::map<std::int32_t, game::design::HullClass> byId;
+    for (const auto& pe : game.sim.players) {
+        for (const auto* list : {&pe.player.designs, &pe.player.legacyDesigns}) {
+            for (const auto& de : *list) {
+                const game::design::HullClass h = ClassifySavedDesign(cat, de.des);
+                ++designsSeen;
+                if (!h.ok()) ++designsUnclassified;
+                byId[de.desID] = h;
+            }
+        }
+    }
+
+    // A fleet names its owner by the player's OBJECT id, not by its position in the player
+    // vector; the archive is keyed by the position. The two are different numbers.
+    std::map<std::int32_t, std::size_t> playerIndex;
+    for (std::size_t i = 0; i < game.sim.players.size(); ++i)
+        playerIndex[game.sim.players[i].playerID] = i;
+
+    for (const auto& fe : game.sim.fleets) {
+        auto owner = playerIndex.find(fe.flt.pid);
+        if (owner == playerIndex.end()) {
+            ++fleetsWithoutOwner;
+            continue;
+        }
+        game::design::ShipCensus& c = byPlayer_[owner->second];
+        for (const auto& se : fe.flt.ships) {
+            auto d = byId.find(se.ship.desID);
+            if (d == byId.end() || !d->second.ok()) {
+                ++shipsWithoutDesign;
+                continue;
+            }
+            c.add(d->second);
+        }
+    }
+}
+
+const game::design::ShipCensus* ShipCensusIndex::ForPlayer(std::size_t vectorIndex) const {
+    if (!modelled_ || vectorIndex >= byPlayer_.size()) return nullptr;
+    return &byPlayer_[vectorIndex];
+}
+
 TurnRecord BuildTurnRecord(const mars::stream::shapes::Player& p,
                            const std::vector<mars::stream::shapes::SysEntry>& systems,
                            std::int32_t frame, std::size_t vectorIndex, bool spineRan,
-                           int* danglingOwnedSystems) {
+                           int* danglingOwnedSystems, const ShipCensusIndex* census) {
     TurnRecord r;
     r.turn = frame;
     // Written by the spine's fourth phase, which the load path does not run.
@@ -59,6 +146,17 @@ TurnRecord BuildTurnRecord(const mars::stream::shapes::Player& p,
         r.population += static_cast<std::int64_t>(sys->pop) + static_cast<std::int64_t>(sys->pbon);
     }
     if (danglingOwnedSystems) *danglingOwnedSystems = dangling;
+
+    // The census is not derived from the player at all -- it is a walk of the fleets the
+    // player owns, and the classification behind it needs the section catalog. It arrives
+    // prebuilt or not at all.
+    if (census) {
+        if (const game::design::ShipCensus* c = census->ForPlayer(vectorIndex)) {
+            r.censusModelled = true;
+            r.ships = c->ships;
+            r.platforms = c->platforms;
+        }
+    }
     return r;
 }
 
@@ -94,6 +192,18 @@ TurnRecordDiff CompareTurnRecord(const TurnRecord& built,
     cmp("inc", built.income, stored.inc);
     cmp("tch", built.completedTech, stored.tch);
     cmp("almem", built.allianceMask, stored.almem);
+    // The six census counters, and only when a catalog produced them: comparing six
+    // unmodelled zeros against a stored record would score whatever the corpus happens to
+    // hold rather than anything this code knows.
+    if (built.censusModelled) {
+        for (std::size_t k = 0; k < stored.classes.size() && k < 3; ++k) {
+            char field[24];
+            std::snprintf(field, sizeof field, "cls%zu.shpt", k);
+            cmp(field, built.ships[k], stored.classes[k].shpt);
+            std::snprintf(field, sizeof field, "cls%zu.satt", k);
+            cmp(field, built.platforms[k], stored.classes[k].satt);
+        }
+    }
     return d;
 }
 

src/app/turn_record.h

@@ -7,18 +7,59 @@
 // a running game: build the record from a save's own state and compare it with the element the
 // save already holds for that turn.
 //
-// Six fields are recoverable from the wire and are reproduced here. Five are not, and each is
+// Seven fields are recoverable from the wire and are reproduced here. Six more -- the
-// named with the reason -- they belong to phases or inputs the standalone does not hold.
+// per-hull-class ship census -- are recoverable only with the game's section catalog, because
+// neither the hull size nor the defence-platform flag is anywhere on the wire; they are filled
+// when the operator supplies a data root and are reported as unmodelled when they are not.
+// The rest are not recoverable at all, and each is named with the reason -- they belong to
+// phases or inputs the standalone does not hold.
 #pragma once
 
+#include <array>
 #include <cstdint>
 #include <string>
 #include <vector>
 
+#include "game/data/catalog.h"
+#include "game/design/hull.h"
 #include "mars/stream/shapes.h"
 
 namespace sots::app {
 
+// The per-player ship census, built once for the whole simulation.
+//
+// The counts cannot come from a save alone: a ship names a design, and a design's hull size
+// and defence-platform flag are recomputed from the section catalog whenever it changes and
+// are never written down. So this needs a data root, and says so when it does not have one.
+//
+// Built once rather than per player because design ids are unique across the whole game and
+// the fleet list is global: classifying every design once and then walking the fleets is the
+// same shape the original uses, and it keeps the catalog lookups off the per-player path.
+class ShipCensusIndex {
+  public:
+    // Without a catalog: an index that reports itself unmodelled and counts nothing.
+    ShipCensusIndex() = default;
+    ShipCensusIndex(const mars::stream::shapes::SaveGame& game, const game::data::Catalog& cat);
+
+    bool modelled() const { return modelled_; }
+    // Null when the census is unmodelled or the index is out of range.
+    const game::design::ShipCensus* ForPlayer(std::size_t vectorIndex) const;
+
+    int designsSeen = 0;
+    // A design none of whose sections resolve against the catalog. NOT counted as class 0:
+    // an unclassifiable design is a gap, and a silent class-0 count would hide it.
+    int designsUnclassified = 0;
+    // A ship naming a design id no player's design list carries, or an unclassifiable one.
+    int shipsWithoutDesign = 0;
+    // Fleets whose owner id names no player in the player vector -- the NPC pools. Skipped,
+    // and counted so "skipped" is a measurement rather than an omission.
+    int fleetsWithoutOwner = 0;
+
+  private:
+    bool modelled_ = false;
+    std::vector<game::design::ShipCensus> byPlayer_;
+};
+
 // The part of a turn record this model can produce.
 struct TurnRecord {
     std::int32_t turn = 0;          // the frame counter
@@ -29,6 +70,13 @@ struct TurnRecord {
     std::int32_t completedTech = 0; // tech-tree entries in the completed state
     std::int32_t allianceMask = 0;  // the shared-vision mask the spine's phase 4 rebuilds
 
+    // The census, by hull size 0/1/2. Filled only when `censusModelled`; six zeros otherwise,
+    // which is exactly the value a wrong model would produce, hence the flag rather than a
+    // sentinel.
+    bool censusModelled = false;
+    std::array<int, 3> ships{};
+    std::array<int, 3> platforms{};
+
     // Fields the archive element also carries that this model does NOT fill, kept as a
     // published list rather than as silence. Each is blocked on something named.
     static const char* const* Unmodelled(std::size_t& count);
@@ -49,12 +97,16 @@ constexpr std::int32_t kTechStateCompleted = 4;
 // alliance mask is written only by the spine. So a record archived by the load path carries
 // a zero mask, and that is a positive prediction of this model, not an exclusion: the
 // earliest turn every save carries has `almem == 0` on every player, in all eleven saves.
+//
+// `census` may be null, and is null whenever no data root was supplied: the six census
+// fields then stay zero and `censusModelled` stays false.
 TurnRecord BuildTurnRecord(const mars::stream::shapes::Player& p,
                            const std::vector<mars::stream::shapes::SysEntry>& systems,
                            std::int32_t frame, std::size_t vectorIndex, bool spineRan,
                            // set when an owned-system id is not present in the table, which
                            // would silently drop a term from the population sum
-                           int* danglingOwnedSystems = nullptr);
+                           int* danglingOwnedSystems = nullptr,
+                           const ShipCensusIndex* census = nullptr);
 
 // The earliest turn the archive carries for a player. The record for that turn was written
 // by the new-game / load path, not by a turn.

tests/app/test_turn_record.cpp

@@ -10,7 +10,10 @@
 // means "compared against the live game", and nothing here is that.
 //
 // Reads $SOTS_SAVES_DIR at run time and skips cleanly when it is unset. No .sav enters this
-// repo.
+// repo. $SOTS_DATA_DIR is read the same way and adds the six ship-census fields to the
+// comparison; without it the census is not modelled and the other seven fields are compared
+// alone. No game data enters this repo either.
+#include <array>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
@@ -21,6 +24,7 @@
 
 #include "app/alliance.h"
 #include "app/turn_record.h"
+#include "game/data/catalog.h"
 #include "mars/stream/save.h"
 
 static int failures = 0;
@@ -55,7 +59,28 @@ int main() {
         return 0;
     }
 
+    // The section catalog, when the operator has one. It is what turns the six census fields
+    // from unmodelled into compared; the count of compared fields below is asserted against
+    // this flag so a silently-empty catalog cannot pass as a green run over seven fields.
+    game::data::Catalog cat;
+    bool haveCatalog = false;
+    if (const char* dataDir = std::getenv("SOTS_DATA_DIR")) {
+        if (*dataDir) {
+            cat = game::data::load_catalog(dataDir);
+            haveCatalog = !cat.sections.empty();
+            if (!haveCatalog)
+                std::printf("app_test_turn_record: %s holds no ship sections; the census is "
+                            "not compared\n", dataDir);
+        }
+    }
+
     int files = 0, players = 0, fields = 0, dangling = 0, noArchive = 0;
+    // Census coverage, reported as loudly as the verdict (earned rule 15) and as a hypothesis
+    // where nothing exercises it (rule 6): a counter that is zero in every archived record
+    // agrees for free and is not evidence of anything.
+    int censusLeaves = 0, censusNonzero = 0;
+    std::array<int, 6> nonzeroByLeaf{};  // cls0 shpt/satt, cls1 shpt/satt, cls2 shpt/satt
+    int designsSeen = 0, designsUnclassified = 0, shipsWithoutDesign = 0, fleetsWithoutOwner = 0;
     // Reported rather than assumed: how much of the alliance rule the corpus actually
     // exercises. A green run over records that are all `alid == -1` would test the self bit
     // and nothing else, and would look identical to a green run that tested everything.
@@ -74,6 +99,14 @@ int main() {
         }
         ++files;
         const auto& sim = doc.game.sim;
+        sots::app::ShipCensusIndex census;
+        if (haveCatalog) {
+            census = sots::app::ShipCensusIndex(doc.game, cat);
+            designsSeen += census.designsSeen;
+            designsUnclassified += census.designsUnclassified;
+            shipsWithoutDesign += census.shipsWithoutDesign;
+            fleetsWithoutOwner += census.fleetsWithoutOwner;
+        }
         int filePlayers = 0, fileFields = 0, fileBad = 0;
         for (std::size_t i = 0; i < sim.players.size(); ++i) {
             if (i >= sim.turnstats.players.size()) break;
@@ -90,9 +123,24 @@ int main() {
             // positive claim the corpus checks on 8 records, not a field skipped.
             const bool spineRan =
                 sim.frame > sots::app::EarliestArchivedTurn(sim.turnstats.players[i].hist);
-            const sots::app::TurnRecord built = sots::app::BuildTurnRecord(
+            const sots::app::TurnRecord built =
-                sim.players[i].player, sim.systems, sim.frame, i, spineRan, &dang);
+                sots::app::BuildTurnRecord(sim.players[i].player, sim.systems, sim.frame, i,
+                                           spineRan, &dang, haveCatalog ? &census : nullptr);
             dangling += dang;
+            CHECK(built.censusModelled == haveCatalog);
+            if (built.censusModelled) {
+                for (std::size_t k = 0; k < stored->classes.size() && k < 3; ++k) {
+                    censusLeaves += 2;
+                    if (stored->classes[k].shpt != 0) {
+                        ++censusNonzero;
+                        ++nonzeroByLeaf[k * 2];
+                    }
+                    if (stored->classes[k].satt != 0) {
+                        ++censusNonzero;
+                        ++nonzeroByLeaf[k * 2 + 1];
+                    }
+                }
+            }
             // An owned-system id the save's table does not carry would drop a term from the
             // population sum without any other symptom, so it is a failure, not a note.
             CHECK(dang == 0);
@@ -118,7 +166,11 @@ int main() {
     // failure mode this campaign has paid for twice.
     CHECK(files > 0);
     CHECK(players > 0);
-    CHECK(fields == players * 7);
+    CHECK(fields == players * (haveCatalog ? 13 : 7));
+    // A ship whose design no player's list carries, or a design nothing in the catalog
+    // resolves, would silently drop out of the census rather than diverge. Both are failures.
+    CHECK(designsUnclassified == 0);
+    CHECK(shipsWithoutDesign == 0);
 
     std::printf("app_test_turn_record: %d save(s), %d player-record(s), %d field(s) compared, "
                 "%d player(s) with no archive element, %d dangling owned-system id(s), "
@@ -129,6 +181,26 @@ int main() {
                 "the load path and are predicted to be zero; %d record(s) have a vector "
                 "position that differs from the player index field\n",
                 alliedRecords, players, loadWrittenRecords, indexDiffers);
+    if (!haveCatalog) {
+        std::printf("  ship census NOT compared: $SOTS_DATA_DIR unset or empty. The six "
+                    "counters need each design's hull size and defence-platform flag, and "
+                    "neither is on the wire.\n");
+    } else {
+        std::printf("  ship census: %d leaf/leaves compared, %d NONZERO in the archive; per "
+                    "leaf (cls0 shpt/satt, cls1 shpt/satt, cls2 shpt/satt) = %d/%d %d/%d %d/%d "
+                    "over %d design(s), %d fleet(s) owned by no player in the vector\n",
+                    censusLeaves, censusNonzero, nonzeroByLeaf[0], nonzeroByLeaf[1],
+                    nonzeroByLeaf[2], nonzeroByLeaf[3], nonzeroByLeaf[4], nonzeroByLeaf[5],
+                    designsSeen, fleetsWithoutOwner);
+        std::printf("  a zero leaf agrees for free: any counter with no nonzero observation "
+                    "above is UNEXERCISED, not verified.\n");
+        std::printf("  NOT SEPARATED by this corpus: hull size is an assignment in slot order, "
+                    "so the original's in-memory visit order (mission, command, engine) and "
+                    "the wire's (command, mission, engine) would differ only on a design whose "
+                    "sections disagree on class. Design rule A6 forbids one, so the memory "
+                    "order used here is carried from the instruction stream, not confirmed by "
+                    "these bytes.\n");
+    }
     if (indexDiffers == 0)
         std::printf("  NOT SEPARATED by this corpus: every player's vector position equals its "
                     "index field, so no comparison here can tell `1 << position` from "