sots-engine/tests/app/test_turn_record.cpp

// The turn-record model, checked against the record the game itself wrote.
//
// The last phase of the post-combat tail archives a per-player summary keyed by turn, and the
// archive is on the wire: every save carries an element for its own frame. So the model can be
// checked with no running game and no VM -- build the record from the save's own state and
// compare it with the element the save already holds.
//
// This is the strongest check available to a lane that holds no game: it compares against
// bytes the original produced. It is NOT the same as `verified` in the phase catalog, which
// means "compared against the live game", and nothing here is that.
//
// Reads $SOTS_SAVES_DIR at run time and skips cleanly when it is unset. No .sav enters this
// repo. $SOTS_DATA_DIR is read the same way and adds the six ship-census fields to the
// comparison; without it the census is not modelled and the other seven fields are compared
// alone. No game data enters this repo either.
#include <array>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <string>
#include <vector>

#include <dirent.h>

#include "app/alliance.h"
#include "app/turn_record.h"
#include "game/data/catalog.h"
#include "mars/stream/save.h"

static int failures = 0;
#define CHECK(c)                                                     \
    do {                                                             \
        if (!(c)) {                                                  \
            std::printf("FAIL %s:%d  %s\n", __FILE__, __LINE__, #c); \
            ++failures;                                              \
        }                                                            \
    } while (0)

int main() {
    const char* dir = std::getenv("SOTS_SAVES_DIR");
    if (!dir || !*dir) {
        std::printf("app_test_turn_record: SOTS_SAVES_DIR unset, skipped\n");
        return 0;
    }
    DIR* d = opendir(dir);
    if (!d) {
        std::fprintf(stderr, "app_test_turn_record: cannot open %s\n", dir);
        return 1;
    }
    std::vector<std::string> saves;
    while (struct dirent* e = readdir(d)) {
        const std::string n = e->d_name;
        if (n.size() > 4 && n.compare(n.size() - 4, 4, ".sav") == 0)
            saves.push_back(std::string(dir) + "/" + n);
    }
    closedir(d);
    if (saves.empty()) {
        std::printf("app_test_turn_record: no .sav in %s, skipped\n", dir);
        return 0;
    }

    // The section catalog, when the operator has one. It is what turns the six census fields
    // from unmodelled into compared; the count of compared fields below is asserted against
    // this flag so a silently-empty catalog cannot pass as a green run over seven fields.
    game::data::Catalog cat;
    bool haveCatalog = false;
    if (const char* dataDir = std::getenv("SOTS_DATA_DIR")) {
        if (*dataDir) {
            cat = game::data::load_catalog(dataDir);
            haveCatalog = !cat.sections.empty();
            if (!haveCatalog)
                std::printf("app_test_turn_record: %s holds no ship sections; the census is "
                            "not compared\n", dataDir);
        }
    }

    int files = 0, players = 0, fields = 0, dangling = 0, noArchive = 0;
    // Census coverage, reported as loudly as the verdict (earned rule 15) and as a hypothesis
    // where nothing exercises it (rule 6): a counter that is zero in every archived record
    // agrees for free and is not evidence of anything.
    int censusLeaves = 0, censusNonzero = 0;
    std::array<int, 6> nonzeroByLeaf{};  // cls0 shpt/satt, cls1 shpt/satt, cls2 shpt/satt
    int designsSeen = 0, designsUnclassified = 0, shipsWithoutDesign = 0, fleetsWithoutOwner = 0;
    // Reported rather than assumed: how much of the alliance rule the corpus actually
    // exercises. A green run over records that are all `alid == -1` would test the self bit
    // and nothing else, and would look identical to a green run that tested everything.
    int alliedRecords = 0, loadWrittenRecords = 0, indexDiffers = 0;
    for (const std::string& path : saves) {
        mars::stream::SaveDocument doc;
        try {
            doc = mars::stream::read_save_file(path);
        } catch (const std::exception& ex) {
            std::printf("  %s: unreadable (%s), skipped\n", path.c_str(), ex.what());
            continue;
        }
        if (doc.count(mars::stream::Issue::Error)) {
            std::printf("  %s: parse errors, skipped\n", path.c_str());
            continue;
        }
        ++files;
        const auto& sim = doc.game.sim;
        sots::app::ShipCensusIndex census;
        if (haveCatalog) {
            census = sots::app::ShipCensusIndex(doc.game, cat);
            designsSeen += census.designsSeen;
            designsUnclassified += census.designsUnclassified;
            shipsWithoutDesign += census.shipsWithoutDesign;
            fleetsWithoutOwner += census.fleetsWithoutOwner;
        }
        int filePlayers = 0, fileFields = 0, fileBad = 0;
        for (std::size_t i = 0; i < sim.players.size(); ++i) {
            if (i >= sim.turnstats.players.size()) break;
            const auto* stored =
                sots::app::FindArchivedRecord(sim.turnstats.players[i].hist, sim.frame);
            if (!stored) {
                ++noArchive;
                continue;
            }
            int dang = 0;
            // The archiving phase runs at the end of a turn AND on load, and only the
            // end-of-turn path has a spine behind it to have written the alliance mask. So
            // the earliest turn the archive carries is predicted to have a zero mask -- a
            // positive claim the corpus checks on 8 records, not a field skipped.
            const bool spineRan =
                sim.frame > sots::app::EarliestArchivedTurn(sim.turnstats.players[i].hist);
            const sots::app::TurnRecord built =
                sots::app::BuildTurnRecord(sim.players[i].player, sim.systems, sim.frame, i,
                                           spineRan, &dang, haveCatalog ? &census : nullptr);
            dangling += dang;
            CHECK(built.censusModelled == haveCatalog);
            if (built.censusModelled) {
                for (std::size_t k = 0; k < stored->classes.size() && k < 3; ++k) {
                    censusLeaves += 2;
                    if (stored->classes[k].shpt != 0) {
                        ++censusNonzero;
                        ++nonzeroByLeaf[k * 2];
                    }
                    if (stored->classes[k].satt != 0) {
                        ++censusNonzero;
                        ++nonzeroByLeaf[k * 2 + 1];
                    }
                }
            }
            // An owned-system id the save's table does not carry would drop a term from the
            // population sum without any other symptom, so it is a failure, not a note.
            CHECK(dang == 0);
            if (sim.players[i].player.alliances.alid != sots::app::kNoAlliance) ++alliedRecords;
            if (!spineRan) ++loadWrittenRecords;
            if (sim.players[i].player.plyrIdx != static_cast<std::int32_t>(i)) ++indexDiffers;
            const sots::app::TurnRecordDiff diff = sots::app::CompareTurnRecord(built, *stored);
            ++filePlayers;
            fileFields += diff.compared;
            for (const auto& m : diff.mismatches) {
                std::printf("FAIL %s player %zu: %s\n", path.c_str(), i, m.c_str());
                ++failures;
                ++fileBad;
            }
        }
        players += filePlayers;
        fields += fileFields;
        std::printf("  %s: turn %d, %d player-record(s), %d field(s), %d mismatch(es)\n",
                    path.c_str(), sim.frame, filePlayers, fileFields, fileBad);
    }

    // The check is worthless if it compared nothing -- a green run over zero records is the
    // failure mode this campaign has paid for twice.
    CHECK(files > 0);
    CHECK(players > 0);
    CHECK(fields == players * (haveCatalog ? 13 : 7));
    // A ship whose design no player's list carries, or a design nothing in the catalog
    // resolves, would silently drop out of the census rather than diverge. Both are failures.
    CHECK(designsUnclassified == 0);
    CHECK(shipsWithoutDesign == 0);

    std::printf("app_test_turn_record: %d save(s), %d player-record(s), %d field(s) compared, "
                "%d player(s) with no archive element, %d dangling owned-system id(s), "
                "%d failure(s)\n",
                files, players, fields, noArchive, dangling, failures);
    // Coverage of the alliance rule, stated as loudly as the verdict (earned rule 15).
    std::printf("  alliance mask: %d of %d record(s) carry an alliance id; %d were written by "
                "the load path and are predicted to be zero; %d record(s) have a vector "
                "position that differs from the player index field\n",
                alliedRecords, players, loadWrittenRecords, indexDiffers);
    if (!haveCatalog) {
        std::printf("  ship census NOT compared: $SOTS_DATA_DIR unset or empty. The six "
                    "counters need each design's hull size and defence-platform flag, and "
                    "neither is on the wire.\n");
    } else {
        std::printf("  ship census: %d leaf/leaves compared, %d NONZERO in the archive; per "
                    "leaf (cls0 shpt/satt, cls1 shpt/satt, cls2 shpt/satt) = %d/%d %d/%d %d/%d "
                    "over %d design(s), %d fleet(s) owned by no player in the vector\n",
                    censusLeaves, censusNonzero, nonzeroByLeaf[0], nonzeroByLeaf[1],
                    nonzeroByLeaf[2], nonzeroByLeaf[3], nonzeroByLeaf[4], nonzeroByLeaf[5],
                    designsSeen, fleetsWithoutOwner);
        std::printf("  a zero leaf agrees for free: any counter with no nonzero observation "
                    "above is UNEXERCISED, not verified.\n");
        std::printf("  NOT SEPARATED by this corpus: hull size is an assignment in slot order, "
                    "so the original's in-memory visit order (mission, command, engine) and "
                    "the wire's (command, mission, engine) would differ only on a design whose "
                    "sections disagree on class. Design rule A6 forbids one, so the memory "
                    "order used here is carried from the instruction stream, not confirmed by "
                    "these bytes.\n");
    }
    if (indexDiffers == 0)
        std::printf("  NOT SEPARATED by this corpus: every player's vector position equals its "
                    "index field, so no comparison here can tell `1 << position` from "
                    "`1 << index`. The instruction stream is what settles it; app_alliance "
                    "holds that reading.\n");
    return failures ? 1 : 0;
}