sots-engine/tests/mars_text/realdata_test.cpp

// Real-data facts test: reads the owner's extracted game text tree from
// $SOTS_DATA_DIR and checks a handful of known facts (counts and values
// established by the RE repo's verification). Skips cleanly when unset.
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <iterator>
#include <string>

#include "mars/text/csv.h"
#include "mars/text/flat_kv.h"
#include "mars/text/manifest.h"

using namespace mars::text;

static int g_failures = 0;
static int g_checks = 0;

#define CHECK(cond)                                                              \
    do {                                                                         \
        ++g_checks;                                                              \
        if (!(cond)) {                                                           \
            ++g_failures;                                                        \
            std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
        }                                                                        \
    } while (0)

static std::string read_file(const std::string& path) {
    std::ifstream in(path, std::ios::binary);
    if (!in) {
        std::fprintf(stderr, "FAIL cannot read %s\n", path.c_str());
        ++g_failures;
        return {};
    }
    return std::string((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
}

int main() {
    const char* root = std::getenv("SOTS_DATA_DIR");
    if (!root || !*root) {
        std::printf("realdata_test: SKIP (SOTS_DATA_DIR not set)\n");
        return 0;
    }
    const std::string base = std::string(root) + "/";

    // globals.txt: a colour and the comment-in-quotes quirk line
    {
        auto r = parse_flat_kv(read_file(base + "Data/globals.txt"));
        CHECK(r.ok());
        CHECK(r.value.size() == 364);
        CHECK(r.value.duplicates().empty());
        auto c = r.value.get_color("mars_default_color");   // case-insensitive
        CHECK(c && c->r == 255 && c->g == 177 && c->b == 39);
        CHECK(r.value.find("MARS_DEFAULT_COLOR")->key == "MARS_DEFAULT_COLOR");
        auto endgame = r.value.get_color("ENDGAME_FILL_COLOR");
        CHECK(endgame && endgame->components == 3);
    }
    // StrategyVars.txt: typed access; the file has no trailing newline, so the
    // engine drops its last pair (CIVILIAN_BURDEN_RATIO keeps the image default)
    {
        auto r = parse_flat_kv(read_file(base + "Data/Strategy/StrategyVars.txt"));
        CHECK(r.value.size() == 96);
        CHECK(r.value.get_float("SLAVES_DEATH_RATE").has_value());
        CHECK(!r.value.has("CIVILIAN_BURDEN_RATIO"));
        CHECK(r.problems.size() == 1 && r.problems[0].kind == Problem::Kind::DroppedTrailingPair);
    }
    // encounters.txt: same, HERALD_SPEECH_MAX_INVERVAL is never read
    {
        auto r = parse_flat_kv(read_file(base + "Data/encounters.txt"));
        CHECK(!r.value.has("HERALD_SPEECH_MAX_INVERVAL"));
        CHECK(r.problems.size() == 1 && r.problems[0].kind == Problem::Kind::DroppedTrailingPair);
        CHECK(r.value.duplicates().empty());
    }
    // _turrets.txt: 42 rows, 8 columns, quoted model in the last column
    {
        auto r = parse_rows(read_file(base + "Weapons/_turrets.txt"));
        CHECK(r.ok());
        CHECK(r.value.size() == 42);
        for (const Row& row : r.value) {
            CHECK(row.tokens.size() == 8);
            CHECK(row.tokens.back().quoted);
            CHECK(row.tokens[3].as_int().has_value());
            CHECK(row.tokens[5].as_float().has_value());
        }
    }
    // _weapons.txt: 123 ids, tombstones 36/58/59, no problems
    {
        auto r = parse_manifest(read_file(base + "Weapons/_weapons.txt"));
        CHECK(r.ok());
        CHECK(r.value.entries().size() == 123);
        CHECK((r.value.deleted() == std::vector<int>{36, 58, 59}));
        CHECK(r.value.id_of("CAN_AM.WEAPON").has_value());
        CHECK(r.value.name_of(1).has_value());
    }
    // Human _shipsections.txt: 145 ids; the misspelt-case DEWar entry resolves both ways
    {
        auto r = parse_manifest(read_file(base + "Species/Human/sections/_shipsections.txt"));
        CHECK(r.ok());
        CHECK(r.value.entries().size() == 145);
        CHECK(r.value.id_of("dewar.shipsection") == 98);
    }
    // Strings.csv: 5,722 raw records / 5,200 data rows, 4-column schema row,
    // exactly one multi-line cell (its line break sits just before the closing
    // quote, so cell stripping removes it), cp1252 bytes untouched
    {
        std::string text = read_file(base + "Locale/EN/Strings.csv");
        auto raw = split_csv_records(text);
        CHECK(raw.ok());
        CHECK(raw.value.size() == 5722);
        int multiline = 0;
        for (const CsvRow& row : raw.value)
            for (const std::string& c : row.cells)
                if (c.find("\r\n") != std::string::npos) ++multiline;
        CHECK(multiline == 1);

        auto r = parse_csv(text);
        CHECK(r.ok());
        CHECK(r.value.size() == 5200);
        CHECK(r.value.header && r.value.header->size() == 4 && (*r.value.header)[1] == "String");
        int highbytes = 0, embedded_newline = 0;
        for (const CsvRow& row : r.value.rows)
            for (const std::string& c : row.cells) {
                if (c.find('\n') != std::string::npos) ++embedded_newline;
                for (unsigned char ch : c)
                    if (ch >= 0x80) { ++highbytes; break; }
            }
        CHECK(embedded_newline == 0);
        CHECK(highbytes > 0);
        // the 4 keys duplicated via a trailing space collapse after stripping
        std::size_t distinct = 0;
        {
            std::vector<std::string> keys;
            for (const CsvRow& row : r.value.rows) keys.push_back(row.cells.at(0));
            std::sort(keys.begin(), keys.end());
            distinct = static_cast<std::size_t>(std::unique(keys.begin(), keys.end()) - keys.begin());
        }
        CHECK(distinct == 5196);
    }
    // AI template CSV: comment-only, but the schema row is recovered
    {
        auto r = parse_csv(read_file(base + "Data/Strategy/AI/aitechpri.csv"));
        CHECK(r.ok());
        CHECK(r.value.size() == 0);
        CHECK(r.value.header && r.value.header->size() == 7 && (*r.value.header)[0] == "tech");
    }
    // stock_diplomacy_messages.csv: every cell quoted, "# species" header, blank ",," rows dropped
    {
        auto r = parse_csv(read_file(base + "Data/Strategy/AI/stock_diplomacy_messages.csv"));
        CHECK(r.value.size() == 755);
        CHECK(r.value.header && (*r.value.header)[0] == "species");
    }

    std::printf("realdata_test: %d checks, %d failures\n", g_checks, g_failures);
    return g_failures == 0 ? 0 : 1;
}