243 lines
10 KiB
C++
243 lines
10 KiB
C++
// Oracle-backed test against the owner's real archives. Skips cleanly unless
|
|
// SOTS_GOB_DIR points at a directory holding sots.gob and sots_local_en.gob.
|
|
//
|
|
// Env:
|
|
// SOTS_GOB_DIR directory with the two archives
|
|
// SOTS_GOB_ORACLE_DIR directory with list_sots.gob.txt / list_sots_local_en.gob.txt
|
|
// (`unzip -l` output); defaults to SOTS_GOB_DIR; listing
|
|
// comparison SKIPs when a file is absent
|
|
// SOTS_GOB_FULL=1 also read + CRC-verify every entry of both archives
|
|
#include <algorithm>
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <map>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "mars/parse/blocks.h"
|
|
#include "mars/text/csv.h"
|
|
#include "mars/vfs/path.h"
|
|
#include "mars/vfs/vfs.h"
|
|
#include "mars/vfs/zip_archive.h"
|
|
|
|
namespace fs = std::filesystem;
|
|
using namespace mars::vfs;
|
|
|
|
static int g_failures = 0;
|
|
static int g_checks = 0;
|
|
|
|
#define CHECK(cond) \
|
|
do { \
|
|
++g_checks; \
|
|
if (!(cond)) { \
|
|
++g_failures; \
|
|
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
|
} \
|
|
} while (0)
|
|
|
|
namespace {
|
|
|
|
struct OracleEntry {
|
|
std::string name;
|
|
std::uint64_t size = 0;
|
|
};
|
|
|
|
// Parse `unzip -l` output: three header lines, entries, a "---------" footer.
|
|
// Names start at a fixed column (they may contain spaces).
|
|
// " 65580 2007-08-07 13:25 Avatars/AI_AV.tga"
|
|
bool load_listing(const fs::path& path, std::vector<OracleEntry>& out) {
|
|
std::ifstream in(path);
|
|
if (!in) return false;
|
|
std::string line;
|
|
int lineno = 0;
|
|
while (std::getline(in, line)) {
|
|
++lineno;
|
|
if (!line.empty() && line.back() == '\r') line.pop_back();
|
|
if (lineno <= 3) continue;
|
|
if (line.rfind("---------", 0) == 0) break;
|
|
if (line.size() < 31) continue;
|
|
OracleEntry e;
|
|
e.size = std::strtoull(line.c_str(), nullptr, 10);
|
|
e.name = line.substr(30);
|
|
out.push_back(std::move(e));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void compare_with_listing(const ZipArchive& z, const fs::path& listing) {
|
|
std::vector<OracleEntry> oracle;
|
|
if (!load_listing(listing, oracle)) {
|
|
std::printf(" listing %s: SKIP (not found)\n", listing.string().c_str());
|
|
return;
|
|
}
|
|
std::size_t dirs = 0;
|
|
for (const auto& e : oracle) if (!e.name.empty() && e.name.back() == '/') ++dirs;
|
|
std::printf(" oracle %s: %zu entries (%zu directories, %zu files)\n", listing.filename().string().c_str(),
|
|
oracle.size(), dirs, oracle.size() - dirs);
|
|
std::printf(" ours %s: %zu entries (%zu files)\n", fs::path(z.path()).filename().string().c_str(),
|
|
z.entries().size(), z.file_count());
|
|
CHECK(z.entries().size() == oracle.size());
|
|
CHECK(z.file_count() == oracle.size() - dirs);
|
|
|
|
std::map<std::string, std::uint64_t> ours;
|
|
for (const ZipEntry& e : z.entries()) ours.emplace(e.name, e.uncompressed_size);
|
|
std::size_t missing = 0, size_mismatch = 0;
|
|
for (const auto& o : oracle) {
|
|
auto it = ours.find(o.name);
|
|
if (it == ours.end()) {
|
|
if (missing < 5) std::fprintf(stderr, " not in ours: %s\n", o.name.c_str());
|
|
++missing;
|
|
} else if (it->second != o.size) {
|
|
if (size_mismatch < 5) std::fprintf(stderr, " size differs: %s\n", o.name.c_str());
|
|
++size_mismatch;
|
|
}
|
|
}
|
|
CHECK(missing == 0);
|
|
CHECK(size_mismatch == 0);
|
|
CHECK(ours.size() == oracle.size()); // no duplicate names either side
|
|
}
|
|
|
|
void full_read(const ZipArchive& z) {
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
std::uint64_t bytes = 0;
|
|
std::size_t errors = 0, files = 0;
|
|
for (const ZipEntry& e : z.entries()) {
|
|
if (e.is_directory) continue;
|
|
++files;
|
|
auto r = z.read(e);
|
|
if (!r) {
|
|
++errors;
|
|
if (errors < 5) std::fprintf(stderr, " read error: %s\n", r.error().message.c_str());
|
|
continue;
|
|
}
|
|
bytes += r->size();
|
|
}
|
|
const double secs = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
|
|
std::printf(" full read %s: %zu files, %.2f GB, %zu errors, %.1fs\n", fs::path(z.path()).filename().string().c_str(),
|
|
files, static_cast<double>(bytes) / 1e9, errors, secs);
|
|
CHECK(errors == 0);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() {
|
|
const char* dir = std::getenv("SOTS_GOB_DIR");
|
|
if (!dir || !*dir) {
|
|
std::printf("realdata_test: SKIP (SOTS_GOB_DIR not set)\n");
|
|
return 0;
|
|
}
|
|
const fs::path gobs(dir);
|
|
const char* odir = std::getenv("SOTS_GOB_ORACLE_DIR");
|
|
const fs::path oracle = (odir && *odir) ? fs::path(odir) : gobs;
|
|
const bool full = std::getenv("SOTS_GOB_FULL") && std::string(std::getenv("SOTS_GOB_FULL")) == "1";
|
|
|
|
// --- archives against the unzip -l listings ---
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
Result<ZipArchive> main_gob = ZipArchive::open((gobs / "sots.gob").string());
|
|
Result<ZipArchive> local_gob = ZipArchive::open((gobs / "sots_local_en.gob").string());
|
|
const double open_secs = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
|
|
CHECK(main_gob.ok());
|
|
CHECK(local_gob.ok());
|
|
if (!main_gob || !local_gob) {
|
|
std::fprintf(stderr, "cannot open archives: %s / %s\n", main_gob ? "ok" : main_gob.error().message.c_str(),
|
|
local_gob ? "ok" : local_gob.error().message.c_str());
|
|
return 1;
|
|
}
|
|
std::printf(" opened both archives in %.3fs\n", open_secs);
|
|
compare_with_listing(main_gob.value(), oracle / "list_sots.gob.txt");
|
|
compare_with_listing(local_gob.value(), oracle / "list_sots_local_en.gob.txt");
|
|
std::size_t deflated = 0;
|
|
for (const ZipEntry& e : main_gob->entries()) if (e.method != 0) ++deflated;
|
|
for (const ZipEntry& e : local_gob->entries()) if (e.method != 0) ++deflated;
|
|
std::printf(" entries using a compression method: %zu\n", deflated);
|
|
CHECK(deflated == 0); // the notes: every entry is stored
|
|
|
|
// --- the layered VFS ---
|
|
Vfs v;
|
|
auto id_main = v.mount_zip((gobs / "sots.gob").string());
|
|
auto id_local = v.mount_zip((gobs / "sots_local_en.gob").string());
|
|
CHECK(id_main.ok() && id_local.ok());
|
|
|
|
// MasterTechList.tech parses into 293 tech blocks, from any spelling.
|
|
{
|
|
CHECK(v.exists("TechTree/MasterTechList.tech"));
|
|
CHECK(v.exists("techtree\\MASTERTECHLIST.TECH"));
|
|
auto st = v.stat("TechTree/MasterTechList.tech");
|
|
CHECK(st && st->mount == id_main.value() && st->size == 62234 && !st->compressed);
|
|
auto text = v.read_text("TechTree/MasterTechList.tech");
|
|
CHECK(text.ok());
|
|
if (text) {
|
|
auto doc = mars::parse::parse_blocks(text.value());
|
|
CHECK(doc.ok());
|
|
if (doc) {
|
|
const std::size_t techs = doc->root.blocks("tech").size();
|
|
std::printf(" MasterTechList.tech: %zu tech blocks\n", techs);
|
|
CHECK(techs == 293);
|
|
}
|
|
}
|
|
}
|
|
// Strings.csv comes from the second archive and yields 5,722 records.
|
|
{
|
|
auto st = v.stat("Locale/EN/Strings.csv");
|
|
CHECK(st && st->mount == id_local.value() && st->size == 397352);
|
|
auto text = v.read_text("locale/en/strings.csv");
|
|
CHECK(text.ok());
|
|
if (text) {
|
|
auto raw = mars::text::split_csv_records(text.value());
|
|
auto csv = mars::text::parse_csv(text.value());
|
|
std::printf(" Strings.csv: %zu raw records, %zu data rows (%zu problems)\n", raw.value.size(),
|
|
csv.value.rows.size(), csv.problems.size());
|
|
CHECK(raw.value.size() == 5722); // the notes' "5,722 rows" (same as mars_text's own check)
|
|
CHECK(csv.value.rows.size() == 5200);
|
|
}
|
|
}
|
|
// Listing: 207 .weapon files in total (123 under Weapons/, 84 under Species/_NPC/).
|
|
{
|
|
auto ends_with_weapon = [](const std::string& n) {
|
|
return n.size() > 7 && normalize_key(n).compare(n.size() - 7, 7, ".weapon") == 0;
|
|
};
|
|
std::size_t total = 0, under_weapons = 0;
|
|
for (const auto& e : v.list()) if (ends_with_weapon(e.name)) ++total;
|
|
for (const auto& e : v.list("Weapons/")) if (ends_with_weapon(e.name)) ++under_weapons;
|
|
std::printf(" *.weapon: %zu total, %zu under Weapons/\n", total, under_weapons);
|
|
CHECK(total == 207);
|
|
CHECK(under_weapons == 123);
|
|
CHECK(v.list().size() == main_gob->file_count() + local_gob->file_count());
|
|
}
|
|
// A loose-file override wins over the archive; registration order can flip that.
|
|
{
|
|
const auto tag = std::chrono::steady_clock::now().time_since_epoch().count();
|
|
fs::path over = fs::temp_directory_path() / ("mars_vfs_real_" + std::to_string(tag));
|
|
fs::create_directories(over / "techtree");
|
|
std::ofstream(over / "techtree" / "mastertechlist.tech") << "tech { name \"OVERRIDE\" }\n";
|
|
Vfs w;
|
|
w.mount_zip((gobs / "sots.gob").string());
|
|
auto nid = w.mount_native(over.string());
|
|
CHECK(nid.ok());
|
|
auto st = w.stat("TechTree/MasterTechList.tech");
|
|
CHECK(st && st->mount == nid.value() && st->kind == MountKind::Native);
|
|
auto text = w.read_text("TechTree/MasterTechList.tech");
|
|
CHECK(text && text->rfind("tech { name \"OVERRIDE\"", 0) == 0);
|
|
|
|
Vfs r(Order::Registration);
|
|
r.mount_zip((gobs / "sots.gob").string());
|
|
r.mount_native(over.string());
|
|
auto st2 = r.stat("TechTree/MasterTechList.tech");
|
|
CHECK(st2 && st2->kind == MountKind::Zip);
|
|
std::error_code ec;
|
|
fs::remove_all(over, ec);
|
|
}
|
|
|
|
if (full) {
|
|
full_read(main_gob.value());
|
|
full_read(local_gob.value());
|
|
} else {
|
|
std::printf(" full read: SKIP (set SOTS_GOB_FULL=1)\n");
|
|
}
|
|
|
|
std::printf("realdata_test: %d checks, %d failures\n", g_checks, g_failures);
|
|
return g_failures == 0 ? 0 : 1;
|
|
}
|