diff --git a/docs/mars-parse.md b/docs/mars-parse.md new file mode 100644 index 0000000..079e7fe --- /dev/null +++ b/docs/mars-parse.md @@ -0,0 +1,222 @@ +# `mars::parse` — the engine's data-file readers + +`src/mars/parse/` reimplements the two text readers the original engine uses +for its catalogs: the recursive **brace-block** key/value format (weapons, ship +sections, the tech tree, combat missions, badge/light defs, GUI scripts, +scenario and other block-form `.txt`) and the separate **`.effect`** format. +C++17, no dependencies, no exceptions across the API, bytes passed through +untouched (the data is cp1252). + +Behaviour is specified by the proven Python readers in the RE repo +(`sots-re/verify/parsers/mars_data.py`, `effect_txt.py`) and the findings in +`findings/subsystems/data-parsers.md`. The C++ is written from that +understanding, not transliterated; the Python readers serve as the test oracle. + +``` +src/mars/parse/ + result.h Diagnostic {line, message}; Result + value.h/.cpp classify / as_int / as_double / as_bool / iequals + blocks.h/.cpp brace-block reader: parse_blocks(text, Options) -> Result + effect.h/.cpp .effect reader: parse_effect(text) -> Result + CMakeLists.txt static library `mars_parse` (include root: src/) +tests/mars_parse/ + build_and_run.sh plain g++ build; unit tests; real-data oracle test + test_*.cpp hand-written samples, one test per quirk + canon.h/.cpp canonical JSON rendering (matches oracle/dump.py) + oracle_check.cpp parses every real file, diffs against the oracle + oracle/dump.py runs the RE-repo readers, emits canonical JSON per file +``` + +## 1. Brace-block format — grammar as implemented + +``` +body := (block | pair | item)* +block := NAME '{' body '}' +pair := NAME value +item := QUOTED -- a quoted string with no key + | NAME -- a bareword directly before '}' or EOF +value := QUOTED | BAREWORD +comment := '//' ... end-of-line -- only at a token boundary +``` + +Tokens (whitespace-driven, not line-based): + +| token | rule | +|---|---| +| whitespace | space, tab, CR, LF, VT, FF | +| `{` `}` | single-character tokens; they also terminate a bareword (`weapon{` is two tokens) | +| QUOTED | `"` … `"`; no escapes at all — backslashes and `//` inside are literal; may span lines; an empty `""` is a valid value | +| BAREWORD | maximal run of anything that is not whitespace, `{`, `}` or `"` — so `abc"def"` is bareword `abc` followed by a quoted token, and `3//x` is one bareword | +| comment | `//` at the start of a token runs to end of line | + +The file's top level is itself a body: catalog files hold one or more named +blocks, scenario `.txt` files mix top-level pairs and `player{}` blocks. + +Disambiguation of a bareword `N`: look at the next token. `{` → a block named +`N`; `}` or EOF → an item; anything else → a pair `N value` (the value may be +bare or quoted). A quoted token in key position is always an item. + +### Leniency (what the shipped data needs) + +| condition | default (`Options::strict = false`) | `strict = true` | +|---|---|---| +| EOF while blocks are open | close every open block, one `Diagnostic` per open block in `Document::warnings` | error at the EOF line | +| `}` at top level | ignored, warning, parsing continues | error | +| unterminated `"` | error | error | +| `{` with no name before it | error | error | + +The two recoveries are exactly what 12 shipped shipsections require (11 never +close their outer block, `Human/CrPropaganda.shipsection` has one `}` too many). +They load in the game, so the engine performs the same repairs. Nothing in the +shipped data hits the two hard errors. + +### Node model + +```cpp +struct Entry { // one thing in a block, in file order + enum class Kind { Pair, Item, Block } kind; + std::string key; // Pair/Block: as written (case preserved) + std::string value; // Pair: value text; Item: the item text + bool quoted; // value/item was in double quotes + std::unique_ptr block; // Block only + int line; // 1-based +}; +struct Node { // a named block; the root has name "" + std::string name; + std::vector entries; + int line; + const Entry* first(key); std::vector all(key); // pairs + blocks + const std::string* first_value(key); std::vector values(key); + const Node* first_block(key); std::vector blocks(key); + std::vector items(); bool has(key); +}; +struct Document { Node root; std::vector warnings; }; +Result parse_blocks(std::string_view, Options = {}); +``` + +Design points, each tied to a data fact: + +- **Keys are matched case-insensitively** (`first`/`all`/… use `iequals`), the + original spelling is kept. The data mixes `Requires`/`requires`, + `badge`/`Badge`; identifiers in values are also case-mixed but that is the + consumer's concern (`iequals` is public for it). +- **Repeated keys are not merged**: `all("requires")` returns them in file + order. 970 `requires`, 3,721 `bank`, and the 19 banks that repeat + `turretsize` all rely on order being kept. +- **Pairs and blocks share the key namespace**: `first`/`all` see both because + 153 shipsections carry `option DRV_X` beside `option { option A option B }`. + `values()`/`blocks()` filter by kind when a consumer wants only one. +- **Values are raw text**. `50`, `.5`, `7e+8`, `TRUE`, `@WEAPON_X` and `"8"` + all stay as written, with `quoted` recording the difference. Conversion is + explicit via `value.h`: `classify()` reports Int / Float / Bool / Text using + the C-float shapes the data uses (`[+-]?d+`, `[+-]?(d+.d*|.d+|d+)([eE][+-]?d+)?`, + `true`/`false` any case); `as_int`/`as_double`/`as_bool` convert. A quoted + string is never a number to the oracle, and consumers should honour `quoted` + the same way. +- **Items**: bare quoted strings (`systemnames.txt` lists) and a lone trailing + bareword are `Kind::Item`, reachable through `items()`. + +## 2. `.effect` format — grammar as implemented + +Line-based, not brace-block. + +``` +line 1: TXT magic (surrounding whitespace ignored) + KEY value [value ...] scalar entry; values are "quoted" or bare + KEY followed (after blank/comment lines) by + BEGIN ... END a nested group +``` + +- Lines split on LF; a trailing CR is dropped (three shipped files are CRLF). +- `//` starts a comment unless inside double quotes; blank lines are skipped; + indentation is cosmetic. +- Tokens: `"…"` (may hold spaces — `NAME "New Emitter"`) or a run of + non-whitespace. A `"` that never closes on its line is just part of a bare + token. +- A line with a single token is a group key and **must** be followed by + `BEGIN`. `BEGIN` with no pending key, `END` with no open group, a group key + followed by anything but `BEGIN`, a trailing group key, and unclosed groups + at EOF are all errors. No leniency has been added: all 415 shipped files are + balanced, and there is no evidence yet of what the engine would do otherwise. +- Order is preserved and repeated keys are kept as separate entries because it + is semantic: `PARTICLEDATATYPE n` precedes the curves that belong to it and + `MODIFIER` repeats once per type. + +```cpp +struct EffectValue { std::string text; bool quoted; }; +struct EffectEntry { + std::string key; // as written (UPPERCASE in the data) + std::vector values; // scalar line: 1+ values + std::unique_ptr group; // group line + int line; + bool is_group() const; std::string_view value() const; // first value +}; +struct EffectGroup { std::vector entries; first(key); all(key); }; +struct EffectFile { EffectGroup root; }; +Result parse_effect(std::string_view); +``` + +## 3. Tests + +`tests/mars_parse/build_and_run.sh` (plain `g++ -std=c++17 -Wall -Wextra +-Wpedantic -Werror`): + +1. **Unit tests** — 39 cases with hand-written samples: nesting, same-line + blocks and glued braces, quoted values with spaces/backslashes/`//`, empty + `""`, comments at token boundaries, repeated keys in order, case-insensitive + lookup, quoted items and trailing barewords, `option` scalar+block, `@TOKEN` + verbatim, EOF-closes-blocks and stray-`}` in lenient and strict mode, + unterminated string and nameless `{` errors, CRLF and line numbers, + multi-line strings, high bytes, and every effect error path. +2. **Real-data oracle test** — runs when `SOTS_DATA_DIR` points at an + extracted `sots.gob` tree. `oracle/dump.py` runs the RE-repo readers + (`SOTS_RE_PARSERS`, default `~/sots-re/verify/parsers`; exit 3 → the C++ + side runs count-only) and writes a canonical JSON per file; `oracle_check` + parses every file with `mars_parse`, renders the same canonical form + (`canon.cpp`) and requires byte equality, plus equal lenient-warning counts. + +The canonical form is the oracle's own dict shape (keys lower-cased and +sorted, repeats → lists, items under `_items`, bare pair values typed), with +floats rendered by both sides as `"\x01" + printf("%.17g")` so formatting is +shared, and cp1252 bytes rendered one `\u00xx` per byte. A one-character +change in an oracle file is detected (verified). + +Skips cleanly with `SOTS_DATA_DIR` unset. Nothing from the game is committed; +`tests/mars_parse/build/` (which holds the oracle JSON) is git-ignored. + +### Oracle results (2026-09-07, owner's `gob-extract`) + +| kind | files | parsed | oracle match | need lenient recovery | +|---|---|---|---|---| +| shipsection | 875 | 875 | 875 | 12 | +| weapon | 207 | 207 | 207 | 0 | +| effect | 415 | 415 | 415 | — | +| tech (MasterTechList) | 1 | 1 | 1 | 0 | +| combat | 3 | 3 | 3 | 0 | +| script | 4 | 4 | 4 | 0 | +| def | 2 | 2 | 2 | 0 | +| block-form txt (Scenarios + 7 Data/Models files) | 24 | 24 | 24 | 0 | +| **total** | **1,531** | **1,531** | **1,531 (100%)** | 12 | + +The 12 strict-mode rejects are exactly the list in `data-parsers.md` +(Hiver 6, Liir 4, Morrigi 1 unclosed; Human `CrPropaganda` stray `}`), and the +per-file warning counts equal the oracle's. The remaining 64 of the Python +suite's 1,595 files are CSV / flat key-value / manifest / positional tables — +different readers, outside this module. + +## 4. Open questions + +- **Engine leniency beyond the two observed repairs.** Unterminated strings, + a nameless `{`, and malformed `.effect` nesting are errors here because no + shipped file exercises them; the original may be more forgiving. Worth a + shim-side probe before the loader is wired to user mods. +- **`//` glued to a bareword** (`3//x`) is one token, matching the oracle. + Whether the engine's tokenizer cuts a comment mid-word is unverified; the + data never depends on it. +- **Whitespace set.** The C++ uses the C `isspace` set; the Python oracle's + `\s` also covers U+00A0 and a few control characters. The data contains + neither (checked: all 1,531 files are pure ASCII), so this cannot be + observed from data. +- **Typed values.** The oracle types barewords (`50` → int) at parse time; + this module keeps text and offers converters. If a consumer needs + oracle-identical typing it should use `classify()` and honour `quoted`. diff --git a/src/mars/parse/CMakeLists.txt b/src/mars/parse/CMakeLists.txt new file mode 100644 index 0000000..02ef106 --- /dev/null +++ b/src/mars/parse/CMakeLists.txt @@ -0,0 +1,10 @@ +# mars_parse -- the engine's data-file readers (brace-block format + .effect). +# Header-only consumers include etc. via the src/ root. +add_library(mars_parse STATIC + blocks.cpp + effect.cpp + value.cpp +) +target_include_directories(mars_parse PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../..) +target_compile_features(mars_parse PUBLIC cxx_std_17) +target_compile_options(mars_parse PRIVATE -Wall -Wextra -Wpedantic) diff --git a/src/mars/parse/blocks.cpp b/src/mars/parse/blocks.cpp new file mode 100644 index 0000000..cc33e15 --- /dev/null +++ b/src/mars/parse/blocks.cpp @@ -0,0 +1,259 @@ +#include "blocks.h" + +#include + +#include "value.h" + +namespace mars::parse { + +// ---- Entry special members (Node is complete here) -------------------------- + +Entry::Entry() = default; +Entry::Entry(Entry&&) noexcept = default; +Entry& Entry::operator=(Entry&&) noexcept = default; +Entry::~Entry() = default; + +// ---- Node lookups ----------------------------------------------------------- + +const Entry* Node::first(std::string_view key) const { + for (const Entry& e : entries) + if (!e.is_item() && iequals(e.key, key)) return &e; + return nullptr; +} + +std::vector Node::all(std::string_view key) const { + std::vector out; + for (const Entry& e : entries) + if (!e.is_item() && iequals(e.key, key)) out.push_back(&e); + return out; +} + +const std::string* Node::first_value(std::string_view key) const { + for (const Entry& e : entries) + if (e.is_pair() && iequals(e.key, key)) return &e.value; + return nullptr; +} + +std::vector Node::values(std::string_view key) const { + std::vector out; + for (const Entry& e : entries) + if (e.is_pair() && iequals(e.key, key)) out.push_back(e.value); + return out; +} + +const Node* Node::first_block(std::string_view key) const { + for (const Entry& e : entries) + if (e.is_block() && iequals(e.key, key)) return e.block.get(); + return nullptr; +} + +std::vector Node::blocks(std::string_view key) const { + std::vector out; + for (const Entry& e : entries) + if (e.is_block() && iequals(e.key, key)) out.push_back(e.block.get()); + return out; +} + +std::vector Node::items() const { + std::vector out; + for (const Entry& e : entries) + if (e.is_item()) out.push_back(e.value); + return out; +} + +// ---- tokenizer -------------------------------------------------------------- + +namespace { + +enum class Tok { Open, Close, Quoted, Bare, End }; + +struct Token { + Tok kind = Tok::End; + std::string_view text; // Quoted: contents without the quotes + int line = 0; +}; + +bool is_space(unsigned char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'; +} + +class Lexer { +public: + explicit Lexer(std::string_view s) : s_(s) {} + + // Returns false (with `err` set) only for an unterminated string. + bool next(Token& out, Diagnostic& err) { + skip_trivia(); + out.line = line_; + if (pos_ >= s_.size()) { + out.kind = Tok::End; + out.text = {}; + return true; + } + char c = s_[pos_]; + if (c == '{') { ++pos_; out.kind = Tok::Open; out.text = s_.substr(pos_ - 1, 1); return true; } + if (c == '}') { ++pos_; out.kind = Tok::Close; out.text = s_.substr(pos_ - 1, 1); return true; } + if (c == '"') { + std::size_t close = s_.find('"', pos_ + 1); + if (close == std::string_view::npos) { + err = {line_, "unterminated string"}; + return false; + } + out.kind = Tok::Quoted; + out.text = s_.substr(pos_ + 1, close - pos_ - 1); + for (char q : out.text) if (q == '\n') ++line_; // strings may span lines + pos_ = close + 1; + return true; + } + std::size_t start = pos_; + while (pos_ < s_.size()) { + unsigned char u = static_cast(s_[pos_]); + if (is_space(u) || u == '{' || u == '}' || u == '"') break; + ++pos_; + } + out.kind = Tok::Bare; + out.text = s_.substr(start, pos_ - start); + return true; + } + +private: + void skip_trivia() { + while (pos_ < s_.size()) { + unsigned char c = static_cast(s_[pos_]); + if (is_space(c)) { + if (c == '\n') ++line_; + ++pos_; + } else if (c == '/' && pos_ + 1 < s_.size() && s_[pos_ + 1] == '/') { + while (pos_ < s_.size() && s_[pos_] != '\n') ++pos_; + } else { + break; + } + } + } + + std::string_view s_; + std::size_t pos_ = 0; + int line_ = 1; +}; + +// ---- parser ----------------------------------------------------------------- + +class Parser { +public: + Parser(std::string_view text, Options opt) : lex_(text), opt_(opt) {} + + Result run() { + Document doc; + if (!advance()) return err_; + if (!body(doc.root, 0)) return err_; + doc.warnings = std::move(warnings_); + return doc; + } + +private: + // Fills `cur_` with the next token. False on lexer error. + bool advance() { return lex_.next(cur_, err_); } + + // Lenient recovery point: warn, or fail under strict. + bool recover(int line, std::string message) { + if (opt_.strict) { + err_ = {line, std::move(message)}; + return false; + } + warnings_.push_back({line, std::move(message)}); + return true; + } + + bool fail(int line, std::string message) { + err_ = {line, std::move(message)}; + return false; + } + + // Parses entries into `node` until the matching '}' (depth > 0) or EOF. + // On return `cur_` is the token after the block. + bool body(Node& node, int depth) { + for (;;) { + switch (cur_.kind) { + case Tok::End: + if (depth > 0 && + !recover(cur_.line, "end of file inside block '" + node.name + + "' (depth " + std::to_string(depth) + ")")) + return false; + return true; // EOF closes every open block + + case Tok::Close: + if (depth == 0) { + if (!recover(cur_.line, "stray '}' at top level")) return false; + if (!advance()) return false; + continue; + } + return advance(); + + case Tok::Open: + return fail(cur_.line, "'{' without a block name"); + + case Tok::Quoted: { + Entry e; + e.kind = Entry::Kind::Item; + e.value = std::string(cur_.text); + e.quoted = true; + e.line = cur_.line; + node.entries.push_back(std::move(e)); + if (!advance()) return false; + continue; + } + + case Tok::Bare: { + Token name = cur_; + if (!advance()) return false; + if (cur_.kind == Tok::End || cur_.kind == Tok::Close) { + // A lone word right before the block closes: a flag item. + Entry e; + e.kind = Entry::Kind::Item; + e.value = std::string(name.text); + e.line = name.line; + node.entries.push_back(std::move(e)); + continue; + } + if (cur_.kind == Tok::Open) { + Entry e; + e.kind = Entry::Kind::Block; + e.key = std::string(name.text); + e.line = name.line; + e.block = std::make_unique(); + e.block->name = e.key; + e.block->line = cur_.line; + if (!advance()) return false; + if (!body(*e.block, depth + 1)) return false; + node.entries.push_back(std::move(e)); + continue; + } + // Bare or Quoted: a key/value pair. + Entry e; + e.kind = Entry::Kind::Pair; + e.key = std::string(name.text); + e.value = std::string(cur_.text); + e.quoted = cur_.kind == Tok::Quoted; + e.line = name.line; + node.entries.push_back(std::move(e)); + if (!advance()) return false; + continue; + } + } + } + } + + Lexer lex_; + Options opt_; + Token cur_; + Diagnostic err_; + std::vector warnings_; +}; + +} // namespace + +Result parse_blocks(std::string_view text, Options options) { + return Parser(text, options).run(); +} + +} // namespace mars::parse diff --git a/src/mars/parse/blocks.h b/src/mars/parse/blocks.h new file mode 100644 index 0000000..1a6f4d3 --- /dev/null +++ b/src/mars/parse/blocks.h @@ -0,0 +1,91 @@ +// mars::parse -- reader for the engine's brace-block key/value data format. +// +// This one grammar carries almost every catalog the game ships: *.weapon, +// *.shipsection, *.tech, *.combat, *.def, *.script and the block-form *.txt +// files (scenarios, tutorial, credits, system names, sky defs, ...). +// +// body := (block | pair | item)* +// block := NAME '{' body '}' +// pair := NAME value +// item := QUOTED | NAME (a NAME directly before '}' or EOF) +// value := QUOTED | BAREWORD +// comment := '//' ... end of line +// +// The tokenizer is whitespace-driven, not line-based: a block may open on the +// same line as a preceding pair, a name may sit on the same line as its brace, +// and a quoted string may span lines. There is no escape syntax -- backslashes +// and '//' inside quotes are literal. Bytes pass through untouched (cp1252). +// +// Leniency (the shipped data needs both; the engine accepts them): +// * end of file closes every open block -> Diagnostic in Document::warnings +// * a stray '}' at top level is ignored -> Diagnostic in Document::warnings +// With Options::strict both become errors. Three things are always errors: +// an unterminated quoted string, a '{' with no name before it, and nothing else. +#pragma once + +#include +#include +#include +#include + +#include "result.h" + +namespace mars::parse { + +struct Node; + +// One thing inside a block, in file order. +struct Entry { + enum class Kind { Pair, Item, Block }; + + Kind kind = Kind::Pair; + std::string key; // Pair/Block: the name as written (original case). Item: empty. + std::string value; // Pair: the value text; Item: the item text. Block: empty. + bool quoted = false; // value/item was written inside double quotes + std::unique_ptr block; // Block only + int line = 0; // 1-based line of the key (or of the item) + + Entry(); + Entry(Entry&&) noexcept; + Entry& operator=(Entry&&) noexcept; + ~Entry(); + Entry(const Entry&) = delete; + Entry& operator=(const Entry&) = delete; + + bool is_pair() const { return kind == Kind::Pair; } + bool is_item() const { return kind == Kind::Item; } + bool is_block() const { return kind == Kind::Block; } +}; + +// A named block (or the document root, whose name is empty). +struct Node { + std::string name; + std::vector entries; + int line = 0; // line of the opening brace (0 for the root) + + // Lookups are case-insensitive on the key and see pairs *and* blocks + // (the data uses `option X` and `option { ... }` under one key). + const Entry* first(std::string_view key) const; + std::vector all(std::string_view key) const; + + // Convenience filters. `first_value` returns the first *pair* under the key. + const std::string* first_value(std::string_view key) const; + std::vector values(std::string_view key) const; + const Node* first_block(std::string_view key) const; + std::vector blocks(std::string_view key) const; + std::vector items() const; // bare quoted / trailing barewords + bool has(std::string_view key) const { return first(key) != nullptr; } +}; + +struct Document { + Node root; // the file's top level is itself a body + std::vector warnings; // lenient recoveries taken +}; + +struct Options { + bool strict = false; // reject files the engine would repair +}; + +Result parse_blocks(std::string_view text, Options options = {}); + +} // namespace mars::parse diff --git a/src/mars/parse/effect.cpp b/src/mars/parse/effect.cpp new file mode 100644 index 0000000..0172f81 --- /dev/null +++ b/src/mars/parse/effect.cpp @@ -0,0 +1,164 @@ +#include "effect.h" + +#include + +#include "value.h" + +namespace mars::parse { + +EffectEntry::EffectEntry() = default; +EffectEntry::EffectEntry(EffectEntry&&) noexcept = default; +EffectEntry& EffectEntry::operator=(EffectEntry&&) noexcept = default; +EffectEntry::~EffectEntry() = default; + +const EffectEntry* EffectGroup::first(std::string_view key) const { + for (const EffectEntry& e : entries) + if (iequals(e.key, key)) return &e; + return nullptr; +} + +std::vector EffectGroup::all(std::string_view key) const { + std::vector out; + for (const EffectEntry& e : entries) + if (iequals(e.key, key)) out.push_back(&e); + return out; +} + +namespace { + +bool is_space(unsigned char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'; +} + +// Cut a trailing '//' comment, ignoring '//' inside double quotes. +std::string_view strip_comment(std::string_view line) { + bool in_quote = false; + for (std::size_t i = 0; i < line.size(); ++i) { + char c = line[i]; + if (c == '"') in_quote = !in_quote; + else if (c == '/' && !in_quote && i + 1 < line.size() && line[i + 1] == '/') + return line.substr(0, i); + } + return line; +} + +std::string_view trim(std::string_view s) { + while (!s.empty() && is_space(static_cast(s.front()))) s.remove_prefix(1); + while (!s.empty() && is_space(static_cast(s.back()))) s.remove_suffix(1); + return s; +} + +// Split into tokens: "quoted text" (may hold spaces) or runs of non-space. +// A '"' that never closes is just part of a bare token. +std::vector split_tokens(std::string_view line) { + std::vector out; + std::size_t i = 0; + while (i < line.size()) { + unsigned char c = static_cast(line[i]); + if (is_space(c)) { ++i; continue; } + if (c == '"') { + std::size_t close = line.find('"', i + 1); + if (close != std::string_view::npos) { + out.push_back({std::string(line.substr(i + 1, close - i - 1)), true}); + i = close + 1; + continue; + } + } + std::size_t start = i; + while (i < line.size() && !is_space(static_cast(line[i]))) ++i; + out.push_back({std::string(line.substr(start, i - start)), false}); + } + return out; +} + +// Iterates lines split on '\n'; a trailing '\r' is dropped (CRLF files exist). +class Lines { +public: + explicit Lines(std::string_view s) : s_(s) {} + bool next(std::string_view& line) { + if (pos_ > s_.size()) return false; + std::size_t nl = s_.find('\n', pos_); + if (nl == std::string_view::npos) { + line = s_.substr(pos_); + pos_ = s_.size() + 1; + if (line.empty()) return false; // no dangling empty line after a final '\n' + } else { + line = s_.substr(pos_, nl - pos_); + pos_ = nl + 1; + } + if (!line.empty() && line.back() == '\r') line.remove_suffix(1); + ++lineno_; + return true; + } + int lineno() const { return lineno_; } + +private: + std::string_view s_; + std::size_t pos_ = 0; + int lineno_ = 0; +}; + +} // namespace + +Result parse_effect(std::string_view text) { + Lines lines(text); + std::string_view raw; + if (!lines.next(raw) || trim(raw) != "TXT") + return Diagnostic{1, "missing TXT magic"}; + + EffectFile file; + std::vector stack{&file.root}; + bool have_pending = false; + std::string pending_key; + int pending_line = 0; + + while (lines.next(raw)) { + const int lineno = lines.lineno(); + std::string_view line = trim(strip_comment(raw)); + if (line.empty()) continue; + + if (line == "BEGIN") { + if (!have_pending) return Diagnostic{lineno, "BEGIN without a key"}; + EffectEntry e; + e.key = std::move(pending_key); + e.line = pending_line; + e.group = std::make_unique(); + EffectGroup* grp = e.group.get(); + stack.back()->entries.push_back(std::move(e)); + stack.push_back(grp); + have_pending = false; + pending_key.clear(); + continue; + } + if (line == "END") { + if (stack.size() == 1) return Diagnostic{lineno, "END without BEGIN"}; + stack.pop_back(); + continue; + } + if (have_pending) + return Diagnostic{lineno, "key '" + pending_key + "' not followed by BEGIN"}; + + std::vector toks = split_tokens(line); + // line is non-empty after trim, so there is at least one token + if (toks.size() == 1) { + have_pending = true; + pending_key = std::move(toks[0].text); + pending_line = lineno; + continue; + } + EffectEntry e; + e.key = std::move(toks[0].text); + e.line = lineno; + e.values.assign(std::make_move_iterator(toks.begin() + 1), + std::make_move_iterator(toks.end())); + stack.back()->entries.push_back(std::move(e)); + } + + if (stack.size() != 1) + return Diagnostic{lines.lineno(), std::to_string(stack.size() - 1) + " unclosed BEGIN group(s)"}; + if (have_pending) + return Diagnostic{pending_line, "trailing key '" + pending_key + "' without BEGIN"}; + return file; +} + +} // namespace mars::parse diff --git a/src/mars/parse/effect.h b/src/mars/parse/effect.h new file mode 100644 index 0000000..41e12c9 --- /dev/null +++ b/src/mars/parse/effect.h @@ -0,0 +1,70 @@ +// mars::parse -- reader for Effects/*.effect (particle-effect definitions). +// +// NOT the brace-block format. Line-based: +// +// TXT magic, first line +// KEY value [value ...] scalar line (number, TRUE/FALSE, "quoted") +// KEY group: the key alone on its line, then +// BEGIN +// ...nested lines / groups... +// END +// +// Indentation is cosmetic. '//' starts a comment unless inside quotes. Order is +// semantic (a PARTICLEDATATYPE line is followed by the curves that belong to +// it; MODIFIER repeats once per type), so each level is an ordered list of +// entries and repeated keys are simply repeated. Bytes pass through (cp1252). +// +// Errors (no leniency has been observed to be needed -- every shipped file is +// balanced): missing TXT magic, BEGIN without a key, END without BEGIN, a lone +// key not followed by BEGIN, unclosed groups at EOF. +#pragma once + +#include +#include +#include +#include + +#include "result.h" + +namespace mars::parse { + +struct EffectGroup; + +struct EffectValue { + std::string text; // raw token text (quotes stripped) + bool quoted = false; +}; + +struct EffectEntry { + std::string key; // as written (the files use UPPERCASE) + std::vector values; // scalar line: 1+ values. Group line: empty. + std::unique_ptr group; // set for KEY + BEGIN...END + int line = 0; + + EffectEntry(); + EffectEntry(EffectEntry&&) noexcept; + EffectEntry& operator=(EffectEntry&&) noexcept; + ~EffectEntry(); + EffectEntry(const EffectEntry&) = delete; + EffectEntry& operator=(const EffectEntry&) = delete; + + bool is_group() const { return group != nullptr; } + // First value's text, or empty. + std::string_view value() const { return values.empty() ? std::string_view{} : values[0].text; } +}; + +struct EffectGroup { + std::vector entries; + + // Case-insensitive lookups over this level only. + const EffectEntry* first(std::string_view key) const; + std::vector all(std::string_view key) const; +}; + +struct EffectFile { + EffectGroup root; +}; + +Result parse_effect(std::string_view text); + +} // namespace mars::parse diff --git a/src/mars/parse/result.h b/src/mars/parse/result.h new file mode 100644 index 0000000..adb6b73 --- /dev/null +++ b/src/mars/parse/result.h @@ -0,0 +1,41 @@ +// mars::parse -- error/result types shared by the data-file readers. +// No exceptions cross this API: every parser returns Result. +#pragma once + +#include +#include +#include +#include + +namespace mars::parse { + +// A problem found while reading. `line` is 1-based; 0 means "whole file". +struct Diagnostic { + int line = 0; + std::string message; +}; + +template +class Result { +public: + Result(T v) : value_(std::move(v)) {} // NOLINT(google-explicit-constructor) + Result(Diagnostic e) : error_(std::move(e)) {} // NOLINT(google-explicit-constructor) + + bool ok() const { return value_.has_value(); } + explicit operator bool() const { return ok(); } + + T& value() & { return *value_; } + const T& value() const& { return *value_; } + T&& value() && { return std::move(*value_); } + + T* operator->() { return &*value_; } + const T* operator->() const { return &*value_; } + + const Diagnostic& error() const { return error_; } + +private: + std::optional value_; + Diagnostic error_; +}; + +} // namespace mars::parse diff --git a/src/mars/parse/value.cpp b/src/mars/parse/value.cpp new file mode 100644 index 0000000..8bbaa29 --- /dev/null +++ b/src/mars/parse/value.cpp @@ -0,0 +1,95 @@ +#include "value.h" + +#include +#include +#include + +namespace mars::parse { + +namespace { + +bool is_digit(unsigned char c) { return c >= '0' && c <= '9'; } + +// Consumes the optional sign, returns the rest. +std::string_view strip_sign(std::string_view s) { + if (!s.empty() && (s[0] == '+' || s[0] == '-')) s.remove_prefix(1); + return s; +} + +bool all_digits(std::string_view s) { + if (s.empty()) return false; + for (unsigned char c : s) + if (!is_digit(c)) return false; + return true; +} + +} // namespace + +unsigned char ascii_lower(unsigned char c) { + return (c >= 'A' && c <= 'Z') ? static_cast(c + ('a' - 'A')) : c; +} + +bool iequals(std::string_view a, std::string_view b) { + if (a.size() != b.size()) return false; + for (std::size_t i = 0; i < a.size(); ++i) + if (ascii_lower(static_cast(a[i])) != + ascii_lower(static_cast(b[i]))) + return false; + return true; +} + +ScalarKind classify(std::string_view tok) { + if (tok.empty()) return ScalarKind::Text; + if (iequals(tok, "true") || iequals(tok, "false")) return ScalarKind::Bool; + + std::string_view body = strip_sign(tok); + if (all_digits(body)) return ScalarKind::Int; + + // mantissa: d+.d* | .d+ | d+ + std::size_t i = 0; + std::size_t int_digits = 0; + while (i < body.size() && is_digit(static_cast(body[i]))) { ++i; ++int_digits; } + std::size_t frac_digits = 0; + if (i < body.size() && body[i] == '.') { + ++i; + while (i < body.size() && is_digit(static_cast(body[i]))) { ++i; ++frac_digits; } + } + if (int_digits == 0 && frac_digits == 0) return ScalarKind::Text; // ".", "-", "e5" + // optional exponent (a dotless mantissa with an exponent, "3e5", is a float) + if (i < body.size() && (body[i] == 'e' || body[i] == 'E')) { + ++i; + if (i < body.size() && (body[i] == '+' || body[i] == '-')) ++i; + std::size_t exp_digits = 0; + while (i < body.size() && is_digit(static_cast(body[i]))) { ++i; ++exp_digits; } + if (exp_digits == 0) return ScalarKind::Text; + } + return i == body.size() ? ScalarKind::Float : ScalarKind::Text; +} + +std::optional as_int(std::string_view tok) { + if (classify(tok) != ScalarKind::Int) return std::nullopt; + std::string buf(tok); + errno = 0; + char* end = nullptr; + long long v = std::strtoll(buf.c_str(), &end, 10); + if (errno == ERANGE || end != buf.c_str() + buf.size()) return std::nullopt; + return static_cast(v); +} + +std::optional as_double(std::string_view tok) { + ScalarKind k = classify(tok); + if (k != ScalarKind::Int && k != ScalarKind::Float) return std::nullopt; + std::string buf(tok); + char* end = nullptr; + double v = std::strtod(buf.c_str(), &end); + if (end != buf.c_str() + buf.size()) return std::nullopt; + return v; +} + +std::optional as_bool(std::string_view tok) { + if (iequals(tok, "true")) return true; + if (iequals(tok, "false")) return false; + return std::nullopt; +} + +} // namespace mars::parse diff --git a/src/mars/parse/value.h b/src/mars/parse/value.h new file mode 100644 index 0000000..bd28691 --- /dev/null +++ b/src/mars/parse/value.h @@ -0,0 +1,30 @@ +// mars::parse -- scalar value helpers shared by the block and effect readers. +// +// Parsers keep every value as the raw bytes that appeared in the file (cp1252 +// passes straight through). These helpers classify and convert a bareword the +// way the data files use them: C-style ints and floats ("7", "-.8", "7e+8", +// "5."), and case-insensitive true/false. Quoted strings are never numbers. +#pragma once + +#include +#include +#include + +namespace mars::parse { + +enum class ScalarKind { Text, Int, Float, Bool }; + +// Classify a bareword. Int: [+-]?digits. Float: [+-]?(d+.d*|.d+|d+)([eE][+-]?d+)? +// (an Int-shaped token is reported as Int, not Float). Bool: true/false, any case. +ScalarKind classify(std::string_view bareword); + +std::optional as_int(std::string_view bareword); // Int shape only +std::optional as_double(std::string_view bareword); // Int or Float shape +std::optional as_bool(std::string_view bareword); // true/false any case + +// Case-insensitive ASCII comparison (keys and identifiers are matched this way +// throughout the data; cp1252 bytes above 0x7f compare as-is). +bool iequals(std::string_view a, std::string_view b); +unsigned char ascii_lower(unsigned char c); + +} // namespace mars::parse diff --git a/tests/mars_parse/.gitignore b/tests/mars_parse/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/tests/mars_parse/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/tests/mars_parse/build_and_run.sh b/tests/mars_parse/build_and_run.sh new file mode 100755 index 0000000..5756e4d --- /dev/null +++ b/tests/mars_parse/build_and_run.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Self-contained build + test for src/mars/parse (plain g++, no CMake needed). +# +# tests/mars_parse/build_and_run.sh +# +# Always runs the unit tests. When SOTS_DATA_DIR points at an extracted +# sots.gob tree it also parses every brace-block and .effect file there and, +# if the reference Python parsers are reachable (SOTS_RE_PARSERS or +# ~/sots-re/verify/parsers), cross-checks each parse against them. +# +# Env: CXX (g++), PYTHON (python3), BUILD_DIR (tests/mars_parse/build), +# SOTS_DATA_DIR, SOTS_RE_PARSERS. +set -euo pipefail + +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ROOT=$(cd "$HERE/../.." && pwd) +BUILD=${BUILD_DIR:-$HERE/build} +CXX=${CXX:-g++} +PYTHON=${PYTHON:-python3} +FLAGS=(-std=c++17 -O2 -Wall -Wextra -Wpedantic -Werror -I"$ROOT/src" -I"$HERE") +LIB=("$ROOT/src/mars/parse/blocks.cpp" "$ROOT/src/mars/parse/effect.cpp" "$ROOT/src/mars/parse/value.cpp") + +mkdir -p "$BUILD" + +echo "== building unit tests" +"$CXX" "${FLAGS[@]}" "${LIB[@]}" "$HERE/canon.cpp" "$HERE/test_main.cpp" \ + "$HERE/test_value.cpp" "$HERE/test_blocks.cpp" "$HERE/test_effect.cpp" \ + -o "$BUILD/unit_tests" +echo "== running unit tests" +"$BUILD/unit_tests" + +if [ -z "${SOTS_DATA_DIR:-}" ]; then + echo "== SOTS_DATA_DIR not set: skipping real-data oracle test" + exit 0 +fi +if [ ! -d "$SOTS_DATA_DIR" ]; then + echo "== SOTS_DATA_DIR=$SOTS_DATA_DIR is not a directory" >&2 + exit 1 +fi + +echo "== building oracle_check" +"$CXX" "${FLAGS[@]}" "${LIB[@]}" "$HERE/canon.cpp" "$HERE/oracle_check.cpp" -o "$BUILD/oracle_check" + +ORACLE="$BUILD/oracle" +rm -rf "$ORACLE" +echo "== running reference parsers (dump.py)" +set +e +"$PYTHON" "$HERE/oracle/dump.py" "$SOTS_DATA_DIR" "$ORACLE" ${SOTS_RE_PARSERS:+--parsers "$SOTS_RE_PARSERS"} +rc=$? +set -e +if [ $rc -eq 3 ]; then + echo "== reference parsers not found: parsing real data without cross-check" + exec "$BUILD/oracle_check" "$SOTS_DATA_DIR" +elif [ $rc -ne 0 ]; then + echo "== dump.py failed (rc=$rc)" >&2 + exit $rc +fi + +echo "== cross-checking against the oracle" +exec "$BUILD/oracle_check" "$SOTS_DATA_DIR" "$ORACLE" diff --git a/tests/mars_parse/canon.cpp b/tests/mars_parse/canon.cpp new file mode 100644 index 0000000..f4a3b72 --- /dev/null +++ b/tests/mars_parse/canon.cpp @@ -0,0 +1,187 @@ +#include "canon.h" + +#include +#include +#include +#include +#include + +#include "mars/parse/value.h" + +namespace canon { + +namespace { + +using mars::parse::Entry; +using mars::parse::Node; + +// Minimal JSON value: enough to mirror the Python dict/list/scalar shapes. +struct Json { + enum class Kind { Raw, String, Array, Object } kind = Kind::Raw; + std::string raw; // Raw: already-rendered literal + std::string str; // String + std::vector items; // Array + std::vector> members; // Object (insertion order) + + static Json rawlit(std::string s) { Json j; j.kind = Kind::Raw; j.raw = std::move(s); return j; } + static Json string(std::string s) { Json j; j.kind = Kind::String; j.str = std::move(s); return j; } + static Json array() { Json j; j.kind = Kind::Array; return j; } + static Json object() { Json j; j.kind = Kind::Object; return j; } + + // Python's _add(): first value is scalar, the second turns it into a list. + void add(const std::string& key, Json value) { + for (auto& m : members) { + if (m.first != key) continue; + if (m.second.kind == Kind::Array && m.second.is_multi) { + m.second.items.push_back(std::move(value)); + } else { + Json list = array(); + list.is_multi = true; + list.items.push_back(std::move(m.second)); + list.items.push_back(std::move(value)); + m.second = std::move(list); + } + return; + } + members.emplace_back(key, std::move(value)); + } + bool is_multi = false; // array created by repetition (vs. a value list) +}; + +void render(const Json& j, std::string& out) { + switch (j.kind) { + case Json::Kind::Raw: out += j.raw; break; + case Json::Kind::String: out += json_string(j.str); break; + case Json::Kind::Array: + out += '['; + for (std::size_t i = 0; i < j.items.size(); ++i) { + if (i) out += ','; + render(j.items[i], out); + } + out += ']'; + break; + case Json::Kind::Object: { + std::vector*> sorted; + for (const auto& m : j.members) sorted.push_back(&m); + std::sort(sorted.begin(), sorted.end(), + [](auto* a, auto* b) { return a->first < b->first; }); + out += '{'; + for (std::size_t i = 0; i < sorted.size(); ++i) { + if (i) out += ','; + out += json_string(sorted[i]->first); + out += ':'; + render(sorted[i]->second, out); + } + out += '}'; + break; + } + } +} + +// Typed rendering of a bareword, mirroring mars_data.coerce(). +Json typed(const std::string& tok) { + using mars::parse::ScalarKind; + switch (mars::parse::classify(tok)) { + case ScalarKind::Int: + if (auto v = mars::parse::as_int(tok)) return Json::rawlit(std::to_string(*v)); + return Json::string(tok); // out of 64-bit range: cannot match the oracle, surfaces as a diff + case ScalarKind::Float: { + char buf[64]; + std::snprintf(buf, sizeof buf, "%.17g", *mars::parse::as_double(tok)); + return Json::string(std::string("\x01") + buf); + } + case ScalarKind::Bool: + return Json::rawlit(*mars::parse::as_bool(tok) ? "true" : "false"); + case ScalarKind::Text: + break; + } + return Json::string(tok); +} + +std::string lower(const std::string& s) { + std::string out(s); + for (char& c : out) c = static_cast(mars::parse::ascii_lower(static_cast(c))); + return out; +} + +Json node_to_json(const Node& n) { + Json obj = Json::object(); + for (const Entry& e : n.entries) { + switch (e.kind) { + case Entry::Kind::Item: + obj.add("_items", Json::string(e.value)); + break; + case Entry::Kind::Pair: + obj.add(lower(e.key), e.quoted ? Json::string(e.value) : typed(e.value)); + break; + case Entry::Kind::Block: + obj.add(lower(e.key), node_to_json(*e.block)); + break; + } + } + return obj; +} + +Json group_to_json(const mars::parse::EffectGroup& g) { + Json list = Json::array(); + for (const mars::parse::EffectEntry& e : g.entries) { + Json pair = Json::array(); + pair.items.push_back(Json::string(e.key)); + if (e.is_group()) { + pair.items.push_back(group_to_json(*e.group)); + } else if (e.values.size() == 1) { + const auto& v = e.values[0]; + pair.items.push_back(v.quoted ? Json::string(v.text) : typed(v.text)); + } else { + Json vals = Json::array(); + for (const auto& v : e.values) + vals.items.push_back(v.quoted ? Json::string(v.text) : typed(v.text)); + pair.items.push_back(std::move(vals)); + } + list.items.push_back(std::move(pair)); + } + return list; +} + +} // namespace + +std::string json_string(const std::string& s) { + std::string out; + out.reserve(s.size() + 2); + out += '"'; + for (unsigned char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + default: + if (c < 0x20 || c > 0x7e) { + char buf[8]; + std::snprintf(buf, sizeof buf, "\\u%04x", c); + out += buf; + } else { + out += static_cast(c); + } + } + } + out += '"'; + return out; +} + +std::string blocks_json(const Node& root) { + std::string out; + render(node_to_json(root), out); + return out; +} + +std::string effect_json(const mars::parse::EffectGroup& root) { + std::string out; + render(group_to_json(root), out); + return out; +} + +} // namespace canon diff --git a/tests/mars_parse/canon.h b/tests/mars_parse/canon.h new file mode 100644 index 0000000..c27d11a --- /dev/null +++ b/tests/mars_parse/canon.h @@ -0,0 +1,27 @@ +// Canonical JSON rendering of parsed trees, byte-identical to what +// tests/mars_parse/oracle/dump.py emits from the reference Python parsers. +// +// Brace documents become the oracle's dict shape: keys lower-cased and sorted, +// a key seen once maps to its value, a key seen more than once maps to a list +// (file order), bare items collect under "_items", pair values that are +// barewords are typed (int -> JSON number, true/false -> JSON bool, float -> +// "" + printf("%.17g")), everything else is a string. +// +// Effect files become the oracle's ordered [[key, value], ...] shape. +#pragma once + +#include + +#include "mars/parse/blocks.h" +#include "mars/parse/effect.h" + +namespace canon { + +std::string blocks_json(const mars::parse::Node& root); +std::string effect_json(const mars::parse::EffectGroup& root); + +// JSON string literal with the oracle's escaping (ensure_ascii; bytes >= 0x7f +// as \u00xx). +std::string json_string(const std::string& bytes); + +} // namespace canon diff --git a/tests/mars_parse/oracle/dump.py b/tests/mars_parse/oracle/dump.py new file mode 100644 index 0000000..d862542 --- /dev/null +++ b/tests/mars_parse/oracle/dump.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Emit the reference (sots-re) parse of every brace-block and .effect data +file as canonical JSON, one file per input, for tests/mars_parse/oracle_check. + + dump.py [--parsers ] + + an extracted sots.gob tree (the owner's copy, never committed) + receives .json per input plus index.txt: + TAB TAB +--parsers directory holding the reference readers mars_data.py / + effect_txt.py / flat_kv.py / verify.py (default: + $SOTS_RE_PARSERS, else ~/sots-re/verify/parsers) + +Canonical form (mirrored by tests/mars_parse/canon.cpp): + * brace files: the reader's dict, json.dumps(sort_keys=True, compact, + ensure_ascii=True); floats rendered as "\\x01" + "%.17g" strings so both + sides format through the same printf rules. + * effect files: the reader's ordered [[key, value], ...] list, same rules. + * strings are the file's cp1252 bytes, one \\u00xx per byte >= 0x7f. + +Exit status 3 when the reference parsers cannot be found (the caller then runs +the C++ side in count-only mode). +""" +from __future__ import annotations + +import json +import os +import sys + + +def find_parsers(explicit: str | None) -> str | None: + cands = [explicit, os.environ.get("SOTS_RE_PARSERS"), + os.path.expanduser("~/sots-re/verify/parsers")] + for c in cands: + if c and os.path.isfile(os.path.join(c, "mars_data.py")): + return c + return None + + +class Canon(json.JSONEncoder): + """Floats -> marker string; str -> cp1252 bytes as latin-1 code points.""" + + def default(self, o): # pragma: no cover - not reached for our types + return super().default(o) + + def iterencode(self, o, _one_shot=False): + return super().iterencode(self._fix(o), _one_shot) + + def _fix(self, o): + if isinstance(o, bool): + return o + if isinstance(o, float): + return "\x01" + ("%.17g" % o) + if isinstance(o, str): + return o.encode("cp1252").decode("latin-1") + if isinstance(o, dict): + return {self._fix(k): self._fix(v) for k, v in o.items()} + if isinstance(o, (list, tuple)): + return [self._fix(v) for v in o] + return o + + +def dumps(obj) -> str: + return json.dumps(obj, cls=Canon, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +def main(argv: list[str]) -> int: + args = [a for a in argv[1:]] + parsers = None + if "--parsers" in args: + i = args.index("--parsers") + parsers = args[i + 1] + del args[i:i + 2] + if len(args) != 2: + print(__doc__, file=sys.stderr) + return 2 + data_dir, out_dir = args + pdir = find_parsers(parsers) + if pdir is None: + print("dump.py: reference parsers not found (set SOTS_RE_PARSERS)", file=sys.stderr) + return 3 + sys.path.insert(0, pdir) + import effect_txt # noqa: E402 + import mars_data # noqa: E402 + import verify # noqa: E402 (for kind_of: which .txt files are brace-form) + + index = [] + counts: dict[str, int] = {} + for dp, _, fns in os.walk(data_dir): + for fn in sorted(fns): + path = os.path.join(dp, fn) + rel = os.path.relpath(path, data_dir).replace(os.sep, "/") + kind = verify.kind_of(rel) + if kind.startswith("brace:"): + warnings: list = [] + obj = mars_data.parse_file(path, warnings=warnings) + nwarn = len(warnings) + elif kind == "effect": + obj = effect_txt.parse_file(path) + nwarn = 0 + else: + continue + out_path = os.path.join(out_dir, rel + ".json") + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "w", encoding="ascii", newline="\n") as f: + f.write(dumps(obj)) + index.append(f"{kind}\t{nwarn}\t{rel}") + counts[kind] = counts.get(kind, 0) + 1 + + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, "index.txt"), "w", encoding="utf-8", newline="\n") as f: + f.write("\n".join(index) + ("\n" if index else "")) + for k in sorted(counts): + print(f"oracle {k}: {counts[k]}") + print(f"oracle total: {sum(counts.values())} files -> {out_dir}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/mars_parse/oracle_check.cpp b/tests/mars_parse/oracle_check.cpp new file mode 100644 index 0000000..210b15f --- /dev/null +++ b/tests/mars_parse/oracle_check.cpp @@ -0,0 +1,216 @@ +// Real-data test: parse every brace-block / .effect file under a data tree and +// compare the canonical rendering with the reference (Python) parse. +// +// oracle_check [] +// +// is what tests/mars_parse/oracle/dump.py produced (index.txt + +// one .json per file). Without it the tool only parses and counts. On a +// mismatch the C++ rendering is written next to the oracle file as +// .cpp.json so the two can be diffed. +// +// Also reports which files need the lenient recoveries (strict mode fails). +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "canon.h" +#include "mars/parse/blocks.h" +#include "mars/parse/effect.h" + +namespace fs = std::filesystem; + +namespace { + +bool read_file(const fs::path& p, std::string& out) { + std::ifstream in(p, std::ios::binary); + if (!in) return false; + out.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); + return true; +} + +std::string ext_lower(const fs::path& p) { + std::string e = p.extension().string(); + for (char& c : e) c = static_cast(std::tolower(static_cast(c))); + return e; +} + +// Which .txt files are brace-form (mirrors verify.kind_of in the RE repo). +bool is_brace_txt(const std::string& rel) { + static const char* const kNamed[] = { + "Data/tutorial.txt", "Data/credits.txt", "Data/Strategy/systemnames.txt", + "Data/Combat/ctechvars.txt", "Data/Combat/shipai.txt", + "Models/Skysphere/skydefs.txt", "Models/Skysphere/NodeSpace-skydefs.txt"}; + if (rel.rfind("Scenarios/", 0) == 0) return true; + for (const char* n : kNamed) + if (rel == n) return true; + return false; +} + +// "" when the file is not one of ours. +std::string kind_of(const std::string& rel) { + std::string e = ext_lower(rel); + if (e == ".weapon" || e == ".shipsection" || e == ".tech" || e == ".combat" || + e == ".def" || e == ".script") + return "brace:" + e.substr(1); + if (e == ".effect") return "effect"; + if (e == ".txt" && is_brace_txt(rel)) return "brace:txt"; + return ""; +} + +struct Entry { + std::string kind; + int oracle_warnings = -1; // -1: no oracle + std::string rel; +}; + +struct Stats { + int files = 0, parsed = 0, failed = 0, compared = 0, matched = 0, lenient = 0; +}; + +std::string first_diff(const std::string& a, const std::string& b) { + std::size_t i = 0; + while (i < a.size() && i < b.size() && a[i] == b[i]) ++i; + std::size_t lo = i > 60 ? i - 60 : 0; + std::ostringstream os; + os << "first difference at byte " << i << "\n cpp: ..." << a.substr(lo, 140) + << "\n oracle: ..." << b.substr(lo, 140); + return os.str(); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2 || argc > 3) { + std::fprintf(stderr, "usage: oracle_check []\n"); + return 2; + } + const fs::path data_dir = argv[1]; + const fs::path oracle_dir = argc == 3 ? fs::path(argv[2]) : fs::path(); + const bool have_oracle = !oracle_dir.empty(); + + // Build the work list: from the oracle index when present, else by walking. + std::vector work; + if (have_oracle) { + std::ifstream idx(oracle_dir / "index.txt"); + if (!idx) { + std::fprintf(stderr, "cannot read %s\n", (oracle_dir / "index.txt").c_str()); + return 2; + } + std::string line; + while (std::getline(idx, line)) { + if (line.empty()) continue; + std::size_t t1 = line.find('\t'), t2 = line.find('\t', t1 + 1); + if (t1 == std::string::npos || t2 == std::string::npos) continue; + Entry e; + e.kind = line.substr(0, t1); + e.oracle_warnings = std::stoi(line.substr(t1 + 1, t2 - t1 - 1)); + e.rel = line.substr(t2 + 1); + work.push_back(e); + } + } else { + for (const auto& de : fs::recursive_directory_iterator(data_dir)) { + if (!de.is_regular_file()) continue; + std::string rel = fs::relative(de.path(), data_dir).generic_string(); + std::string k = kind_of(rel); + if (k.empty()) continue; + work.push_back({k, -1, rel}); + } + } + + std::map by_kind; + std::vector failures, divergences, lenient_files; + + for (const Entry& e : work) { + Stats& st = by_kind[e.kind]; + ++st.files; + std::string src; + if (!read_file(data_dir / e.rel, src)) { + ++st.failed; + failures.push_back(e.rel + ": cannot read"); + continue; + } + + std::string rendered; + int warnings = 0; + if (e.kind == "effect") { + auto r = mars::parse::parse_effect(src); + if (!r.ok()) { + ++st.failed; + failures.push_back(e.rel + ": line " + std::to_string(r.error().line) + ": " + r.error().message); + continue; + } + rendered = canon::effect_json(r->root); + } else { + auto r = mars::parse::parse_blocks(src); + if (!r.ok()) { + ++st.failed; + failures.push_back(e.rel + ": line " + std::to_string(r.error().line) + ": " + r.error().message); + continue; + } + warnings = static_cast(r->warnings.size()); + rendered = canon::blocks_json(r->root); + mars::parse::Options strict; + strict.strict = true; + if (!mars::parse::parse_blocks(src, strict).ok()) { + ++st.lenient; + lenient_files.push_back(e.rel + " (" + r->warnings[0].message + ")"); + } + } + ++st.parsed; + + if (!have_oracle) continue; + std::string expected; + if (!read_file(oracle_dir / (e.rel + ".json"), expected)) { + failures.push_back(e.rel + ": oracle json missing"); + continue; + } + ++st.compared; + bool same = rendered == expected && warnings == e.oracle_warnings; + if (same) { + ++st.matched; + } else { + std::string why = warnings != e.oracle_warnings + ? "warning count cpp=" + std::to_string(warnings) + + " oracle=" + std::to_string(e.oracle_warnings) + : first_diff(rendered, expected); + divergences.push_back(e.rel + ": " + why); + std::ofstream out(oracle_dir / (e.rel + ".cpp.json"), std::ios::binary); + out << rendered; + } + } + + std::printf("%-18s %6s %6s %6s %8s %7s %8s\n", "kind", "files", "parsed", "failed", "compared", "match", "lenient"); + Stats total; + for (const auto& [k, s] : by_kind) { + std::printf("%-18s %6d %6d %6d %8d %7d %8d\n", k.c_str(), s.files, s.parsed, s.failed, s.compared, s.matched, s.lenient); + total.files += s.files; total.parsed += s.parsed; total.failed += s.failed; + total.compared += s.compared; total.matched += s.matched; total.lenient += s.lenient; + } + std::printf("%-18s %6d %6d %6d %8d %7d %8d\n", "TOTAL", total.files, total.parsed, total.failed, total.compared, total.matched, total.lenient); + + if (!lenient_files.empty()) { + std::printf("\nfiles needing lenient recovery (strict mode rejects): %zu\n", lenient_files.size()); + for (const auto& f : lenient_files) std::printf(" %s\n", f.c_str()); + } + if (!failures.empty()) { + std::printf("\nPARSE FAILURES: %zu\n", failures.size()); + for (const auto& f : failures) std::printf(" %s\n", f.c_str()); + } + if (!divergences.empty()) { + std::printf("\nDIVERGENCES FROM ORACLE: %zu\n", divergences.size()); + for (const auto& d : divergences) std::printf(" %s\n", d.c_str()); + } + if (have_oracle) + std::printf("\noracle agreement: %d / %d\n", total.matched, total.compared); + if (total.files == 0) { + std::printf("no data files found under %s\n", data_dir.c_str()); + return 1; + } + return (failures.empty() && divergences.empty()) ? 0 : 1; +} diff --git a/tests/mars_parse/test_blocks.cpp b/tests/mars_parse/test_blocks.cpp new file mode 100644 index 0000000..3986449 --- /dev/null +++ b/tests/mars_parse/test_blocks.cpp @@ -0,0 +1,261 @@ +// Unit tests for the brace-block reader. Every sample is hand-written. +#include + +#include "canon.h" +#include "mars/parse/blocks.h" +#include "test_main.h" + +using namespace mars::parse; + +namespace { + +Document must_parse(std::string_view text, Options o = {}) { + auto r = parse_blocks(text, o); + if (!r.ok()) { + std::printf(" unexpected parse error line %d: %s\n", r.error().line, r.error().message.c_str()); + CHECK(r.ok()); + return Document{}; + } + return std::move(r).value(); +} + +std::string canon_of(std::string_view text, Options o = {}) { + return canon::blocks_json(must_parse(text, o).root); +} + +} // namespace + +TEST(basic_block_pairs_and_nesting) { + Document d = must_parse( + "weapon\n{\n\tname @WEAPON_X\n\tcost 50\n\tbank { mount { node N1 } }\n}\n"); + CHECK_EQ(d.root.entries.size(), std::size_t{1}); + const Node* w = d.root.first_block("weapon"); + CHECK(w != nullptr); + CHECK_EQ(w->name, "weapon"); + CHECK_EQ(*w->first_value("name"), "@WEAPON_X"); // @TOKEN preserved verbatim + CHECK_EQ(*w->first_value("cost"), "50"); // numbers kept as text + const Node* mount = w->first_block("bank")->first_block("mount"); + CHECK(mount != nullptr); + CHECK_EQ(*mount->first_value("node"), "N1"); + CHECK(d.warnings.empty()); + CHECK_EQ(canon_of("weapon { name @X cost 50 bank { mount { node N1 } } }"), + R"({"weapon":{"bank":{"mount":{"node":"N1"}},"cost":50,"name":"@X"}})"); +} + +TEST(quoted_values_keep_spaces_backslashes_and_slashes) { + Document d = must_parse( + "a \"hello world\"\n" + "b \"C:\\path\\to\\file.X\"\n" + "c \"http://x//y\" // real comment\n" + "d \"\"\n"); + CHECK_EQ(*d.root.first_value("a"), "hello world"); + CHECK_EQ(*d.root.first_value("b"), "C:\\path\\to\\file.X"); + CHECK_EQ(*d.root.first_value("c"), "http://x//y"); + CHECK_EQ(*d.root.first_value("d"), ""); + CHECK(d.root.first("a")->quoted); + CHECK(d.root.first("d")->quoted); + CHECK_EQ(d.root.entries.size(), std::size_t{4}); +} + +TEST(comments_are_stripped) { + std::string s = canon_of("// leading\nx 1 // trailing\n//y 2\nz 3\n"); + CHECK_EQ(s, R"({"x":1,"z":3})"); + // '//' only starts a comment at a token boundary: glued to a bareword it is + // part of the word (the reference reader behaves this way; the shipped data + // never relies on either reading). + CHECK_EQ(canon_of("z 3//glued\n"), R"({"z":"3//glued"})"); + CHECK_EQ(canon_of("z 3 //not glued\n"), R"({"z":3})"); +} + +TEST(repeated_keys_stay_in_order) { + Document d = must_parse("t { requires A requires B other 1 requires C }"); + auto all = d.root.first_block("t")->all("requires"); + CHECK_EQ(all.size(), std::size_t{3}); + CHECK_EQ(all[0]->value, "A"); + CHECK_EQ(all[1]->value, "B"); + CHECK_EQ(all[2]->value, "C"); + CHECK_EQ(*d.root.first_block("t")->first_value("requires"), "A"); + CHECK_EQ(canon_of("t { requires A requires B other 1 requires C }"), + R"({"t":{"other":1,"requires":["A","B","C"]}})"); +} + +TEST(keys_are_case_insensitive_but_case_is_kept) { + Document d = must_parse("tech { Requires X requires Y Badge b }"); + const Node* t = d.root.first_block("TECH"); + CHECK(t != nullptr); + CHECK_EQ(t->all("REQUIRES").size(), std::size_t{2}); + CHECK_EQ(t->first("badge")->key, "Badge"); + CHECK_EQ(canon_of("tech { Requires X requires Y }"), R"({"tech":{"requires":["X","Y"]}})"); +} + +TEST(bare_quoted_items_and_trailing_bareword) { + Document d = must_parse("names { \"Sol\" \"Alpha Centauri\" }\nflags { on off }"); + auto items = d.root.first_block("names")->items(); + CHECK_EQ(items.size(), std::size_t{2}); + CHECK_EQ(std::string(items[1]), "Alpha Centauri"); + // "on off": 'on' takes 'off' as its value; nothing is left over. + const Node* f = d.root.first_block("flags"); + CHECK_EQ(*f->first_value("on"), "off"); + // A lone word right before '}' is an item, not a key. + Document e = must_parse("flags { a 1 solo }"); + CHECK_EQ(e.root.first_block("flags")->items().size(), std::size_t{1}); + CHECK_EQ(std::string(e.root.first_block("flags")->items()[0]), "solo"); + CHECK_EQ(canon_of("flags { a 1 solo }"), R"({"flags":{"_items":"solo","a":1}})"); + CHECK_EQ(canon_of("n { \"x\" \"y\" }"), R"({"n":{"_items":["x","y"]}})"); + // lone bareword at EOF, top level + CHECK_EQ(canon_of("a 1 trailing"), R"({"_items":"trailing","a":1})"); +} + +TEST(option_scalar_and_block_share_a_key) { + Document d = must_parse( + "shipsection { option DRV_A option { option T1 option T2 } option DRV_B }"); + const Node* s = d.root.first_block("shipsection"); + auto all = s->all("option"); + CHECK_EQ(all.size(), std::size_t{3}); + CHECK(all[0]->is_pair()); + CHECK(all[1]->is_block()); + CHECK(all[2]->is_pair()); + CHECK_EQ(s->values("option").size(), std::size_t{2}); + CHECK_EQ(s->blocks("option").size(), std::size_t{1}); + CHECK_EQ(s->blocks("option")[0]->values("option").size(), std::size_t{2}); + CHECK_EQ(canon_of("s { option A option { option T1 option T2 } option B }"), + R"({"s":{"option":["A",{"option":["T1","T2"]},"B"]}})"); +} + +TEST(tokenizer_is_not_line_based) { + // block opens on the same line as a preceding pair; name glued to brace; + // several entries per line; entries split across lines + std::string s = canon_of( + "bank { turretsize small mount { node N }\n}\n" + "weapon{ name\n\nX }\n" + "a\n{\nb\n1\n}"); + CHECK_EQ(s, R"({"a":{"b":1},"bank":{"mount":{"node":"N"},"turretsize":"small"},"weapon":{"name":"X"}})"); +} + +TEST(braces_split_barewords) { + CHECK_EQ(canon_of("a{b 1}c 2"), R"({"a":{"b":1},"c":2})"); +} + +TEST(quotes_split_barewords) { + // abc"def" -> bareword abc, then quoted def (a pair abc="def") + CHECK_EQ(canon_of("abc\"def\""), R"({"abc":"def"})"); +} + +TEST(eof_closes_open_blocks_lenient) { + auto r = parse_blocks("shipsection {\n model M\n bank { mount { node N }\n"); + CHECK(r.ok()); + CHECK_EQ(r->warnings.size(), std::size_t{2}); // bank and shipsection both unclosed + CHECK_EQ(r->warnings[0].line, 4); + const Node* s = r->root.first_block("shipsection"); + CHECK(s != nullptr); + CHECK_EQ(*s->first_value("model"), "M"); + CHECK_EQ(*s->first_block("bank")->first_block("mount")->first_value("node"), "N"); +} + +TEST(eof_inside_block_is_an_error_when_strict) { + Options strict; + strict.strict = true; + auto r = parse_blocks("shipsection {\n model M\n", strict); + CHECK(!r.ok()); + CHECK_EQ(r.error().line, 3); +} + +TEST(stray_top_level_close_is_ignored_lenient) { + auto r = parse_blocks("a { x 1 }\n}\nb { y 2 }\n"); + CHECK(r.ok()); + CHECK_EQ(r->warnings.size(), std::size_t{1}); + CHECK_EQ(r->warnings[0].line, 2); + CHECK(r->root.first_block("a") != nullptr); + CHECK(r->root.first_block("b") != nullptr); // parsing continues after the stray brace + CHECK_EQ(canon::blocks_json(r->root), R"({"a":{"x":1},"b":{"y":2}})"); +} + +TEST(stray_top_level_close_is_an_error_when_strict) { + Options strict; + strict.strict = true; + auto r = parse_blocks("a { x 1 }\n}\n", strict); + CHECK(!r.ok()); + CHECK_EQ(r.error().line, 2); +} + +TEST(unterminated_string_is_always_an_error) { + auto r = parse_blocks("a \"never closed\n b 1\n"); + CHECK(!r.ok()); + CHECK_EQ(r.error().line, 1); + Options strict; + strict.strict = true; + CHECK(!parse_blocks("a \"never closed", strict).ok()); +} + +TEST(open_brace_without_name_is_always_an_error) { + auto r = parse_blocks("a 1\n{ b 2 }"); + CHECK(!r.ok()); + CHECK_EQ(r.error().line, 2); + // ... also when it follows a complete pair on the same line + CHECK(!parse_blocks("a 1 { b 2 }").ok()); +} + +TEST(crlf_and_line_numbers) { + Document d = must_parse("a 1\r\nb 2\r\n\r\nc { d 3 }\r\n"); + CHECK_EQ(d.root.first("a")->line, 1); + CHECK_EQ(d.root.first("b")->line, 2); + CHECK_EQ(d.root.first("c")->line, 4); + CHECK_EQ(d.root.first_block("c")->first("d")->line, 4); + CHECK_EQ(*d.root.first_value("b"), "2"); // no '\r' leaks into values +} + +TEST(multiline_quoted_string_counts_lines) { + Document d = must_parse("a \"line one\nline two\"\nb 2\n"); + CHECK_EQ(*d.root.first_value("a"), "line one\nline two"); + CHECK_EQ(d.root.first("b")->line, 3); +} + +TEST(empty_and_comment_only_input) { + Document d = must_parse(""); + CHECK(d.root.entries.empty()); + CHECK(d.warnings.empty()); + Document e = must_parse(" // nothing here\n\n"); + CHECK(e.root.entries.empty()); + CHECK_EQ(canon_of(""), "{}"); +} + +TEST(top_level_pairs_and_blocks_mix) { + // scenario .txt shape: top-level pairs plus repeated player{} blocks + std::string s = canon_of("name @S\nnumplayers \"8\"\nplayer { recommended 1 }\nplayer { recommended 2 }"); + CHECK_EQ(s, R"({"name":"@S","numplayers":"8","player":[{"recommended":1},{"recommended":2}]})"); +} + +TEST(numbers_are_typed_only_in_canonical_form_and_only_when_bare) { + // The Node keeps raw text; the canonical form types barewords like the oracle + std::string s = canon_of("i 7 f .5 e 7e+8 t TRUE q \"8\" n -.8 d 5."); + CHECK_EQ(s, "{\"d\":\"\\u00015\",\"e\":\"\\u0001700000000\",\"f\":\"\\u00010.5\"," + "\"i\":7,\"n\":\"\\u0001-0.80000000000000004\",\"q\":\"8\",\"t\":true}"); +} + +TEST(high_bytes_pass_through) { + std::string src = "a \"caf\xe9\"\nb na\xefve\n"; + Document d = must_parse(src); + CHECK_EQ(*d.root.first_value("a"), "caf\xe9"); + CHECK_EQ(*d.root.first_value("b"), "na\xefve"); + CHECK_EQ(canon::blocks_json(d.root), R"({"a":"caf\u00e9","b":"na\u00efve"})"); +} + +TEST(deep_nesting_returns_to_the_right_parent) { + Document d = must_parse("a { b { c { d 1 } e 2 } f 3 } g 4"); + const Node* a = d.root.first_block("a"); + CHECK_EQ(*a->first_block("b")->first_block("c")->first_value("d"), "1"); + CHECK_EQ(*a->first_block("b")->first_value("e"), "2"); + CHECK_EQ(*a->first_value("f"), "3"); + CHECK_EQ(*d.root.first_value("g"), "4"); + CHECK_EQ(a->first_block("b")->line, 1); +} + +TEST(lookup_misses_return_null_or_empty) { + Document d = must_parse("a 1"); + CHECK(d.root.first("zzz") == nullptr); + CHECK(d.root.first_value("zzz") == nullptr); + CHECK(d.root.first_block("a") == nullptr); // a is a pair, not a block + CHECK(d.root.all("zzz").empty()); + CHECK(!d.root.has("zzz")); + CHECK(d.root.has("A")); +} diff --git a/tests/mars_parse/test_effect.cpp b/tests/mars_parse/test_effect.cpp new file mode 100644 index 0000000..07fb076 --- /dev/null +++ b/tests/mars_parse/test_effect.cpp @@ -0,0 +1,141 @@ +// Unit tests for the .effect reader. Every sample is hand-written. +#include + +#include "canon.h" +#include "mars/parse/effect.h" +#include "test_main.h" + +using namespace mars::parse; + +namespace { + +const char* kSample = + "TXT\n" + "EFFECTLENGTH 3.000000\n" + "EFFECTLOOPS -1\n" + "NAME \"New Emitter\"\n" + "RANGEUP\n" + "\tBEGIN\n" + "\t\tNUMKEYS 1\n" + "\t\tTYPE 1\n" + "\tEND\n" + "PARTICLEDATATYPE 0\n" + "MODIFIER\n" + "\tBEGIN\n" + "\t\tCREATION\n" + "\t\t\tBEGIN\n" + "\t\t\t\tVALUE 0.500000\n" + "\t\t\tEND\n" + "\tEND\n" + "PARTICLEDATATYPE 1\n" + "MODIFIER\n" + "\tBEGIN\n" + "\t\tVISIBLE FALSE\n" + "\tEND\n"; + +EffectFile must_parse(std::string_view text) { + auto r = parse_effect(text); + if (!r.ok()) { + std::printf(" unexpected parse error line %d: %s\n", r.error().line, r.error().message.c_str()); + CHECK(r.ok()); + return EffectFile{}; + } + return std::move(r).value(); +} + +} // namespace + +TEST(effect_basic_shape_and_order) { + EffectFile f = must_parse(kSample); + const EffectGroup& root = f.root; + CHECK_EQ(root.entries.size(), std::size_t{8}); + CHECK_EQ(root.entries[0].key, "EFFECTLENGTH"); + CHECK_EQ(std::string(root.entries[0].value()), "3.000000"); + CHECK_EQ(std::string(root.entries[1].value()), "-1"); + CHECK_EQ(std::string(root.entries[2].value()), "New Emitter"); + CHECK(root.entries[2].values[0].quoted); + CHECK(root.entries[3].is_group()); + CHECK_EQ(root.entries[3].key, "RANGEUP"); + CHECK_EQ(root.entries[3].group->entries.size(), std::size_t{2}); + // order: PARTICLEDATATYPE 0, MODIFIER{...}, PARTICLEDATATYPE 1, MODIFIER{...} + CHECK_EQ(root.entries[4].key, "PARTICLEDATATYPE"); + CHECK_EQ(root.entries[5].key, "MODIFIER"); + CHECK_EQ(root.entries[6].key, "PARTICLEDATATYPE"); + CHECK_EQ(std::string(root.entries[6].value()), "1"); + CHECK_EQ(root.entries[7].key, "MODIFIER"); + // nested group inside a group + const EffectEntry* creation = root.entries[5].group->first("creation"); + CHECK(creation != nullptr && creation->is_group()); + CHECK_EQ(std::string(creation->group->first("VALUE")->value()), "0.500000"); + CHECK_EQ(root.all("modifier").size(), std::size_t{2}); + CHECK_EQ(root.entries[3].line, 5); + CHECK_EQ(root.entries[4].line, 10); +} + +TEST(effect_canonical_form) { + EffectFile f = must_parse("TXT\nA 1\nB \"x y\"\nG\n BEGIN\n C TRUE\n END\nA 2\n"); + CHECK_EQ(canon::effect_json(f.root), + R"([["A",1],["B","x y"],["G",[["C",true]]],["A",2]])"); +} + +TEST(effect_crlf) { + EffectFile f = must_parse("TXT\r\nA 1\r\nG\r\n\tBEGIN\r\n\t\tB 2\r\n\tEND\r\n"); + CHECK_EQ(f.root.entries.size(), std::size_t{2}); + CHECK_EQ(std::string(f.root.entries[0].value()), "1"); + CHECK_EQ(std::string(f.root.entries[1].group->entries[0].value()), "2"); +} + +TEST(effect_magic_required) { + CHECK(!parse_effect("").ok()); + CHECK(!parse_effect("A 1\n").ok()); + CHECK(!parse_effect("TXTX\n").ok()); + CHECK(parse_effect("TXT").ok()); + CHECK(parse_effect(" TXT \r\n").ok()); // whitespace around the magic is fine + CHECK(parse_effect("TXT\n").ok()); + CHECK_EQ(parse_effect("nope").error().line, 1); +} + +TEST(effect_structural_errors) { + auto r1 = parse_effect("TXT\nBEGIN\nEND\n"); + CHECK(!r1.ok()); + CHECK_EQ(r1.error().line, 2); + auto r2 = parse_effect("TXT\nA 1\nEND\n"); + CHECK(!r2.ok()); + CHECK_EQ(r2.error().line, 3); + auto r3 = parse_effect("TXT\nG\nA 1\n"); // key not followed by BEGIN + CHECK(!r3.ok()); + CHECK_EQ(r3.error().line, 3); + auto r4 = parse_effect("TXT\nG\n BEGIN\n A 1\n"); // unclosed + CHECK(!r4.ok()); + auto r5 = parse_effect("TXT\nA 1\nG\n"); // trailing key + CHECK(!r5.ok()); + CHECK_EQ(r5.error().line, 3); +} + +TEST(effect_comments_and_blank_lines) { + EffectFile f = must_parse("TXT\n// c\nA 1 // trailing\n\n \nB \"a//b\"\n"); + CHECK_EQ(f.root.entries.size(), std::size_t{2}); + CHECK_EQ(std::string(f.root.entries[0].value()), "1"); + CHECK_EQ(std::string(f.root.entries[1].value()), "a//b"); +} + +TEST(effect_multi_value_line) { + EffectFile f = must_parse("TXT\nCOLOR 1 0.5 \"z\"\n"); + CHECK_EQ(f.root.entries[0].values.size(), std::size_t{3}); + CHECK_EQ(f.root.entries[0].values[1].text, "0.5"); + CHECK(f.root.entries[0].values[2].quoted); + CHECK_EQ(canon::effect_json(f.root), "[[\"COLOR\",[1,\"\\u00010.5\",\"z\"]]]"); +} + +TEST(effect_unterminated_quote_is_a_bare_token) { + EffectFile f = must_parse("TXT\nA \"open\n"); + CHECK_EQ(f.root.entries[0].values[0].text, "\"open"); + CHECK(!f.root.entries[0].values[0].quoted); +} + +TEST(effect_lookup_misses) { + EffectFile f = must_parse("TXT\nA 1\n"); + CHECK(f.root.first("B") == nullptr); + CHECK(f.root.all("B").empty()); + CHECK(f.root.first("a") != nullptr); +} diff --git a/tests/mars_parse/test_main.cpp b/tests/mars_parse/test_main.cpp new file mode 100644 index 0000000..ab9b648 --- /dev/null +++ b/tests/mars_parse/test_main.cpp @@ -0,0 +1,32 @@ +#include "test_main.h" + +namespace testing { + +std::vector& registry() { + static std::vector r; + return r; +} + +int& failures() { + static int n = 0; + return n; +} + +void report_failure(const char* file, int line, const std::string& expr) { + std::printf(" FAIL %s:%d: %s\n", file, line, expr.c_str()); + ++failures(); +} + +} // namespace testing + +int main() { + int ran = 0; + for (const auto& c : testing::registry()) { + int before = testing::failures(); + c.fn(); + ++ran; + if (testing::failures() != before) std::printf("[FAILED] %s\n", c.name); + } + std::printf("%d unit tests, %d failed assertions\n", ran, testing::failures()); + return testing::failures() ? 1 : 0; +} diff --git a/tests/mars_parse/test_main.h b/tests/mars_parse/test_main.h new file mode 100644 index 0000000..6c78252 --- /dev/null +++ b/tests/mars_parse/test_main.h @@ -0,0 +1,49 @@ +// Tiny self-contained test harness (no third-party deps). +#pragma once + +#include +#include +#include +#include + +namespace testing { + +struct Case { + const char* name; + std::function fn; +}; + +std::vector& registry(); +int& failures(); +void report_failure(const char* file, int line, const std::string& expr); + +struct Register { + Register(const char* name, std::function fn) { registry().push_back({name, std::move(fn)}); } +}; + +} // namespace testing + +#define TEST(name) \ + static void test_##name(); \ + static testing::Register reg_##name(#name, test_##name); \ + static void test_##name() + +#define CHECK(expr) \ + do { \ + if (!(expr)) testing::report_failure(__FILE__, __LINE__, #expr); \ + } while (0) + +#define CHECK_EQ(a, b) \ + do { \ + if (!((a) == (b))) \ + testing::report_failure(__FILE__, __LINE__, \ + std::string(#a " == " #b " [got: ") + \ + testing_to_string(a) + " vs " + \ + testing_to_string(b) + "]"); \ + } while (0) + +inline std::string testing_to_string(const std::string& s) { return "\"" + s + "\""; } +inline std::string testing_to_string(const char* s) { return std::string("\"") + s + "\""; } +inline std::string testing_to_string(bool b) { return b ? "true" : "false"; } +template +std::string testing_to_string(const T& v) { return std::to_string(v); } diff --git a/tests/mars_parse/test_value.cpp b/tests/mars_parse/test_value.cpp new file mode 100644 index 0000000..1aec66c --- /dev/null +++ b/tests/mars_parse/test_value.cpp @@ -0,0 +1,60 @@ +#include "mars/parse/value.h" +#include "test_main.h" + +using namespace mars::parse; + +TEST(classify_ints) { + CHECK(classify("7") == ScalarKind::Int); + CHECK(classify("+7") == ScalarKind::Int); + CHECK(classify("-0") == ScalarKind::Int); + CHECK(classify("007") == ScalarKind::Int); + CHECK_EQ(*as_int("-12"), -12); + CHECK_EQ(*as_int("007"), 7); +} + +TEST(classify_floats) { + CHECK(classify(".5") == ScalarKind::Float); + CHECK(classify("-.8") == ScalarKind::Float); + CHECK(classify("7e+8") == ScalarKind::Float); + CHECK(classify("5.") == ScalarKind::Float); + CHECK(classify("1.5E-3") == ScalarKind::Float); + CHECK(classify("3e5") == ScalarKind::Float); + CHECK(*as_double(".5") == 0.5); + CHECK(*as_double("7e+8") == 7e8); + CHECK(*as_double("12") == 12.0); // Int shape is a valid double too + CHECK(!as_int(".5").has_value()); // but not the other way round +} + +TEST(classify_text) { + CHECK(classify("") == ScalarKind::Text); + CHECK(classify(".") == ScalarKind::Text); + CHECK(classify("-") == ScalarKind::Text); + CHECK(classify("1e") == ScalarKind::Text); + CHECK(classify("e5") == ScalarKind::Text); + CHECK(classify("1.2.3") == ScalarKind::Text); + CHECK(classify("0x10") == ScalarKind::Text); + CHECK(classify("12abc") == ScalarKind::Text); + CHECK(classify("@WEAPON_X") == ScalarKind::Text); + CHECK(classify("inf") == ScalarKind::Text); + CHECK(classify("nan") == ScalarKind::Text); + CHECK(!as_double("abc").has_value()); +} + +TEST(classify_bools) { + CHECK(classify("true") == ScalarKind::Bool); + CHECK(classify("TRUE") == ScalarKind::Bool); + CHECK(classify("False") == ScalarKind::Bool); + CHECK(*as_bool("FALSE") == false); + CHECK(!as_bool("yes").has_value()); +} + +TEST(int_overflow_is_rejected) { + CHECK(!as_int("99999999999999999999").has_value()); +} + +TEST(iequals_is_ascii_only) { + CHECK(iequals("Requires", "REQUIRES")); + CHECK(!iequals("a", "ab")); + CHECK(iequals("\xe9", "\xe9")); + CHECK(!iequals("\xe9", "\xc9")); // cp1252 bytes are not folded +}