sots-engine/tests/mars_parse/test_blocks.cpp

// Unit tests for the brace-block reader. Every sample is hand-written.
#include <string>

#include "canon.h"
#include "mars/parse/blocks.h"
#include "mars/parse/script.h"
#include "test_main.h"

using namespace mars::parse;

namespace {

Document must_parse(std::string_view text, Options o = {}) {
    auto r = parse_blocks(text, o);
    if (!r.ok()) {
        std::printf("  unexpected parse error line %d: %s\n", r.error().line, r.error().message.c_str());
        CHECK(r.ok());
        return Document{};
    }
    return std::move(r).value();
}

std::string canon_of(std::string_view text, Options o = {}) {
    return canon::blocks_json(must_parse(text, o).root);
}

Options strict_opts() {
    Options o;
    o.strict = true;
    return o;
}

}  // namespace

// ---- Script: the engine's pull tokenizer -------------------------------------

TEST(script_read_token_statuses) {
    Script s("a b\n");
    Script::Raw t;
    CHECK(s.read_token(t) == ReadStatus::Ok);
    CHECK_EQ(std::string(t.text), "a");
    CHECK(s.read_token(t) == ReadStatus::Ok);   // "b" is followed by the newline
    CHECK_EQ(std::string(t.text), "b");
    CHECK(s.read_token(t) == ReadStatus::NoInput);
    CHECK(t.text.empty());

    Script e("a b");
    CHECK(e.read_token(t) == ReadStatus::Ok);
    CHECK(e.read_token(t) == ReadStatus::AtEnd);  // "b" touches the end of input
    CHECK_EQ(std::string(t.text), "b");
    CHECK(e.read_token(t) == ReadStatus::NoInput);
}

TEST(script_next_returns_the_value_read_status) {
    Script s("k v\nname {\n}\nlast 1");
    ScriptToken t;
    CHECK(s.next(t) == ReadStatus::Ok);
    CHECK(t.type == ScriptToken::Type::Pair);
    CHECK_EQ(std::string(t.key), "k");
    CHECK_EQ(std::string(t.value), "v");
    CHECK(s.next(t) == ReadStatus::Ok);
    CHECK(t.type == ScriptToken::Type::Open);
    CHECK_EQ(std::string(t.key), "name");
    CHECK(s.next(t) == ReadStatus::Ok);
    CHECK(t.type == ScriptToken::Type::Close);
    CHECK(s.next(t) == ReadStatus::AtEnd);      // the step was read but is incomplete
    CHECK_EQ(std::string(t.key), "last");        // ... its text is still available
    CHECK_EQ(std::string(t.value), "1");
    CHECK(s.next(t) == ReadStatus::NoInput);
}

TEST(script_skip_block_counts_whole_brace_tokens_only) {
    Script s("x 1 a{ } b } after 2\n");
    ScriptToken t;
    CHECK(s.next(t) == ReadStatus::Ok);           // x 1
    CHECK(s.skip_block(1) == ReadStatus::Ok);     // a{ is a word; first whole } closes
    CHECK(s.next(t) == ReadStatus::Ok);
    CHECK_EQ(std::string(t.key), "b");            // b } -> pair b="}"
    CHECK_EQ(std::string(t.value), "}");
    Script u("a { b {");
    CHECK(u.skip_block(1) != ReadStatus::Ok);     // input ends inside the block
}

TEST(script_token_text_is_capped_at_1023_bytes) {
    const std::string text = std::string(1100, 'x') + " 1\n";
    Script s(text);
    ScriptToken t;
    CHECK(s.next(t) == ReadStatus::Ok);
    CHECK_EQ(t.key.size(), Script::kMaxTokenLength);
    CHECK_EQ(std::string(t.value), "1");          // the whole word was consumed
}

// ---- tree ---------------------------------------------------------------------

TEST(basic_block_pairs_and_nesting) {
    Document d = must_parse(
        "weapon\n{\n\tname @WEAPON_X\n\tcost 50\n\tbank { mount { node N1 } }\n}\n");
    CHECK_EQ(d.root.entries.size(), std::size_t{1});
    const Node* w = d.root.first_block("weapon");
    CHECK(w != nullptr);
    CHECK_EQ(w->name, "weapon");
    CHECK_EQ(*w->first_value("name"), "@WEAPON_X");   // @TOKEN preserved verbatim
    CHECK_EQ(*w->first_value("cost"), "50");           // numbers kept as text
    const Node* mount = w->first_block("bank")->first_block("mount");
    CHECK(mount != nullptr);
    CHECK_EQ(*mount->first_value("node"), "N1");
    CHECK(d.warnings.empty());
    CHECK_EQ(canon_of("weapon { name @X cost 50 bank { mount { node N1 } } }"),
             R"({"weapon":{"bank":{"mount":{"node":"N1"}},"cost":50,"name":"@X"}})");
}

TEST(quoted_values_keep_spaces_backslashes_and_slashes) {
    Document d = must_parse(
        "a \"hello world\"\n"
        "b \"C:\\path\\to\\file.X\"\n"
        "c \"http://x//y\" // real comment\n"
        "d \"\"\n");
    CHECK_EQ(*d.root.first_value("a"), "hello world");
    CHECK_EQ(*d.root.first_value("b"), "C:\\path\\to\\file.X");
    CHECK_EQ(*d.root.first_value("c"), "http://x//y");
    CHECK_EQ(*d.root.first_value("d"), "");
    CHECK(d.root.first("a")->quoted);
    CHECK(d.root.first("d")->quoted);
    CHECK(!d.root.first("a")->key_quoted);
    CHECK_EQ(d.root.entries.size(), std::size_t{4});
}

TEST(single_quote_and_backtick_also_quote) {
    Document d = must_parse("a 'x y'\nb `p q`\nc \"it's\"\nd 'say \"hi\"'\n");
    CHECK_EQ(*d.root.first_value("a"), "x y");
    CHECK(d.root.first("a")->quoted);
    CHECK_EQ(*d.root.first_value("b"), "p q");
    CHECK_EQ(*d.root.first_value("c"), "it's");     // only the opening character closes
    CHECK_EQ(*d.root.first_value("d"), "say \"hi\"");
    CHECK(d.warnings.empty());
}

TEST(comments_are_stripped) {
    std::string s = canon_of("// leading\nx 1 // trailing\n//y 2\nz 3\n");
    CHECK_EQ(s, R"({"x":1,"z":3})");
    // The comment test is made on the extracted token: glued to a word it is
    // part of the word ...
    CHECK_EQ(canon_of("z 3//glued\n"), R"({"z":"3//glued"})");
    CHECK_EQ(canon_of("z 3 //not glued\n"), R"({"z":3})");
    // ... and a quoted token whose content starts with // is a comment too.
    CHECK_EQ(canon_of("\"// whole line\" junk\na 1\n"), R"({"a":1})");
    // a closing quote glued to // : the quoted token ends, then the comment starts
    CHECK_EQ(canon_of("c \"0 0 0\"// \" 92 76 20\"\n"), R"({"c":"0 0 0"})");
    // comment on the last line without a newline: the pair before it survives
    CHECK_EQ(canon_of("a 1 // end"), R"({"a":1})");
}

TEST(repeated_keys_stay_in_order) {
    Document d = must_parse("t { requires A requires B other 1 requires C }");
    auto all = d.root.first_block("t")->all("requires");
    CHECK_EQ(all.size(), std::size_t{3});
    CHECK_EQ(all[0]->value, "A");
    CHECK_EQ(all[1]->value, "B");
    CHECK_EQ(all[2]->value, "C");
    CHECK_EQ(*d.root.first_block("t")->first_value("requires"), "A");
    CHECK_EQ(canon_of("t { requires A requires B other 1 requires C }"),
             R"({"t":{"other":1,"requires":["A","B","C"]}})");
}

TEST(keys_are_case_insensitive_but_case_is_kept) {
    Document d = must_parse("tech { Requires X requires Y Badge b }");
    const Node* t = d.root.first_block("TECH");
    CHECK(t != nullptr);
    CHECK_EQ(t->all("REQUIRES").size(), std::size_t{2});
    CHECK_EQ(t->first("badge")->key, "Badge");
    CHECK_EQ(canon_of("tech { Requires X requires Y }"), R"({"tech":{"requires":["X","Y"]}})");
}

TEST(quoted_strings_in_key_position_are_keys) {
    // systemnames.txt lists: the engine pairs them up like any KEY value step
    Document d = must_parse("names { \"Sol\" \"Alpha Centauri\" \"Vega\" \"Deneb\" }\n");
    const Node* n = d.root.first_block("names");
    CHECK_EQ(n->entries.size(), std::size_t{2});
    CHECK_EQ(n->entries[0].key, "Sol");
    CHECK(n->entries[0].key_quoted);
    CHECK_EQ(n->entries[0].value, "Alpha Centauri");
    CHECK(n->entries[0].quoted);
    CHECK_EQ(*n->first_value("vega"), "Deneb");
    CHECK_EQ(canon_of("n { \"x\" \"y\" }\n"), R"({"n":{"x":"y"}})");
}

TEST(odd_item_count_pairs_the_last_item_with_the_brace) {
    // The `}` becomes a value, the block stays open and swallows what follows
    // (hiver/liir lists in the shipped systemnames.txt).
    auto r = parse_blocks("n { \"A\" \"B\" \"C\" }\nm { \"D\" \"E\" }\n");
    CHECK(r.ok());
    CHECK_EQ(canon::blocks_json(r->root), R"({"n":{"a":"B","c":"}","m":{"d":"E"}}})");
    CHECK_EQ(r->warnings.size(), std::size_t{1});   // n never closes
}

TEST(lone_word_before_a_brace_takes_the_brace_as_its_value) {
    auto r = parse_blocks("flags { a 1 solo }\n");
    CHECK(r.ok());
    CHECK_EQ(canon::blocks_json(r->root), R"({"flags":{"a":1,"solo":"}"}})");
    CHECK_EQ(r->warnings.size(), std::size_t{1});   // flags is left open
    // "on off": 'on' takes 'off' as its value
    CHECK_EQ(canon_of("flags { on off }\n"), R"({"flags":{"on":"off"}})");
}

TEST(option_scalar_and_block_share_a_key) {
    Document d = must_parse(
        "shipsection { option DRV_A option { option T1 option T2 } option DRV_B }");
    const Node* s = d.root.first_block("shipsection");
    auto all = s->all("option");
    CHECK_EQ(all.size(), std::size_t{3});
    CHECK(all[0]->is_pair());
    CHECK(all[1]->is_block());
    CHECK(all[2]->is_pair());
    CHECK_EQ(s->values("option").size(), std::size_t{2});
    CHECK_EQ(s->blocks("option").size(), std::size_t{1});
    CHECK_EQ(s->blocks("option")[0]->values("option").size(), std::size_t{2});
    CHECK_EQ(canon_of("s { option A option { option T1 option T2 } option B }"),
             R"({"s":{"option":["A",{"option":["T1","T2"]},"B"]}})");
}

TEST(tokenizer_is_not_line_based) {
    // block opens on the same line as a preceding pair; several entries per
    // line; entries split across lines
    std::string s = canon_of(
        "bank { turretsize small   mount { node N }\n}\n"
        "weapon { name\n\nX }\n"
        "a\n{\nb\n1\n}\n");
    CHECK_EQ(s, R"({"a":{"b":1},"bank":{"mount":{"node":"N"},"turretsize":"small"},"weapon":{"name":"X"}})");
}

TEST(braces_are_not_delimiters) {
    // Only a whole-token { or } is a brace: glued to a word it is part of the word.
    // a{b -> key, 1}c -> value, then `2` is a trailing key without a value (dropped)
    auto r = parse_blocks("a{b 1}c 2\n");
    CHECK(r.ok());
    CHECK_EQ(canon::blocks_json(r->root), R"({"a{b":"1}c"})");
    CHECK_EQ(r->warnings.size(), std::size_t{1});
    auto w = parse_blocks("weapon{ name X }\n");
    CHECK_EQ(canon::blocks_json(w->root), R"({"weapon{":"name","x":"}"})");
    CHECK(w->root.first_block("weapon") == nullptr);
    CHECK_EQ(canon_of("weapon { name X }\n"), R"({"weapon":{"name":"X"}})");
}

TEST(quotes_do_not_split_barewords) {
    // abc"def" is one word; a quote only opens a string at the start of a token
    CHECK_EQ(canon_of("abc\"def\" 1\n"), R"({"abc\"def\"":1})");
    // a closing quote followed directly by text: the next token starts there
    CHECK_EQ(canon_of("k \"v\"x 1\n"), R"({"k":"v","x":1})");
}

TEST(quoted_braces_count_as_braces) {
    // Next() compares the stripped text, so "{" and "}" open and close blocks.
    CHECK_EQ(canon_of("a \"{\" x 1 \"}\"\n"), R"({"a":{"x":1}})");
}

TEST(whitespace_is_space_tab_cr_lf_only) {
    Document d = must_parse("a\v1 2\nb\f3 4\n");
    CHECK(d.root.first("a\v1") != nullptr);
    CHECK_EQ(*d.root.first_value("a\v1"), "2");
    CHECK_EQ(*d.root.first_value("b\f3"), "4");
}

TEST(eof_closes_open_blocks_lenient) {
    auto r = parse_blocks("shipsection {\n model M\n bank { mount { node N }\n");
    CHECK(r.ok());
    CHECK_EQ(r->warnings.size(), std::size_t{2});  // bank and shipsection both unclosed
    CHECK_EQ(r->warnings[0].line, 4);
    const Node* s = r->root.first_block("shipsection");
    CHECK(s != nullptr);
    CHECK_EQ(*s->first_value("model"), "M");
    CHECK_EQ(*s->first_block("bank")->first_block("mount")->first_value("node"), "N");
}

TEST(eof_inside_block_is_an_error_when_strict) {
    auto r = parse_blocks("shipsection {\n model M\n", strict_opts());
    CHECK(!r.ok());
    CHECK_EQ(r.error().line, 3);
}

TEST(closing_brace_touching_eof_is_a_plain_close) {
    auto r = parse_blocks("a { x 1 }");
    CHECK(r.ok());
    CHECK(r->warnings.empty());
    CHECK_EQ(canon::blocks_json(r->root), R"({"a":{"x":1}})");
    auto two = parse_blocks("a { b { x 1 } }");
    CHECK(two->warnings.empty());
    auto one_short = parse_blocks("a { b { x 1 }");
    CHECK_EQ(one_short->warnings.size(), std::size_t{1});   // a is left open
    CHECK(one_short->root.first_block("a")->first_block("b") != nullptr);
    auto stray = parse_blocks("a { x 1 }\n}");
    CHECK_EQ(stray->warnings.size(), std::size_t{1});       // stray at top level
    CHECK(parse_blocks("a { x 1 }", strict_opts()).ok());
}

TEST(stray_top_level_close_is_ignored_lenient) {
    auto r = parse_blocks("a { x 1 }\n}\nb { y 2 }\n");
    CHECK(r.ok());
    CHECK_EQ(r->warnings.size(), std::size_t{1});
    CHECK_EQ(r->warnings[0].line, 2);
    CHECK(r->root.first_block("a") != nullptr);
    CHECK(r->root.first_block("b") != nullptr);  // parsing continues after the stray brace
    CHECK_EQ(canon::blocks_json(r->root), R"({"a":{"x":1},"b":{"y":2}})");
}

TEST(stray_top_level_close_is_an_error_when_strict) {
    auto r = parse_blocks("a { x 1 }\n}\n", strict_opts());
    CHECK(!r.ok());
    CHECK_EQ(r.error().line, 2);
}

TEST(final_pair_without_trailing_newline_is_dropped) {
    auto r = parse_blocks("a 1\nb 2");
    CHECK(r.ok());
    CHECK_EQ(canon::blocks_json(r->root), R"({"a":1})");
    CHECK_EQ(r->warnings.size(), std::size_t{1});
    CHECK_EQ(r->warnings[0].line, 2);
    CHECK(!parse_blocks("a 1\nb 2", strict_opts()).ok());
    // any trailing whitespace saves it
    CHECK_EQ(canon_of("a 1\nb 2 "), R"({"a":1,"b":2})");
    CHECK_EQ(canon_of("a 1\nb 2\r\n"), R"({"a":1,"b":2})");
    // a quoted value whose closing quote is the last byte is dropped too
    CHECK_EQ(canon_of("a 1\nb \"2\""), R"({"a":1})");
    // inside a block the pair is dropped and the block is left open
    auto in_block = parse_blocks("s {\n a 1\n b 2");
    CHECK_EQ(canon::blocks_json(in_block->root), R"({"s":{"a":1}})");
    CHECK_EQ(in_block->warnings.size(), std::size_t{2});
}

TEST(trailing_key_without_value_is_dropped) {
    auto r = parse_blocks("a 1\nb\n");
    CHECK_EQ(canon::blocks_json(r->root), R"({"a":1})");
    CHECK_EQ(r->warnings.size(), std::size_t{1});
    auto t = parse_blocks("a 1\nb");   // key itself touches the end
    CHECK_EQ(canon::blocks_json(t->root), R"({"a":1})");
    CHECK_EQ(t->warnings.size(), std::size_t{1});
    auto h = parse_blocks("a 1\nb {");  // block header touching the end
    CHECK_EQ(canon::blocks_json(h->root), R"({"a":1})");
    CHECK_EQ(h->warnings.size(), std::size_t{1});
    CHECK_EQ(canon_of("a 1 trailing\n"), R"({"a":1})");
}

TEST(unterminated_quote_runs_to_end_of_input) {
    // The value swallows the rest of the file and then touches EOF -> dropped.
    auto r = parse_blocks("a \"never closed\n b 1\n");
    CHECK(r.ok());
    CHECK_EQ(canon::blocks_json(r->root), "{}");
    CHECK_EQ(r->warnings.size(), std::size_t{2});   // unterminated + dropped
    CHECK(!parse_blocks("a \"never closed", strict_opts()).ok());
    // the text is still visible through the tokenizer
    Script s("a \"x\ny");
    ScriptToken t;
    CHECK(s.next(t) == ReadStatus::AtEnd);
    CHECK(t.value_unterminated);
    CHECK_EQ(std::string(t.value), "x\ny");
}

TEST(open_brace_in_key_position_is_a_key) {
    auto r = parse_blocks("a 1\n{ b 2 }\n");
    CHECK(r.ok());
    CHECK_EQ(canon::blocks_json(r->root), R"({"2":"}","a":1,"{":"b"})");
    CHECK_EQ(r->warnings.size(), std::size_t{1});
    CHECK(!parse_blocks("a 1\n{ b 2 }\n", strict_opts()).ok());
}

TEST(crlf_and_line_numbers) {
    Document d = must_parse("a 1\r\nb 2\r\n\r\nc { d 3 }\r\n");
    CHECK_EQ(d.root.first("a")->line, 1);
    CHECK_EQ(d.root.first("b")->line, 2);
    CHECK_EQ(d.root.first("c")->line, 4);
    CHECK_EQ(d.root.first_block("c")->first("d")->line, 4);
    CHECK_EQ(*d.root.first_value("b"), "2");  // no '\r' leaks into values
}

TEST(multiline_quoted_string_counts_lines) {
    Document d = must_parse("a \"line one\nline two\"\nb 2\n");
    CHECK_EQ(*d.root.first_value("a"), "line one\nline two");
    CHECK_EQ(d.root.first("b")->line, 3);
}

TEST(empty_and_comment_only_input) {
    Document d = must_parse("");
    CHECK(d.root.entries.empty());
    CHECK(d.warnings.empty());
    Document e = must_parse("  // nothing here\n\n");
    CHECK(e.root.entries.empty());
    CHECK_EQ(canon_of(""), "{}");
    CHECK_EQ(canon_of("// no newline at all"), "{}");
}

TEST(top_level_pairs_and_blocks_mix) {
    // scenario .txt shape: top-level pairs plus repeated player{} blocks
    std::string s = canon_of("name @S\nnumplayers \"8\"\nplayer { recommended 1 }\nplayer { recommended 2 }");
    CHECK_EQ(s, R"({"name":"@S","numplayers":"8","player":[{"recommended":1},{"recommended":2}]})");
}

TEST(numbers_are_typed_only_in_canonical_form_and_only_when_bare) {
    // The Node keeps raw text; the canonical form types barewords like the oracle
    std::string s = canon_of("i 7 f .5 e 7e+8 t TRUE q \"8\" n -.8 d 5.\n");
    CHECK_EQ(s, "{\"d\":\"\\u00015\",\"e\":\"\\u0001700000000\",\"f\":\"\\u00010.5\","
                "\"i\":7,\"n\":\"\\u0001-0.80000000000000004\",\"q\":\"8\",\"t\":true}");
}

TEST(high_bytes_pass_through) {
    std::string src = "a \"caf\xe9\"\nb na\xefve\n";
    Document d = must_parse(src);
    CHECK_EQ(*d.root.first_value("a"), "caf\xe9");
    CHECK_EQ(*d.root.first_value("b"), "na\xefve");
    CHECK_EQ(canon::blocks_json(d.root), R"({"a":"caf\u00e9","b":"na\u00efve"})");
}

TEST(deep_nesting_returns_to_the_right_parent) {
    Document d = must_parse("a { b { c { d 1 } e 2 } f 3 } g 4\n");
    const Node* a = d.root.first_block("a");
    CHECK_EQ(*a->first_block("b")->first_block("c")->first_value("d"), "1");
    CHECK_EQ(*a->first_block("b")->first_value("e"), "2");
    CHECK_EQ(*a->first_value("f"), "3");
    CHECK_EQ(*d.root.first_value("g"), "4");
    CHECK_EQ(a->first_block("b")->line, 1);
}

TEST(lookup_misses_return_null_or_empty) {
    Document d = must_parse("a 1\n");
    CHECK(d.root.first("zzz") == nullptr);
    CHECK(d.root.first_value("zzz") == nullptr);
    CHECK(d.root.first_block("a") == nullptr);   // a is a pair, not a block
    CHECK(d.root.all("zzz").empty());
    CHECK(!d.root.has("zzz"));
    CHECK(d.root.has("A"));
}