// Unit tests for the brace-block reader. Every sample is hand-written. #include #include "canon.h" #include "mars/parse/blocks.h" #include "mars/parse/script.h" #include "test_main.h" using namespace mars::parse; namespace { Document must_parse(std::string_view text, Options o = {}) { auto r = parse_blocks(text, o); if (!r.ok()) { std::printf(" unexpected parse error line %d: %s\n", r.error().line, r.error().message.c_str()); CHECK(r.ok()); return Document{}; } return std::move(r).value(); } std::string canon_of(std::string_view text, Options o = {}) { return canon::blocks_json(must_parse(text, o).root); } Options strict_opts() { Options o; o.strict = true; return o; } } // namespace // ---- Script: the engine's pull tokenizer ------------------------------------- TEST(script_read_token_statuses) { Script s("a b\n"); Script::Raw t; CHECK(s.read_token(t) == ReadStatus::Ok); CHECK_EQ(std::string(t.text), "a"); CHECK(s.read_token(t) == ReadStatus::Ok); // "b" is followed by the newline CHECK_EQ(std::string(t.text), "b"); CHECK(s.read_token(t) == ReadStatus::NoInput); CHECK(t.text.empty()); Script e("a b"); CHECK(e.read_token(t) == ReadStatus::Ok); CHECK(e.read_token(t) == ReadStatus::AtEnd); // "b" touches the end of input CHECK_EQ(std::string(t.text), "b"); CHECK(e.read_token(t) == ReadStatus::NoInput); } TEST(script_next_returns_the_value_read_status) { Script s("k v\nname {\n}\nlast 1"); ScriptToken t; CHECK(s.next(t) == ReadStatus::Ok); CHECK(t.type == ScriptToken::Type::Pair); CHECK_EQ(std::string(t.key), "k"); CHECK_EQ(std::string(t.value), "v"); CHECK(s.next(t) == ReadStatus::Ok); CHECK(t.type == ScriptToken::Type::Open); CHECK_EQ(std::string(t.key), "name"); CHECK(s.next(t) == ReadStatus::Ok); CHECK(t.type == ScriptToken::Type::Close); CHECK(s.next(t) == ReadStatus::AtEnd); // the step was read but is incomplete CHECK_EQ(std::string(t.key), "last"); // ... its text is still available CHECK_EQ(std::string(t.value), "1"); CHECK(s.next(t) == ReadStatus::NoInput); } TEST(script_skip_block_counts_whole_brace_tokens_only) { Script s("x 1 a{ } b } after 2\n"); ScriptToken t; CHECK(s.next(t) == ReadStatus::Ok); // x 1 CHECK(s.skip_block(1) == ReadStatus::Ok); // a{ is a word; first whole } closes CHECK(s.next(t) == ReadStatus::Ok); CHECK_EQ(std::string(t.key), "b"); // b } -> pair b="}" CHECK_EQ(std::string(t.value), "}"); Script u("a { b {"); CHECK(u.skip_block(1) != ReadStatus::Ok); // input ends inside the block } TEST(script_token_text_is_capped_at_1023_bytes) { const std::string text = std::string(1100, 'x') + " 1\n"; Script s(text); ScriptToken t; CHECK(s.next(t) == ReadStatus::Ok); CHECK_EQ(t.key.size(), Script::kMaxTokenLength); CHECK_EQ(std::string(t.value), "1"); // the whole word was consumed } // ---- tree --------------------------------------------------------------------- TEST(basic_block_pairs_and_nesting) { Document d = must_parse( "weapon\n{\n\tname @WEAPON_X\n\tcost 50\n\tbank { mount { node N1 } }\n}\n"); CHECK_EQ(d.root.entries.size(), std::size_t{1}); const Node* w = d.root.first_block("weapon"); CHECK(w != nullptr); CHECK_EQ(w->name, "weapon"); CHECK_EQ(*w->first_value("name"), "@WEAPON_X"); // @TOKEN preserved verbatim CHECK_EQ(*w->first_value("cost"), "50"); // numbers kept as text const Node* mount = w->first_block("bank")->first_block("mount"); CHECK(mount != nullptr); CHECK_EQ(*mount->first_value("node"), "N1"); CHECK(d.warnings.empty()); CHECK_EQ(canon_of("weapon { name @X cost 50 bank { mount { node N1 } } }"), R"({"weapon":{"bank":{"mount":{"node":"N1"}},"cost":50,"name":"@X"}})"); } TEST(quoted_values_keep_spaces_backslashes_and_slashes) { Document d = must_parse( "a \"hello world\"\n" "b \"C:\\path\\to\\file.X\"\n" "c \"http://x//y\" // real comment\n" "d \"\"\n"); CHECK_EQ(*d.root.first_value("a"), "hello world"); CHECK_EQ(*d.root.first_value("b"), "C:\\path\\to\\file.X"); CHECK_EQ(*d.root.first_value("c"), "http://x//y"); CHECK_EQ(*d.root.first_value("d"), ""); CHECK(d.root.first("a")->quoted); CHECK(d.root.first("d")->quoted); CHECK(!d.root.first("a")->key_quoted); CHECK_EQ(d.root.entries.size(), std::size_t{4}); } TEST(single_quote_and_backtick_also_quote) { Document d = must_parse("a 'x y'\nb `p q`\nc \"it's\"\nd 'say \"hi\"'\n"); CHECK_EQ(*d.root.first_value("a"), "x y"); CHECK(d.root.first("a")->quoted); CHECK_EQ(*d.root.first_value("b"), "p q"); CHECK_EQ(*d.root.first_value("c"), "it's"); // only the opening character closes CHECK_EQ(*d.root.first_value("d"), "say \"hi\""); CHECK(d.warnings.empty()); } TEST(comments_are_stripped) { std::string s = canon_of("// leading\nx 1 // trailing\n//y 2\nz 3\n"); CHECK_EQ(s, R"({"x":1,"z":3})"); // The comment test is made on the extracted token: glued to a word it is // part of the word ... CHECK_EQ(canon_of("z 3//glued\n"), R"({"z":"3//glued"})"); CHECK_EQ(canon_of("z 3 //not glued\n"), R"({"z":3})"); // ... and a quoted token whose content starts with // is a comment too. CHECK_EQ(canon_of("\"// whole line\" junk\na 1\n"), R"({"a":1})"); // a closing quote glued to // : the quoted token ends, then the comment starts CHECK_EQ(canon_of("c \"0 0 0\"// \" 92 76 20\"\n"), R"({"c":"0 0 0"})"); // comment on the last line without a newline: the pair before it survives CHECK_EQ(canon_of("a 1 // end"), R"({"a":1})"); } TEST(repeated_keys_stay_in_order) { Document d = must_parse("t { requires A requires B other 1 requires C }"); auto all = d.root.first_block("t")->all("requires"); CHECK_EQ(all.size(), std::size_t{3}); CHECK_EQ(all[0]->value, "A"); CHECK_EQ(all[1]->value, "B"); CHECK_EQ(all[2]->value, "C"); CHECK_EQ(*d.root.first_block("t")->first_value("requires"), "A"); CHECK_EQ(canon_of("t { requires A requires B other 1 requires C }"), R"({"t":{"other":1,"requires":["A","B","C"]}})"); } TEST(keys_are_case_insensitive_but_case_is_kept) { Document d = must_parse("tech { Requires X requires Y Badge b }"); const Node* t = d.root.first_block("TECH"); CHECK(t != nullptr); CHECK_EQ(t->all("REQUIRES").size(), std::size_t{2}); CHECK_EQ(t->first("badge")->key, "Badge"); CHECK_EQ(canon_of("tech { Requires X requires Y }"), R"({"tech":{"requires":["X","Y"]}})"); } TEST(quoted_strings_in_key_position_are_keys) { // systemnames.txt lists: the engine pairs them up like any KEY value step Document d = must_parse("names { \"Sol\" \"Alpha Centauri\" \"Vega\" \"Deneb\" }\n"); const Node* n = d.root.first_block("names"); CHECK_EQ(n->entries.size(), std::size_t{2}); CHECK_EQ(n->entries[0].key, "Sol"); CHECK(n->entries[0].key_quoted); CHECK_EQ(n->entries[0].value, "Alpha Centauri"); CHECK(n->entries[0].quoted); CHECK_EQ(*n->first_value("vega"), "Deneb"); CHECK_EQ(canon_of("n { \"x\" \"y\" }\n"), R"({"n":{"x":"y"}})"); } TEST(odd_item_count_pairs_the_last_item_with_the_brace) { // The `}` becomes a value, the block stays open and swallows what follows // (hiver/liir lists in the shipped systemnames.txt). auto r = parse_blocks("n { \"A\" \"B\" \"C\" }\nm { \"D\" \"E\" }\n"); CHECK(r.ok()); CHECK_EQ(canon::blocks_json(r->root), R"({"n":{"a":"B","c":"}","m":{"d":"E"}}})"); CHECK_EQ(r->warnings.size(), std::size_t{1}); // n never closes } TEST(lone_word_before_a_brace_takes_the_brace_as_its_value) { auto r = parse_blocks("flags { a 1 solo }\n"); CHECK(r.ok()); CHECK_EQ(canon::blocks_json(r->root), R"({"flags":{"a":1,"solo":"}"}})"); CHECK_EQ(r->warnings.size(), std::size_t{1}); // flags is left open // "on off": 'on' takes 'off' as its value CHECK_EQ(canon_of("flags { on off }\n"), R"({"flags":{"on":"off"}})"); } TEST(option_scalar_and_block_share_a_key) { Document d = must_parse( "shipsection { option DRV_A option { option T1 option T2 } option DRV_B }"); const Node* s = d.root.first_block("shipsection"); auto all = s->all("option"); CHECK_EQ(all.size(), std::size_t{3}); CHECK(all[0]->is_pair()); CHECK(all[1]->is_block()); CHECK(all[2]->is_pair()); CHECK_EQ(s->values("option").size(), std::size_t{2}); CHECK_EQ(s->blocks("option").size(), std::size_t{1}); CHECK_EQ(s->blocks("option")[0]->values("option").size(), std::size_t{2}); CHECK_EQ(canon_of("s { option A option { option T1 option T2 } option B }"), R"({"s":{"option":["A",{"option":["T1","T2"]},"B"]}})"); } TEST(tokenizer_is_not_line_based) { // block opens on the same line as a preceding pair; several entries per // line; entries split across lines std::string s = canon_of( "bank { turretsize small mount { node N }\n}\n" "weapon { name\n\nX }\n" "a\n{\nb\n1\n}\n"); CHECK_EQ(s, R"({"a":{"b":1},"bank":{"mount":{"node":"N"},"turretsize":"small"},"weapon":{"name":"X"}})"); } TEST(braces_are_not_delimiters) { // Only a whole-token { or } is a brace: glued to a word it is part of the word. // a{b -> key, 1}c -> value, then `2` is a trailing key without a value (dropped) auto r = parse_blocks("a{b 1}c 2\n"); CHECK(r.ok()); CHECK_EQ(canon::blocks_json(r->root), R"({"a{b":"1}c"})"); CHECK_EQ(r->warnings.size(), std::size_t{1}); auto w = parse_blocks("weapon{ name X }\n"); CHECK_EQ(canon::blocks_json(w->root), R"({"weapon{":"name","x":"}"})"); CHECK(w->root.first_block("weapon") == nullptr); CHECK_EQ(canon_of("weapon { name X }\n"), R"({"weapon":{"name":"X"}})"); } TEST(quotes_do_not_split_barewords) { // abc"def" is one word; a quote only opens a string at the start of a token CHECK_EQ(canon_of("abc\"def\" 1\n"), R"({"abc\"def\"":1})"); // a closing quote followed directly by text: the next token starts there CHECK_EQ(canon_of("k \"v\"x 1\n"), R"({"k":"v","x":1})"); } TEST(quoted_braces_count_as_braces) { // Next() compares the stripped text, so "{" and "}" open and close blocks. CHECK_EQ(canon_of("a \"{\" x 1 \"}\"\n"), R"({"a":{"x":1}})"); } TEST(whitespace_is_space_tab_cr_lf_only) { Document d = must_parse("a\v1 2\nb\f3 4\n"); CHECK(d.root.first("a\v1") != nullptr); CHECK_EQ(*d.root.first_value("a\v1"), "2"); CHECK_EQ(*d.root.first_value("b\f3"), "4"); } TEST(eof_closes_open_blocks_lenient) { auto r = parse_blocks("shipsection {\n model M\n bank { mount { node N }\n"); CHECK(r.ok()); CHECK_EQ(r->warnings.size(), std::size_t{2}); // bank and shipsection both unclosed CHECK_EQ(r->warnings[0].line, 4); const Node* s = r->root.first_block("shipsection"); CHECK(s != nullptr); CHECK_EQ(*s->first_value("model"), "M"); CHECK_EQ(*s->first_block("bank")->first_block("mount")->first_value("node"), "N"); } TEST(eof_inside_block_is_an_error_when_strict) { auto r = parse_blocks("shipsection {\n model M\n", strict_opts()); CHECK(!r.ok()); CHECK_EQ(r.error().line, 3); } TEST(closing_brace_touching_eof_is_a_plain_close) { auto r = parse_blocks("a { x 1 }"); CHECK(r.ok()); CHECK(r->warnings.empty()); CHECK_EQ(canon::blocks_json(r->root), R"({"a":{"x":1}})"); auto two = parse_blocks("a { b { x 1 } }"); CHECK(two->warnings.empty()); auto one_short = parse_blocks("a { b { x 1 }"); CHECK_EQ(one_short->warnings.size(), std::size_t{1}); // a is left open CHECK(one_short->root.first_block("a")->first_block("b") != nullptr); auto stray = parse_blocks("a { x 1 }\n}"); CHECK_EQ(stray->warnings.size(), std::size_t{1}); // stray at top level CHECK(parse_blocks("a { x 1 }", strict_opts()).ok()); } TEST(stray_top_level_close_is_ignored_lenient) { auto r = parse_blocks("a { x 1 }\n}\nb { y 2 }\n"); CHECK(r.ok()); CHECK_EQ(r->warnings.size(), std::size_t{1}); CHECK_EQ(r->warnings[0].line, 2); CHECK(r->root.first_block("a") != nullptr); CHECK(r->root.first_block("b") != nullptr); // parsing continues after the stray brace CHECK_EQ(canon::blocks_json(r->root), R"({"a":{"x":1},"b":{"y":2}})"); } TEST(stray_top_level_close_is_an_error_when_strict) { auto r = parse_blocks("a { x 1 }\n}\n", strict_opts()); CHECK(!r.ok()); CHECK_EQ(r.error().line, 2); } TEST(final_pair_without_trailing_newline_is_dropped) { auto r = parse_blocks("a 1\nb 2"); CHECK(r.ok()); CHECK_EQ(canon::blocks_json(r->root), R"({"a":1})"); CHECK_EQ(r->warnings.size(), std::size_t{1}); CHECK_EQ(r->warnings[0].line, 2); CHECK(!parse_blocks("a 1\nb 2", strict_opts()).ok()); // any trailing whitespace saves it CHECK_EQ(canon_of("a 1\nb 2 "), R"({"a":1,"b":2})"); CHECK_EQ(canon_of("a 1\nb 2\r\n"), R"({"a":1,"b":2})"); // a quoted value whose closing quote is the last byte is dropped too CHECK_EQ(canon_of("a 1\nb \"2\""), R"({"a":1})"); // inside a block the pair is dropped and the block is left open auto in_block = parse_blocks("s {\n a 1\n b 2"); CHECK_EQ(canon::blocks_json(in_block->root), R"({"s":{"a":1}})"); CHECK_EQ(in_block->warnings.size(), std::size_t{2}); } TEST(trailing_key_without_value_is_dropped) { auto r = parse_blocks("a 1\nb\n"); CHECK_EQ(canon::blocks_json(r->root), R"({"a":1})"); CHECK_EQ(r->warnings.size(), std::size_t{1}); auto t = parse_blocks("a 1\nb"); // key itself touches the end CHECK_EQ(canon::blocks_json(t->root), R"({"a":1})"); CHECK_EQ(t->warnings.size(), std::size_t{1}); auto h = parse_blocks("a 1\nb {"); // block header touching the end CHECK_EQ(canon::blocks_json(h->root), R"({"a":1})"); CHECK_EQ(h->warnings.size(), std::size_t{1}); CHECK_EQ(canon_of("a 1 trailing\n"), R"({"a":1})"); } TEST(unterminated_quote_runs_to_end_of_input) { // The value swallows the rest of the file and then touches EOF -> dropped. auto r = parse_blocks("a \"never closed\n b 1\n"); CHECK(r.ok()); CHECK_EQ(canon::blocks_json(r->root), "{}"); CHECK_EQ(r->warnings.size(), std::size_t{2}); // unterminated + dropped CHECK(!parse_blocks("a \"never closed", strict_opts()).ok()); // the text is still visible through the tokenizer Script s("a \"x\ny"); ScriptToken t; CHECK(s.next(t) == ReadStatus::AtEnd); CHECK(t.value_unterminated); CHECK_EQ(std::string(t.value), "x\ny"); } TEST(open_brace_in_key_position_is_a_key) { auto r = parse_blocks("a 1\n{ b 2 }\n"); CHECK(r.ok()); CHECK_EQ(canon::blocks_json(r->root), R"({"2":"}","a":1,"{":"b"})"); CHECK_EQ(r->warnings.size(), std::size_t{1}); CHECK(!parse_blocks("a 1\n{ b 2 }\n", strict_opts()).ok()); } TEST(crlf_and_line_numbers) { Document d = must_parse("a 1\r\nb 2\r\n\r\nc { d 3 }\r\n"); CHECK_EQ(d.root.first("a")->line, 1); CHECK_EQ(d.root.first("b")->line, 2); CHECK_EQ(d.root.first("c")->line, 4); CHECK_EQ(d.root.first_block("c")->first("d")->line, 4); CHECK_EQ(*d.root.first_value("b"), "2"); // no '\r' leaks into values } TEST(multiline_quoted_string_counts_lines) { Document d = must_parse("a \"line one\nline two\"\nb 2\n"); CHECK_EQ(*d.root.first_value("a"), "line one\nline two"); CHECK_EQ(d.root.first("b")->line, 3); } TEST(empty_and_comment_only_input) { Document d = must_parse(""); CHECK(d.root.entries.empty()); CHECK(d.warnings.empty()); Document e = must_parse(" // nothing here\n\n"); CHECK(e.root.entries.empty()); CHECK_EQ(canon_of(""), "{}"); CHECK_EQ(canon_of("// no newline at all"), "{}"); } TEST(top_level_pairs_and_blocks_mix) { // scenario .txt shape: top-level pairs plus repeated player{} blocks std::string s = canon_of("name @S\nnumplayers \"8\"\nplayer { recommended 1 }\nplayer { recommended 2 }"); CHECK_EQ(s, R"({"name":"@S","numplayers":"8","player":[{"recommended":1},{"recommended":2}]})"); } TEST(numbers_are_typed_only_in_canonical_form_and_only_when_bare) { // The Node keeps raw text; the canonical form types barewords like the oracle std::string s = canon_of("i 7 f .5 e 7e+8 t TRUE q \"8\" n -.8 d 5.\n"); CHECK_EQ(s, "{\"d\":\"\\u00015\",\"e\":\"\\u0001700000000\",\"f\":\"\\u00010.5\"," "\"i\":7,\"n\":\"\\u0001-0.80000000000000004\",\"q\":\"8\",\"t\":true}"); } TEST(high_bytes_pass_through) { std::string src = "a \"caf\xe9\"\nb na\xefve\n"; Document d = must_parse(src); CHECK_EQ(*d.root.first_value("a"), "caf\xe9"); CHECK_EQ(*d.root.first_value("b"), "na\xefve"); CHECK_EQ(canon::blocks_json(d.root), R"({"a":"caf\u00e9","b":"na\u00efve"})"); } TEST(deep_nesting_returns_to_the_right_parent) { Document d = must_parse("a { b { c { d 1 } e 2 } f 3 } g 4\n"); const Node* a = d.root.first_block("a"); CHECK_EQ(*a->first_block("b")->first_block("c")->first_value("d"), "1"); CHECK_EQ(*a->first_block("b")->first_value("e"), "2"); CHECK_EQ(*a->first_value("f"), "3"); CHECK_EQ(*d.root.first_value("g"), "4"); CHECK_EQ(a->first_block("b")->line, 1); } TEST(lookup_misses_return_null_or_empty) { Document d = must_parse("a 1\n"); CHECK(d.root.first("zzz") == nullptr); CHECK(d.root.first_value("zzz") == nullptr); CHECK(d.root.first_block("a") == nullptr); // a is a pair, not a block CHECK(d.root.all("zzz").empty()); CHECK(!d.root.has("zzz")); CHECK(d.root.has("A")); }