// Unit tests for mars::text -- hand-written minimal samples covering every // quirk the shipped data exercises (see docs/mars-text.md). No game data. #include #include #include "mars/text/csv.h" #include "mars/text/flat_kv.h" #include "mars/text/manifest.h" #include "mars/text/value.h" using namespace mars::text; static int g_failures = 0; static int g_checks = 0; #define CHECK(cond) \ do { \ ++g_checks; \ if (!(cond)) { \ ++g_failures; \ std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ } \ } while (0) #define CHECK_EQ(a, b) CHECK((a) == (b)) // ---- value --------------------------------------------------------------- static void test_classify() { CHECK(classify("12") == ValueKind::Int); CHECK(classify("-7") == ValueKind::Int); CHECK(classify("+3") == ValueKind::Int); CHECK(classify("0.05") == ValueKind::Float); CHECK(classify(".5") == ValueKind::Float); CHECK(classify("-.8") == ValueKind::Float); CHECK(classify("1.") == ValueKind::Float); CHECK(classify("7e+8") == ValueKind::Float); CHECK(classify("1E5") == ValueKind::Float); CHECK(classify("1e") == ValueKind::Bareword); CHECK(classify(".") == ValueKind::Bareword); CHECK(classify("-") == ValueKind::Bareword); CHECK(classify("0x10") == ValueKind::Bareword); CHECK(classify("inf") == ValueKind::Bareword); CHECK(classify("nan") == ValueKind::Bareword); CHECK(classify("1,5") == ValueKind::Bareword); CHECK(classify("true") == ValueKind::Bool); CHECK(classify("FALSE") == ValueKind::Bool); CHECK(classify("True") == ValueKind::Bool); CHECK(classify("yes") == ValueKind::Bareword); CHECK(classify("") == ValueKind::Bareword); CHECK_EQ(parse_int("42").value(), 42); CHECK_EQ(parse_int("-42").value(), -42); CHECK_EQ(parse_int("+9").value(), 9); CHECK(!parse_int("4.0")); CHECK(!parse_int("99999999999999999999")); // does not fit in 64 bits CHECK_EQ(parse_int("-9223372036854775808").value(), INT64_MIN); CHECK_EQ(parse_float("0.05").value(), 0.05); CHECK_EQ(parse_float(".5").value(), 0.5); CHECK_EQ(parse_float("-.8").value(), -0.8); CHECK_EQ(parse_float("1.").value(), 1.0); CHECK_EQ(parse_float("7e+8").value(), 7e8); CHECK_EQ(parse_float("3").value(), 3.0); // int shape converts too CHECK(!parse_float("inf")); CHECK(!parse_float("1e")); CHECK_EQ(parse_bool("TRUE").value(), true); CHECK_EQ(parse_bool("false").value(), false); CHECK(!parse_bool("0")); Token quoted{"8", true}; CHECK(quoted.kind() == ValueKind::String); CHECK(!quoted.as_int()); Token bare{"8", false}; CHECK(bare.kind() == ValueKind::Int); CHECK_EQ(bare.as_int().value(), 8); CHECK_EQ(bare.as_float().value(), 8.0); } static void test_color() { auto c = parse_color("255\t177\t\t39"); // tabs inside the quotes, as in globals.txt CHECK(c && c->components == 3 && c->r == 255 && c->g == 177 && c->b == 39 && c->a == 1); auto c4 = parse_color(" 0 0.5 1 0.25 "); CHECK(c4 && c4->components == 4 && c4->g == 0.5 && c4->a == 0.25); CHECK(!parse_color("1 2")); CHECK(!parse_color("1 2 3 4 5")); CHECK(!parse_color("red green blue")); CHECK(!parse_color("")); } // ---- flat kv ------------------------------------------------------------- static void test_strip_comment_and_tokens() { CHECK_EQ(strip_comment("A 1 // c"), std::string_view("A 1 ")); CHECK_EQ(strip_comment("A \"x // y\" // c"), std::string_view("A \"x // y\" ")); CHECK_EQ(strip_comment("A \"unclosed // not a comment"), std::string_view("A \"unclosed // not a comment")); CHECK_EQ(strip_comment("A / 1"), std::string_view("A / 1")); CHECK_EQ(strip_comment("//"), std::string_view("")); auto t = split_tokens(" a\t\"b c\" \"\" d\"e f\" g"); CHECK_EQ(t.size(), 6u); CHECK(t[0].text == "a" && !t[0].quoted); CHECK(t[1].text == "b c" && t[1].quoted); CHECK(t[2].text == "" && t[2].quoted); // empty quoted token survives CHECK(t[3].text == "d\"e" && !t[3].quoted); // quote mid-bareword is literal CHECK(t[4].text == "f\"" && !t[4].quoted); CHECK(t[5].text == "g" && !t[5].quoted); auto u = split_tokens("\"open x"); CHECK_EQ(u.size(), 2u); CHECK(u[0].text == "\"open" && !u[0].quoted); // unpaired quote: bareword CHECK(u[1].text == "x"); CHECK(split_tokens(" ").empty()); } static void test_flat_kv() { const char* text = "// header comment\r\n" "\r\n" "SLAVES_DEATH_RATE 0.05\r\n" "Max_Ships\t\t12 // trailing comment\n" "TITLE \"Sword // of the Stars\"\n" "EMPTY \"\"\n" "COLOR \"255\t177\t\t39\"\n" "FLAG true\n" "path \"Data\\\\x.txt\"\n" "slaves_death_rate 0.10\n" "ENDGAME \"0 0 0\"// \" 92 76 20\"\n" "LAST 7\n"; auto r = parse_flat_kv(text); const FlatKV& kv = r.value; CHECK_EQ(kv.size(), 10u); // every pair in file order, the duplicate included // typed access, case-insensitive lookup, original spelling preserved CHECK_EQ(kv.get_int("max_ships").value(), 12); CHECK_EQ(kv.find("MAX_SHIPS")->key, std::string("Max_Ships")); CHECK_EQ(kv.find("MAX_SHIPS")->line, 4); CHECK(!kv.get_int("TITLE")); CHECK_EQ(kv.get_string("TITLE").value(), std::string_view("Sword // of the Stars")); CHECK(kv.find("TITLE")->value.quoted); CHECK_EQ(kv.get_string("EMPTY").value(), std::string_view("")); auto c = kv.get_color("COLOR"); CHECK(c && c->r == 255 && c->g == 177 && c->b == 39); CHECK_EQ(kv.get_bool("flag").value(), true); CHECK_EQ(kv.get_string("path").value(), std::string_view("Data\\\\x.txt")); // no escape processing CHECK_EQ(kv.get_string("ENDGAME").value(), std::string_view("0 0 0")); // closing quote glued to // CHECK_EQ(kv.get_int("LAST").value(), 7); CHECK(!kv.has("missing")); CHECK(!kv.get_int("missing")); // duplicates: FIRST occurrence wins (engine erases a key once consumed); // the later line is reported and stays visible in entries()/duplicates() CHECK_EQ(kv.get_float("slaves_death_rate").value(), 0.05); CHECK_EQ(kv.find("SLAVES_DEATH_RATE")->line, 3); auto dups = kv.duplicates(); CHECK_EQ(dups.size(), 1u); CHECK(dups[0].first == "slaves_death_rate"); CHECK((dups[0].second == std::vector{3, 10})); CHECK_EQ(r.problems.size(), 1u); CHECK(r.problems[0].kind == Problem::Kind::DuplicateKey && r.problems[0].line == 10); // key case only differs: the same key to the engine (_stricmp) auto r2 = parse_flat_kv("Foo 1\nFOO 2\n"); CHECK_EQ(r2.value.get_int("foo").value(), 1); CHECK_EQ(r2.value.duplicates().size(), 1u); CHECK(r2.problems.size() == 1 && r2.problems[0].kind == Problem::Kind::DuplicateKey); // empty / comment-only input CHECK_EQ(parse_flat_kv("").value.size(), 0u); CHECK_EQ(parse_flat_kv("// nothing\n\n \n").value.size(), 0u); CHECK(parse_flat_kv("// no newline").ok()); // line numbers count '\n' only (mixed terminators) auto r3 = parse_flat_kv("A 1\rB 2\r\nC 3\nD 4\n"); CHECK_EQ(r3.value.find("B")->line, 1); CHECK_EQ(r3.value.find("D")->line, 3); // high bytes pass through untouched auto r4 = parse_flat_kv("K \"caf\xe9 \x93q\x94\"\n"); CHECK_EQ(r4.value.get_string("K").value(), std::string_view("caf\xe9 \x93q\x94")); } // The loader is the engine's token stepper, not a line reader. static void test_flat_kv_engine_steps() { // one value token per key: extra tokens start the next pair auto r = parse_flat_kv("LIST 1 2 three\n"); CHECK_EQ(r.value.size(), 2u); CHECK_EQ(r.value.get_int("LIST").value(), 1); CHECK_EQ(r.value.get_string("2").value(), std::string_view("three")); // a key alone on a line takes the next line's key as its value auto k = parse_flat_kv("A\nB 2\nC 3\n"); CHECK_EQ(k.value.get_string("A").value(), std::string_view("B")); CHECK_EQ(k.value.get_string("2").value(), std::string_view("C")); CHECK(!k.value.has("B")); CHECK(k.problems.size() == 1 && k.problems[0].kind == Problem::Kind::DroppedTrailingPair); // "3" // braces are not delimiters auto b = parse_flat_kv("A{ 1\n"); CHECK(b.value.has("A{") && !b.value.has("A")); // the final pair is dropped when its value touches the end of input auto d = parse_flat_kv("A 1\nB 2"); CHECK(d.value.has("A") && !d.value.has("B")); CHECK_EQ(d.value.size(), 1u); CHECK(d.problems.size() == 1 && d.problems[0].kind == Problem::Kind::DroppedTrailingPair && d.problems[0].line == 2); CHECK(parse_flat_kv("A 1\nB \"2\"").problems.size() == 1); // closing quote as last byte: same CHECK(parse_flat_kv("A 1\nB 2 ").value.has("B")); // any trailing whitespace saves it CHECK(parse_flat_kv("A 1\nB 2 // c").value.has("B")); // ... or a comment auto lone = parse_flat_kv("A 1\nB\n"); CHECK(!lone.value.has("B") && lone.problems.size() == 1); auto lone2 = parse_flat_kv("A 1\nB"); CHECK(!lone2.value.has("B") && lone2.problems.size() == 1); // quotes: ", ' and backtick, no escapes; a quote inside a word is literal auto q = parse_flat_kv("A 'x y'\nB `p q`\nC it's\nD \"say \\\"hi\" 5\n"); CHECK_EQ(q.value.get_string("A").value(), std::string_view("x y")); CHECK(q.value.find("A")->value.quoted); CHECK_EQ(q.value.get_string("B").value(), std::string_view("p q")); CHECK_EQ(q.value.get_string("C").value(), std::string_view("it's")); CHECK_EQ(q.value.get_string("D").value(), std::string_view("say \\")); // backslash does not escape CHECK_EQ(q.value.get_int("hi\"").value(), 5); // the rest is a new key CHECK(q.ok()); // an unterminated quote runs to the end of input (and so the pair is dropped) auto u = parse_flat_kv("A \"never\nB 2\n"); CHECK_EQ(u.value.size(), 0u); CHECK_EQ(u.problems.size(), 2u); CHECK(u.problems[0].kind == Problem::Kind::UnterminatedQuote); CHECK(u.problems[1].kind == Problem::Kind::DroppedTrailingPair); // comments: the test is on the extracted token auto c = parse_flat_kv("A 3//x\n\"// whole line\" junk\nB 2\n"); CHECK_EQ(c.value.get_string("A").value(), std::string_view("3//x")); CHECK(c.value.find("A")->value.kind() == ValueKind::Bareword); CHECK_EQ(c.value.get_int("B").value(), 2); CHECK_EQ(c.value.size(), 2u); // a block is skipped; a stray } is ignored auto blk = parse_flat_kv("A 1\nblk {\n X 2\n sub { Y 3 }\n}\n}\nB 4\n"); CHECK_EQ(blk.value.size(), 2u); CHECK(blk.value.has("A") && blk.value.has("B") && !blk.value.has("X")); CHECK(blk.problems.size() == 1 && blk.problems[0].kind == Problem::Kind::SkippedBlock); // tokens are capped at 1023 bytes std::string longkey(1100, 'k'); auto cap = parse_flat_kv(longkey + " 1\n"); CHECK_EQ(cap.value.entries()[0].key.size(), 1023u); CHECK(cap.value.has(longkey.substr(0, 1023))); // whitespace is space, tab, CR, LF only auto ws = parse_flat_kv("A\v1 2\n"); CHECK(ws.value.has("A\v1")); } static void test_rows() { const char* text = "// size weapon-size class health \"model\"\r\n" "\r\n" "small\ttiny\tstandard 20 360 1.00 \"turret_pd.x\"\r\n" "large small standard 800 120 1.00 \"turret with space.x\" // c\n" "\tHiver\t\tBADGE_HIVER_GW100\t\ttrue\n"; auto r = parse_rows(text); CHECK(r.ok()); CHECK_EQ(r.value.size(), 3u); CHECK_EQ(r.value[0].line, 3); CHECK_EQ(r.value[0].tokens.size(), 7u); CHECK_EQ(r.value[0].tokens[3].as_int().value(), 20); CHECK(r.value[0].tokens[5].kind() == ValueKind::Float); CHECK(r.value[0].tokens[6].quoted && r.value[0].tokens[6].text == "turret_pd.x"); CHECK_EQ(r.value[1].tokens[6].text, std::string("turret with space.x")); CHECK_EQ(r.value[2].tokens.size(), 3u); CHECK_EQ(r.value[2].tokens[2].as_bool().value(), true); } // ---- manifest ------------------------------------------------------------ static void test_manifest() { const char* text = "// Only add to this list: always use new IDs.\r\n" "\r\n" "1 can_am.weapon\r\n" "2\tDEWar.SHIPSECTION\r\n" "// DELETED - 36\r\n" "//deleted-58\n" " // Deleted - 59 \n" "3 x.weapon // trailing comment is not accepted\n" "4 two words\n" "not_an_id y.weapon\n" "1 dup.weapon\n" "7 z.weapon\n" "// DELETED - 7\n"; auto r = parse_manifest(text); const Manifest& m = r.value; CHECK_EQ(m.entries().size(), 4u); CHECK((m.deleted() == std::vector{36, 58, 59, 7})); CHECK_EQ(m.entries()[1].line, 4); CHECK_EQ(m.name_of(2).value(), std::string_view("DEWar.SHIPSECTION")); CHECK_EQ(m.id_of("dewar.shipsection").value(), 2); // case-folded lookup CHECK_EQ(m.id_of("CAN_AM.WEAPON").value(), 1); CHECK(!m.id_of("nope")); CHECK(!m.name_of(99)); CHECK(m.is_deleted(36) && !m.is_deleted(1)); CHECK_EQ(m.name_of(1).value(), std::string_view("dup.weapon")); // last assignment wins CHECK_EQ(r.problems.size(), 5u); CHECK(r.problems[0].kind == Problem::Kind::Unrecognised && r.problems[0].line == 8); CHECK(r.problems[1].kind == Problem::Kind::Unrecognised && r.problems[1].line == 9); CHECK(r.problems[2].kind == Problem::Kind::Unrecognised && r.problems[2].line == 10); CHECK(r.problems[3].kind == Problem::Kind::DuplicateId && r.problems[3].line == 11 && r.problems[3].id == 1); CHECK(r.problems[4].kind == Problem::Kind::DeletedAndAssigned && r.problems[4].id == 7); CHECK(parse_manifest("").ok()); CHECK(parse_manifest("").value.entries().empty()); } // ---- csv ----------------------------------------------------------------- static void test_csv_records() { // raw split: quoting, escapes, terminators, multi-line cells auto r = split_csv_records("a,b\r\n\r\n\"x,y\",\"say \"\"hi\"\"\"\n\"multi\r\nline\",end\r\nlast,"); CHECK(r.ok()); const auto& rows = r.value; CHECK_EQ(rows.size(), 5u); CHECK((rows[0].cells == std::vector{"a", "b"})); CHECK(rows[1].cells.empty()); CHECK((rows[2].cells == std::vector{"x,y", "say \"hi\""})); CHECK((rows[3].cells == std::vector{"multi\r\nline", "end"})); CHECK_EQ(rows[3].line, 4); CHECK((rows[4].cells == std::vector{"last", ""})); // trailing comma, no final newline CHECK_EQ(rows[4].line, 6); // quote is literal unless it is the first byte of the cell auto q = split_csv_records("KEY\",text\n \"Effects/x.effect\",1\n\"tail\"junk,2\n"); CHECK((q.value[0].cells == std::vector{"KEY\"", "text"})); CHECK((q.value[1].cells == std::vector{" \"Effects/x.effect\"", "1"})); CHECK((q.value[2].cells == std::vector{"tailjunk", "2"})); // lenient, like the reference // lone CR terminators, and CR LF sequences counted once auto cr = split_csv_records("a\rb\r\nc"); CHECK_EQ(cr.value.size(), 3u); CHECK_EQ(cr.value[2].line, 3); // unterminated quoted cell: content kept, problem reported auto u = split_csv_records("a,\"never closed\nmore"); CHECK_EQ(u.value.size(), 1u); CHECK((u.value[0].cells == std::vector{"a", "never closed\nmore"})); CHECK_EQ(u.problems.size(), 1u); CHECK(u.problems[0].kind == Problem::Kind::UnterminatedQuotedCell); CHECK(split_csv_records("").value.empty()); CHECK_EQ(split_csv_records("\n").value.size(), 1u); } static void test_csv() { const char* text = "# ,,\r\n" "#\r\n" "# tech: tech label\r\n" "\r\n" " \r\n" ",,\r\n" "// slash comment, with comma\r\n" " WEP_Laser , 10, 20 \r\n" " # indented comment,1,2\r\n" "WEP_Gauss,\"1,5\",\"a \"\"b\"\"\"\r\n" "\"multi\r\nline\",x,y\r\n"; auto r = parse_csv(text); CHECK(r.ok()); const Csv& csv = r.value; CHECK(csv.header.has_value()); CHECK((csv.header.value() == std::vector{"tech", "human-pri", "hiver-pri"})); CHECK_EQ(csv.size(), 3u); CHECK((csv.rows[0].cells == std::vector{"WEP_Laser", "10", "20"})); // stripped CHECK_EQ(csv.rows[0].line, 8); CHECK((csv.rows[1].cells == std::vector{"WEP_Gauss", "1,5", "a \"b\""})); CHECK((csv.rows[2].cells == std::vector{"multi\r\nline", "x", "y"})); CHECK_EQ(csv.rows[2].line, 11); // header variants auto h1 = parse_csv("\"# species\",\"event\",\"message\"\n,,\n\"Human\",\"E\",\"M\"\n"); CHECK((h1.value.header.value() == std::vector{"species", "event", "message"})); CHECK_EQ(h1.value.size(), 1u); auto h2 = parse_csv("#Key+A955,String,Size,Notes\nK,v,,\n"); CHECK((h2.value.header.value() == std::vector{"Key+A955", "String", "Size", "Notes"})); CHECK((h2.value.rows[0].cells == std::vector{"K", "v", "", ""})); auto h3 = parse_csv("# just a comment\nA,B\n# ,\n"); CHECK(!h3.value.header.has_value()); // single-cell '#' rows are not schemas; data ends the search CHECK_EQ(h3.value.size(), 1u); auto h4 = parse_csv("A\nB\n"); CHECK(!h4.value.header.has_value()); CHECK_EQ(h4.value.size(), 2u); CHECK_EQ(parse_csv("").value.size(), 0u); CHECK_EQ(parse_csv("#a,b\n#c\n").value.size(), 0u); // comment-only template // bytes >= 0x80 pass through auto hb = parse_csv("K,caf\xe9 \x85\n"); CHECK_EQ(hb.value.rows[0].cells[1], std::string("caf\xe9 \x85")); } int main() { test_classify(); test_color(); test_strip_comment_and_tokens(); test_flat_kv(); test_flat_kv_engine_steps(); test_rows(); test_manifest(); test_csv_records(); test_csv(); std::printf("mars_text unit tests: %d checks, %d failures\n", g_checks, g_failures); return g_failures == 0 ? 0 : 1; }