"""verify.py -- parse every shipped SOTS1 data file, cross-link the catalogs, and emit normalized JSON artifacts. usage: python3 verify.py Prints a markdown report to stdout; writes to : tech_tree.json weapons.json shipsections.json strings.json schema_stats.json crosslink.json tech_tree.dot """ from __future__ import annotations import collections import json import os import re import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import effect_txt import flat_kv import manifest import mars_data from mars_data import get_list RACES = ["Human", "Zuul", "Hiver", "Tarkas", "Liir", "Morrigi"] PLAYABLE = RACES ALL_RACE_DIRS = RACES + ["_NPC"] BRACE_TXT = {"Data/tutorial.txt", "Data/credits.txt", "Data/Strategy/systemnames.txt", "Data/Combat/ctechvars.txt", "Data/Combat/shipai.txt", "Models/Skysphere/skydefs.txt", "Models/Skysphere/NodeSpace-skydefs.txt"} ROWS_TXT = {"Weapons/_turrets.txt", "Weapons/_defaultweapons.txt", "Data/Combat/damfx.txt", "Data/Combat/damfx_levels.txt", "Data/Strategy/playercolors.txt", "Badges/BadgeTable.txt", "Avatars/AvatarTable.txt", "GUI/WeaponIconPlacements.txt"} PROSE = {"Locale/EN/ChatTrans.txt"} def kind_of(rel: str) -> str: ext = rel.rsplit(".", 1)[-1].lower() base = os.path.basename(rel) if ext in ("weapon", "shipsection", "tech", "combat", "def", "script"): return "brace:" + ext if ext == "effect": return "effect" if ext == "csv": return "csv" if ext in ("fx", "fxh"): return "hlsl" if ext == "txt": if base in ("_weapons.txt", "_shipsections.txt"): return "manifest" if rel.startswith("Scenarios/") or rel in BRACE_TXT: return "brace:txt" if rel in ROWS_TXT: return "rows" if rel.startswith("Locale/EN/Desc") or rel in PROSE: return "prose" return "kv" return "other" def walk(root): for dp, _, fn in os.walk(root): for f in sorted(fn): p = os.path.join(dp, f) yield p, os.path.relpath(p, root).replace(os.sep, "/") # --- schema stats ------------------------------------------------------------ def schema_walk(node, path, stats): """Count key occurrences per block path, and which keys are blocks.""" st = stats.setdefault(path, {"blocks": 0, "keys": collections.Counter(), "sub": collections.Counter()}) st["blocks"] += 1 for k, v in node.items(): vals = v if isinstance(v, list) else [v] for x in vals: if isinstance(x, dict): st["sub"][k] += 1 schema_walk(x, path + "." + k, stats) else: st["keys"][k] += 1 # --- tech tree --------------------------------------------------------------- _RP_RE = re.compile(r"^RP:(\d+)$", re.I) _PCT_RE = re.compile(r"^(\w+):(\d+)$") def parse_allows(s: str): toks = s.split() child = toks[0] rp = None pct = {} extra = [] for t in toks[1:]: m = _RP_RE.match(t) if m: rp = int(m.group(1)) continue m = _PCT_RE.match(t) if m and m.group(1) in RACES: pct[m.group(1)] = int(m.group(2)) continue extra.append(t) return child, rp, pct, extra def main(root: str, out: str) -> int: os.makedirs(out, exist_ok=True) rep = [] P = rep.append # ---- 1. parse everything --------------------------------------------- ok = collections.Counter() fail = collections.Counter() fails = [] warns = [] parsed = {} # rel -> object for p, rel in walk(root): k = kind_of(rel) try: if k.startswith("brace"): w = [] obj = mars_data.parse_file(p, warnings=w) # strict re-parse to record the recovery if w: warns.append((rel, w)) elif k == "effect": obj = effect_txt.parse_file(p) elif k == "csv": obj = manifest.parse_csv_file(p) elif k == "manifest": obj = manifest.parse_manifest_file(p) if obj.problems: raise ValueError("; ".join(obj.problems)) elif k == "rows": obj = flat_kv.parse_rows_file(p) elif k == "kv": txt = manifest.read_text(p) w = [] obj = flat_kv.parse_kv(txt, warnings=w) # engine rules: first occurrence wins if w: warns.append((rel, w)) else: continue parsed[rel] = obj ok[k] += 1 except Exception as e: # noqa: BLE001 fail[k] += 1 fails.append((rel, repr(e))) P("## Parse results") P("") P("| reader | file kind | files | parsed | failed |") P("|---|---|---|---|---|") reader_of = {"brace": "mars_data", "effect": "effect_txt", "csv": "manifest.parse_csv", "manifest": "manifest.parse_manifest", "rows": "flat_kv.parse_rows", "kv": "flat_kv.parse_kv"} for k in sorted(set(ok) | set(fail)): P(f"| {reader_of[k.split(':')[0]]} | {k} | {ok[k] + fail[k]} | {ok[k]} | {fail[k]} |") P("") P(f"Total: {sum(ok.values())} parsed, {sum(fail.values())} failed " f"(skipped: {sum(1 for _, r in walk(root) if kind_of(r) in ('hlsl', 'prose', 'other'))} " f"HLSL/prose files that are not data).") if fails: P("") P("Failures:") for rel, e in fails: P(f"- `{rel}`: {e}") if warns: P("") P("Engine tolerances taken (what the original loader does with these files; strict=True would reject):") for rel, w in warns: P(f"- `{rel}`: {'; '.join(w)}") P("") # ---- 2. schema stats --------------------------------------------------- stats = {} for rel, obj in parsed.items(): k = kind_of(rel) if k in ("brace:weapon", "brace:shipsection", "brace:tech", "brace:combat", "brace:def", "brace:script"): schema_walk(obj, k.split(":")[1], stats) schema_json = {path: {"blocks": st["blocks"], "keys": dict(st["keys"].most_common()), "subblocks": dict(st["sub"].most_common())} for path, st in sorted(stats.items())} json.dump(schema_json, open(os.path.join(out, "schema_stats.json"), "w"), indent=1) P("## Schema stats (key frequency per block type)") P("") P("Full table in `schema_stats.json`. Block paths with instance counts and the") P("keys seen in them (count = number of block instances carrying the key):") P("") for path in ["weapon.weapon", "shipsection.shipsection", "tech.tech"]: st = stats[path] P(f"### `{path}` ({st['blocks']} instances)") P("") P("keys: " + ", ".join(f"{k}:{n}" for k, n in st["keys"].most_common())) P("") P("sub-blocks: " + ", ".join(f"{k}:{n}" for k, n in st["sub"].most_common())) P("") P("All block paths: " + ", ".join(f"`{p}`({st['blocks']})" for p, st in sorted(stats.items()))) P("") # ---- 3. build catalogs ------------------------------------------------- techs = parsed["TechTree/MasterTechList.tech"]["tech"] tech_by = {t["name"].lower(): t for t in techs} groups = collections.defaultdict(list) for t in techs: if "group" in t: groups[str(t["group"]).upper()].append(t["name"]) strings_rows = parsed["Locale/EN/Strings.csv"] strings = {} string_dups = [] for r in strings_rows: k, v = r[0], (r[1] if len(r) > 1 else "") if k in strings: string_dups.append((k, strings[k], v)) strings[k] = v strings_lc = {k.lower(): v for k, v in strings.items()} def s(key): return strings_lc.get(key.lower()) weapons = {} # stem -> record for rel, obj in parsed.items(): if kind_of(rel) != "brace:weapon": continue stem = os.path.basename(rel)[:-7] w = dict(obj["weapon"]) weapons[stem.lower()] = {"stem": stem, "file": rel, "scope": "NPC" if rel.startswith("Species/_NPC") else "player", "id": None, **w} wman = parsed["Weapons/_weapons.txt"] for i, name in wman.entries: key = name.lower()[:-7] if key in weapons and weapons[key]["scope"] == "player": weapons[key]["id"] = i sections = {} # (race, stem) -> record for rel, obj in parsed.items(): if kind_of(rel) != "brace:shipsection": continue race = rel.split("/")[1] stem = os.path.basename(rel)[:-12] sections[(race, stem.lower())] = {"race": race, "stem": stem, "file": rel, "id": None, **obj["shipsection"]} for race in ALL_RACE_DIRS: m = parsed[f"Species/{race}/sections/_shipsections.txt"] for i, name in m.entries: key = (race, name.lower()[:-12]) if key in sections: sections[key]["id"] = i section_stems = collections.defaultdict(list) # stem.lower -> [races] for (race, st), rec in sections.items(): section_stems[st].append(race) # ---- 4. cross-links ---------------------------------------------------- X = {} P("## Cross-link results") P("") def tech_exists(name): n = name.lower() if n in tech_by: return True if n.startswith("grp_") and n[4:].upper() in groups: return True return False # weapon.requires -> tech (repeated `requires` lines = AND of techs) dang = [] case_mismatch = [] nref = 0 multi = 0 for w in weapons.values(): reqs = get_list(w, "requires") multi += len(reqs) > 1 for r in reqs: nref += 1 r = str(r) if not tech_exists(r): dang.append((w["file"], r)) elif r.lower() in tech_by and tech_by[r.lower()]["name"] != r: case_mismatch.append((w["file"], r)) no_req = [w["file"] for w in weapons.values() if "requires" not in w] X["weapon_requires_dangling"] = dang X["weapon_requires_case_mismatch"] = case_mismatch X["weapon_without_requires"] = no_req P(f"- weapon `requires` -> tech: {nref} refs in {len(weapons) - len(no_req)} weapons ({multi} weapons list 2+ techs), " f"{len(dang)} dangling, {len(case_mismatch)} case-mismatched; {len(no_req)} weapons have no `requires` " f"(NPC: {sum(1 for f in no_req if f.startswith('Species/_NPC'))}, player: " f"{[os.path.basename(f) for f in no_req if not f.startswith('Species/_NPC')]}).") for f, r in dang: P(f" - DANGLING `{f}` requires `{r}`") for f, r in case_mismatch: P(f" - case: `{f}` requires `{r}` (tech is `{tech_by[r.lower()]['name']}`)") # shipsection.requires / option -> tech dang = [] case_mm = [] opt_dang = [] scalar_opts = [] nreq = 0 nopt = 0 for rec in sections.values(): for r in get_list(rec, "requires"): nreq += 1 if not tech_exists(str(r)): dang.append((rec["file"], r)) elif str(r).lower() in tech_by and tech_by[str(r).lower()]["name"] != r: case_mm.append((rec["file"], r)) for blk_key in ("option", "optiondef"): for blk in get_list(rec, blk_key): # a few files write a bare `option TECH` at section level # instead of wrapping it in option { } opts = get_list(blk, "option") if isinstance(blk, dict) else [blk] if not isinstance(blk, dict): scalar_opts.append((rec["file"], blk)) for o in opts: nopt += 1 if not tech_exists(str(o)): opt_dang.append((rec["file"], o)) X["shipsection_requires_dangling"] = dang X["shipsection_requires_case_mismatch"] = case_mm X["shipsection_option_dangling"] = opt_dang P(f"- shipsection `requires` -> tech: {nreq} refs, {len(dang)} dangling, {len(case_mm)} case-mismatched.") for f, r in dang: P(f" - DANGLING `{f}` requires `{r}`") for f, r in case_mm: P(f" - case: `{f}` requires `{r}`") X["shipsection_scalar_option"] = scalar_opts P(f"- shipsection `option{{option T}}`/`optiondef` -> tech: {nopt} refs, {len(opt_dang)} dangling. " f"Two forms coexist: `option {{ option A option B }}` (a mutually-exclusive choice group) and a bare " f"section-level `option T` ({len(scalar_opts)} occurrences in {len(set(f for f, _ in scalar_opts))} files, " f"e.g. `option DRV_PlsmFoc` on engine sections) -- both merge under the key `option`, so consumers must " f"accept str-or-dict list members.") for f, r in sorted(set(opt_dang)): P(f" - DANGLING `{f}` option `{r}`") # tech.ship.section -> shipsection dang = [] nsec = 0 for t in techs: for blk in get_list(t, "ship"): for sname in get_list(blk, "section"): nsec += 1 if str(sname).lower() not in section_stems: dang.append((t["name"], sname)) X["tech_ship_section_dangling"] = dang P(f"- tech `ship{{section}}` -> shipsection: {nsec} refs, {len(dang)} dangling " f"(matched against the union of all race catalogs, case-insensitive).") for t, sname in dang: P(f" - DANGLING tech `{t}` unlocks section `{sname}`") # tech.weapon.filename -> file disk = {rel.lower() for _, rel in walk(root)} dang = [(t["name"], w["filename"]) for t in techs for w in get_list(t, "weapon") if w["filename"].lower() not in disk] X["tech_weapon_filename_dangling"] = dang nw = sum(len(get_list(t, "weapon")) for t in techs) P(f"- tech `weapon{{filename}}` -> file: {nw} refs, {len(dang)} dangling.") # tech.requires / allows -> tech dang_req = [(t["name"], r) for t in techs for r in get_list(t, "requires") if not tech_exists(str(r))] edges = [] dang_allow = [] bad_allow = [] for t in techs: for a in get_list(t, "allows"): child, rp, pct, extra = parse_allows(a) if extra or rp is None: bad_allow.append((t["name"], a)) if child.lower() not in tech_by: dang_allow.append((t["name"], child)) edges.append({"from": t["name"], "to": child, "rp": rp, "pct": pct}) X["tech_requires_dangling"] = dang_req X["tech_allows_dangling"] = dang_allow X["tech_allows_unparsed"] = bad_allow P(f"- tech `requires` -> tech/GRP_: {sum(len(get_list(t, 'requires')) for t in techs)} refs, {len(dang_req)} dangling. " f"Groups: {dict((g, len(v)) for g, v in groups.items())}.") for t, r in dang_req: P(f" - DANGLING tech `{t}` requires `{r}`") P(f"- tech `allows` edges: {len(edges)}, {len(dang_allow)} point at unknown techs, {len(bad_allow)} unparsable.") for t, c in dang_allow: P(f" - DANGLING tech `{t}` allows `{c}`") roots = [t["name"] for t in techs if not any(e["to"].lower() == t["name"].lower() for e in edges)] P(f"- techs never allowed by anything (roots/orphans): {len(roots)}: {', '.join(roots)}") dup_names = [n for n, c in collections.Counter(t["name"].lower() for t in techs).items() if c > 1] P(f"- duplicate tech names: {dup_names or 'none'}") # manifests <-> files P("- id manifests <-> files:") man_rep = {} wfiles = {os.path.basename(rel).lower() for rel in parsed if rel.startswith("Weapons/") and rel.endswith(".weapon")} listed = {n.lower() for _, n in wman.entries} man_rep["Weapons"] = {"ids": len(wman.entries), "deleted": wman.deleted, "listed_but_no_file": sorted(listed - wfiles), "file_but_unlisted": sorted(wfiles - listed)} for race in ALL_RACE_DIRS: m = parsed[f"Species/{race}/sections/_shipsections.txt"] files = {os.path.basename(rel).lower() for rel in parsed if rel.startswith(f"Species/{race}/sections/") and rel.endswith(".shipsection")} listed = {n.lower() for _, n in m.entries} man_rep[race] = {"ids": len(m.entries), "deleted": m.deleted, "listed_but_no_file": sorted(listed - files), "file_but_unlisted": sorted(files - listed)} X["manifests"] = man_rep for k, v in man_rep.items(): P(f" - `{k}`: {v['ids']} ids (deleted {v['deleted'] or 'none'}); " f"listed-but-no-file {len(v['listed_but_no_file'])}; file-but-unlisted {len(v['file_but_unlisted'])}") for n in v["listed_but_no_file"]: P(f" - MISSING FILE for id {[i for i, nn in (wman if k == 'Weapons' else parsed[f'Species/{k}/sections/_shipsections.txt']).entries if nn.lower() == n][0]}: `{n}`") for n in v["file_but_unlisted"]: P(f" - UNLISTED file `{n}` (no network/save id)") npc_weapons_unlisted = sorted(w["file"] for w in weapons.values() if w["scope"] == "NPC") P(f" - `Species/_NPC/weapons/*.weapon` ({len(npc_weapons_unlisted)} files) have no manifest at all; " f"they are referenced by filename from `_NPC` shipsection `bank{{weapon}}` lines.") # strings P("- localization:") P(f" - `Strings.csv`: {len(strings_rows)} data rows -> {len(strings)} keys. {len(string_dups)} keys occur twice " f"because one copy carries a trailing space (parse_csv strips cells; the later row wins):") for k, a, b in string_dups: P(f" - `{k}`: {a!r} then {b!r}") miss_tn = [t["name"] for t in techs if s("TECHNAME_" + t["name"]) is None] miss_td = [t["name"] for t in techs if s("TECHDESC_" + t["name"]) is None] P(f" - TECHNAME_/TECHDESC_ for {len(techs)} techs: {len(miss_tn)} / {len(miss_td)} missing. {miss_tn} {miss_td}") stems = sorted(section_stems) miss_sn = [st for st in stems if s("SECTIONNAME_" + st) is None] miss_sd = [st for st in stems if s("SECTIONDESC_" + st) is None] P(f" - SECTIONNAME_/SECTIONDESC_ for {len(stems)} distinct section stems: {len(miss_sn)} / {len(miss_sd)} missing.") for label, miss in (("SECTIONNAME_", miss_sn), ("SECTIONDESC_", miss_sd)): npc_only = [st for st in miss if section_stems[st] == ["_NPC"]] other = [st for st in miss if st not in npc_only] P(f" - missing {label}: {len(npc_only)} are `_NPC`-only stems (never shown in the design UI); " f"player-race stems: {len(other)} {other}") miss_wn = [(w["file"], w.get("name")) for w in weapons.values() if isinstance(w.get("name"), str) and w["name"].startswith("@") and s(w["name"][1:]) is None] unnamed = [w["file"] for w in weapons.values() if "name" not in w] P(f" - weapon `name @TOKEN`: {len(miss_wn)} unresolved of {sum(1 for w in weapons.values() if 'name' in w)}; " f"{len(unnamed)} weapons carry no `name`.") for f, n in miss_wn: P(f" - UNRESOLVED `{f}` name `{n}`") # every @token anywhere in brace files at_missing = collections.Counter() at_total = 0 for rel, obj in parsed.items(): if not kind_of(rel).startswith("brace"): continue for tok in re.findall(r"@([A-Za-z0-9_]+)", manifest.read_text(os.path.join(root, rel))): at_total += 1 if s(tok) is None: at_missing[(rel, tok)] += 1 P(f" - all `@TOKEN` refs in brace-block files: {at_total} refs, {len(at_missing)} unresolved.") for (rel, tok), n in sorted(at_missing.items()): P(f" - UNRESOLVED `{rel}` `@{tok}`") X["strings"] = {"missing_techname": miss_tn, "missing_techdesc": miss_td, "missing_sectionname": miss_sn, "missing_sectiondesc": miss_sd, "unresolved_weapon_name": miss_wn, "unresolved_at_tokens": sorted(f"{r}:@{t}" for r, t in at_missing)} # turrets turrets = parsed["Weapons/_turrets.txt"] def last_lc(d, key): v = get_list(d, key) return str(v[-1]).lower() if v else None tpairs = {(str(r[1]).lower(), str(r[2]).lower()) for r in turrets} # (weapon-size, class) tslots = {(str(r[0]).lower(), str(r[2]).lower()) for r in turrets} # (mount size, class) wpairs = collections.Counter((last_lc(w, "turretsize"), last_lc(w, "turretclass")) for w in weapons.values()) w_unfit = sorted((p, n) for p, n in wpairs.items() if p not in tpairs) bpairs = collections.Counter() nobank = 0 dupkeys = 0 for rec in sections.values(): for b in get_list(rec, "bank"): if "turretsize" not in b: nobank += 1 continue if isinstance(b.get("turretsize"), list) or isinstance(b.get("turretclass"), list): dupkeys += 1 bpairs[(last_lc(b, "turretsize"), last_lc(b, "turretclass"))] += 1 b_unfit = sorted((p, n) for p, n in bpairs.items() if p not in tslots) X["turrets"] = {"turret_rows": len(turrets), "weapon_size_class_pairs_without_turret": w_unfit, "bank_size_class_pairs_without_turret": b_unfit, "banks_without_turretsize": nobank, "banks_with_repeated_size_or_class": dupkeys} P(f"- `_turrets.txt` ({len(turrets)} rows; size/class values compared case-insensitively -- the data mixes " f"`Large`/`large`, `Missile`/`missile`, `Standard`/`standard`):") P(f" - weapon (turretsize,turretclass) pairs with no turret row: {w_unfit or 'none'}") P(f" - section bank (turretsize,turretclass) pairs with no turret row: {b_unfit or 'none'}") P(f" - banks with no turretsize at all (NPC fixed-weapon banks): {nobank}; banks that repeat " f"turretsize/turretclass inside one bank{{}} (last value taken): {dupkeys}") # NPC bank{weapon} refs dang = [] n = 0 for rec in sections.values(): for b in get_list(rec, "bank"): for wf in get_list(b, "weapon"): n += 1 if str(wf).lower() not in disk: dang.append((rec["file"], wf)) X["bank_weapon_dangling"] = dang P(f"- shipsection `bank{{weapon }}` -> file: {n} refs, {len(dang)} dangling.") for f, w in dang: P(f" - DANGLING `{f}` -> `{w}`") # default weapons dw = parsed["Weapons/_defaultweapons.txt"] dang = [r for r in dw if ("weapons/" + str(r[2])).lower() not in disk] P(f"- `_defaultweapons.txt`: {len(dw)} rows, {len(dang)} name a missing weapon file. {dang or ''}") # AI tables def csv_col(rel, col): return [r[col] for r in parsed[rel] if len(r) > col and r[col]] ai = {} for rel in ("Data/Strategy/AI/aitechpri.csv", "Data/Strategy/AI/aitechgrp.csv", "Data/Strategy/AI/aitechmode.csv"): rows = parsed[rel] bad = [t for t in csv_col(rel, 0) if t.lower() not in tech_by] ai[rel] = bad if not rows: P(f"- `{rel}`: 0 data rows -- the shipped file is a comment-only template (schema documented in its " f"header, no entries); the AI's tech priorities must therefore come from code.") else: P(f"- `{rel}`: {len(rows)} rows; col0 not a tech: {bad or 'none'}") bad = [x for x in csv_col("Data/Strategy/AI/affinity_section.csv", 0) if x.lower() not in section_stems] ai["affinity_section_unknown"] = bad P(f"- `AI/affinity_section.csv`: {len(parsed['Data/Strategy/AI/affinity_section.csv'])} rows; unknown sections: {bad or 'none'}") bad = [x for x in csv_col("Data/Strategy/AI/raider_sections.csv", 0) if x.lower() not in section_stems] P(f"- `AI/raider_sections.csv`: unknown sections: {bad or 'none'}") wr = parsed["Data/Strategy/AI/weapon_replacements.csv"] bad = [x for r in wr for x in r if x and x.lower() not in weapons] ai["weapon_replacements_unknown"] = bad P(f"- `AI/weapon_replacements.csv`: {len(wr)} rows; unknown weapon stems: {bad or 'none'}") fams = collections.Counter(str(w.get("weaponfamily")) for w in weapons.values() if "weaponfamily" in w) aw = csv_col("Data/Strategy/AI/affinity_weapon.csv", 0) bad = [x for x in aw if x not in fams] P(f"- `AI/affinity_weapon.csv`: families {sorted(set(aw))}; not a weaponfamily in any .weapon: {bad or 'none'}. " f"weaponfamily values in data: {dict(fams)}") # scenarios for rel in sorted(parsed): if rel.startswith("Scenarios/") and rel.endswith("Templates.csv"): bad = [(r[0], x) for r in parsed[rel] for x in r[1:4] if x.lower() not in section_stems] P(f"- `{rel}`: {len(parsed[rel])} rows; unknown sections: {bad or 'none'}") if rel.startswith("Scenarios/") and rel.endswith("Techs.csv"): bad = [x for x in csv_col(rel, 0) if x.lower() not in tech_by] P(f"- `{rel}`: {len(parsed[rel])} rows; unknown techs: {bad or 'none'}") X["ai"] = ai json.dump(X, open(os.path.join(out, "crosslink.json"), "w"), indent=1) P("") # ---- 5. artifacts ------------------------------------------------------ nodes = [] for t in techs: strat = get_list(t, "strategy") inc = [x for b in strat for x in get_list(b, "inc")] dec = [x for b in strat for x in get_list(b, "dec")] nodes.append({ "name": t["name"], "display_name": s("TECHNAME_" + t["name"]), "description": s("TECHDESC_" + t["name"]), "family": t.get("family"), "family_inferred": t["name"].split("_", 1)[0].upper(), "type": t.get("type"), "threat": t.get("threat"), "group": t.get("group"), "option_cost": t.get("option_cost"), "requires": [str(r) for r in get_list(t, "requires")], "benefits_inc": inc, "benefits_dec": dec, "sections": [str(x) for b in get_list(t, "ship") for x in get_list(b, "section")], "weapons": [w["filename"] for w in get_list(t, "weapon")], "allows": [e["to"] for e in edges if e["from"] == t["name"]], }) tech_tree = { "_about": "SOTS1 MasterTechList.tech normalized. family is only written on ~half the nodes; " "family_inferred is the name prefix (IND/WEP/DRV/...). edges[].pct: per-race availability % as written; " "a race absent from pct has no override in the file (the engine default -- believed to be 100 -- " "is code-owned, not asserted here). rp = research-point cost of the edge. " "requires may name GRP_, satisfied by any tech with group .", "races": RACES, "groups": dict(groups), "nodes": nodes, "edges": edges, } json.dump(tech_tree, open(os.path.join(out, "tech_tree.json"), "w"), indent=1) def norm_weapon(w): d = dict(w) d["display_name"] = s(w["name"][1:]) if isinstance(w.get("name"), str) and w["name"].startswith("@") else w.get("name") return d json.dump({"_about": "All *.weapon files (Weapons/ = player catalog with ids from _weapons.txt; Species/_NPC/weapons = NPC, no ids). " "Keys lower-cased; repeated keys -> lists; bareword numbers typed.", "weapons": [norm_weapon(w) for w in sorted(weapons.values(), key=lambda w: (w["scope"], w["stem"].lower()))]}, open(os.path.join(out, "weapons.json"), "w"), indent=1) def norm_section(r): d = dict(r) d["display_name"] = s("SECTIONNAME_" + r["stem"]) d["description"] = s("SECTIONDESC_" + r["stem"]) d["unlocked_by"] = [t["name"] for t in techs for b in get_list(t, "ship") if r["stem"].lower() in [str(x).lower() for x in get_list(b, "section")]] return d json.dump({"_about": "All Species//sections/*.shipsection; id from the race's _shipsections.txt (null = unlisted). " "Keys lower-cased; repeated keys (bank, option, thruster, requires) -> lists.", "sections": [norm_section(r) for r in sorted(sections.values(), key=lambda r: (r["race"], r["stem"].lower()))]}, open(os.path.join(out, "shipsections.json"), "w"), indent=1) json.dump(strings, open(os.path.join(out, "strings.json"), "w"), indent=1, ensure_ascii=False) with open(os.path.join(out, "tech_tree.dot"), "w") as f: f.write("digraph sots_tech {\n rankdir=LR; node [shape=box, fontsize=9];\n") fam_color = {"IND": "#f4d03f", "NRG": "#e74c3c", "SLD": "#3498db", "DRV": "#9b59b6", "TRP": "#e67e22", "WAR": "#c0392b", "BAL": "#7f8c8d", "BIO": "#2ecc71", "CCC": "#1abc9c", "DRN": "#95a5a6", "XNC": "#d35400"} for n in nodes: col = fam_color.get(str(n["family"]), "#ffffff") label = n["display_name"] or n["name"] f.write(f' "{n["name"]}" [label="{label}\\n{n["name"]}", style=filled, fillcolor="{col}"];\n') for e in edges: lab = f"{e['rp']}" if e["rp"] is not None else "" if e["pct"]: lab += "\\n" + " ".join(f"{r[:2]}{v}" for r, v in e["pct"].items()) f.write(f' "{e["from"]}" -> "{e["to"]}" [label="{lab}", fontsize=7];\n') f.write("}\n") P("## Artifacts") P("") for fn in ("tech_tree.json", "weapons.json", "shipsections.json", "strings.json", "schema_stats.json", "crosslink.json", "tech_tree.dot"): P(f"- `{fn}` ({os.path.getsize(os.path.join(out, fn)) // 1024} KB)") P(f"- tech_tree.json: {len(nodes)} nodes, {len(edges)} edges; weapons.json: {len(weapons)}; " f"shipsections.json: {len(sections)}; strings.json: {len(strings)} keys") print("\n".join(rep)) return 0 if not fails else 1 if __name__ == "__main__": sys.exit(main(sys.argv[1], sys.argv[2]))