634 lines
29 KiB
Python
634 lines
29 KiB
Python
"""verify.py -- parse every shipped SOTS1 data file, cross-link the catalogs,
|
|
and emit normalized JSON artifacts.
|
|
|
|
usage: python3 verify.py <gob-extract-dir> <out-dir>
|
|
|
|
Prints a markdown report to stdout; writes to <out-dir>:
|
|
tech_tree.json weapons.json shipsections.json strings.json
|
|
schema_stats.json crosslink.json tech_tree.dot
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import collections
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import effect_txt
|
|
import flat_kv
|
|
import manifest
|
|
import mars_data
|
|
from mars_data import get_list
|
|
|
|
RACES = ["Human", "Zuul", "Hiver", "Tarkas", "Liir", "Morrigi"]
|
|
PLAYABLE = RACES
|
|
ALL_RACE_DIRS = RACES + ["_NPC"]
|
|
|
|
BRACE_TXT = {"Data/tutorial.txt", "Data/credits.txt", "Data/Strategy/systemnames.txt",
|
|
"Data/Combat/ctechvars.txt", "Data/Combat/shipai.txt",
|
|
"Models/Skysphere/skydefs.txt", "Models/Skysphere/NodeSpace-skydefs.txt"}
|
|
ROWS_TXT = {"Weapons/_turrets.txt", "Weapons/_defaultweapons.txt", "Data/Combat/damfx.txt",
|
|
"Data/Combat/damfx_levels.txt", "Data/Strategy/playercolors.txt",
|
|
"Badges/BadgeTable.txt", "Avatars/AvatarTable.txt", "GUI/WeaponIconPlacements.txt"}
|
|
PROSE = {"Locale/EN/ChatTrans.txt"}
|
|
|
|
|
|
def kind_of(rel: str) -> str:
|
|
ext = rel.rsplit(".", 1)[-1].lower()
|
|
base = os.path.basename(rel)
|
|
if ext in ("weapon", "shipsection", "tech", "combat", "def", "script"):
|
|
return "brace:" + ext
|
|
if ext == "effect":
|
|
return "effect"
|
|
if ext == "csv":
|
|
return "csv"
|
|
if ext in ("fx", "fxh"):
|
|
return "hlsl"
|
|
if ext == "txt":
|
|
if base in ("_weapons.txt", "_shipsections.txt"):
|
|
return "manifest"
|
|
if rel.startswith("Scenarios/") or rel in BRACE_TXT:
|
|
return "brace:txt"
|
|
if rel in ROWS_TXT:
|
|
return "rows"
|
|
if rel.startswith("Locale/EN/Desc") or rel in PROSE:
|
|
return "prose"
|
|
return "kv"
|
|
return "other"
|
|
|
|
|
|
def walk(root):
|
|
for dp, _, fn in os.walk(root):
|
|
for f in sorted(fn):
|
|
p = os.path.join(dp, f)
|
|
yield p, os.path.relpath(p, root).replace(os.sep, "/")
|
|
|
|
|
|
# --- schema stats ------------------------------------------------------------
|
|
|
|
def schema_walk(node, path, stats):
|
|
"""Count key occurrences per block path, and which keys are blocks."""
|
|
st = stats.setdefault(path, {"blocks": 0, "keys": collections.Counter(), "sub": collections.Counter()})
|
|
st["blocks"] += 1
|
|
for k, v in node.items():
|
|
vals = v if isinstance(v, list) else [v]
|
|
for x in vals:
|
|
if isinstance(x, dict):
|
|
st["sub"][k] += 1
|
|
schema_walk(x, path + "." + k, stats)
|
|
else:
|
|
st["keys"][k] += 1
|
|
|
|
|
|
# --- tech tree ---------------------------------------------------------------
|
|
|
|
_RP_RE = re.compile(r"^RP:(\d+)$", re.I)
|
|
_PCT_RE = re.compile(r"^(\w+):(\d+)$")
|
|
|
|
|
|
def parse_allows(s: str):
|
|
toks = s.split()
|
|
child = toks[0]
|
|
rp = None
|
|
pct = {}
|
|
extra = []
|
|
for t in toks[1:]:
|
|
m = _RP_RE.match(t)
|
|
if m:
|
|
rp = int(m.group(1))
|
|
continue
|
|
m = _PCT_RE.match(t)
|
|
if m and m.group(1) in RACES:
|
|
pct[m.group(1)] = int(m.group(2))
|
|
continue
|
|
extra.append(t)
|
|
return child, rp, pct, extra
|
|
|
|
|
|
def main(root: str, out: str) -> int:
|
|
os.makedirs(out, exist_ok=True)
|
|
rep = []
|
|
P = rep.append
|
|
|
|
# ---- 1. parse everything ---------------------------------------------
|
|
ok = collections.Counter()
|
|
fail = collections.Counter()
|
|
fails = []
|
|
warns = []
|
|
parsed = {} # rel -> object
|
|
for p, rel in walk(root):
|
|
k = kind_of(rel)
|
|
try:
|
|
if k.startswith("brace"):
|
|
w = []
|
|
obj = mars_data.parse_file(p, warnings=w)
|
|
# strict re-parse to record the recovery
|
|
if w:
|
|
warns.append((rel, w))
|
|
elif k == "effect":
|
|
obj = effect_txt.parse_file(p)
|
|
elif k == "csv":
|
|
obj = manifest.parse_csv_file(p)
|
|
elif k == "manifest":
|
|
obj = manifest.parse_manifest_file(p)
|
|
if obj.problems:
|
|
raise ValueError("; ".join(obj.problems))
|
|
elif k == "rows":
|
|
obj = flat_kv.parse_rows_file(p)
|
|
elif k == "kv":
|
|
txt = manifest.read_text(p)
|
|
w = []
|
|
obj = flat_kv.parse_kv(txt, warnings=w) # engine rules: first occurrence wins
|
|
if w:
|
|
warns.append((rel, w))
|
|
else:
|
|
continue
|
|
parsed[rel] = obj
|
|
ok[k] += 1
|
|
except Exception as e: # noqa: BLE001
|
|
fail[k] += 1
|
|
fails.append((rel, repr(e)))
|
|
|
|
P("## Parse results")
|
|
P("")
|
|
P("| reader | file kind | files | parsed | failed |")
|
|
P("|---|---|---|---|---|")
|
|
reader_of = {"brace": "mars_data", "effect": "effect_txt", "csv": "manifest.parse_csv",
|
|
"manifest": "manifest.parse_manifest", "rows": "flat_kv.parse_rows", "kv": "flat_kv.parse_kv"}
|
|
for k in sorted(set(ok) | set(fail)):
|
|
P(f"| {reader_of[k.split(':')[0]]} | {k} | {ok[k] + fail[k]} | {ok[k]} | {fail[k]} |")
|
|
P("")
|
|
P(f"Total: {sum(ok.values())} parsed, {sum(fail.values())} failed "
|
|
f"(skipped: {sum(1 for _, r in walk(root) if kind_of(r) in ('hlsl', 'prose', 'other'))} "
|
|
f"HLSL/prose files that are not data).")
|
|
if fails:
|
|
P("")
|
|
P("Failures:")
|
|
for rel, e in fails:
|
|
P(f"- `{rel}`: {e}")
|
|
if warns:
|
|
P("")
|
|
P("Engine tolerances taken (what the original loader does with these files; strict=True would reject):")
|
|
for rel, w in warns:
|
|
P(f"- `{rel}`: {'; '.join(w)}")
|
|
P("")
|
|
|
|
# ---- 2. schema stats ---------------------------------------------------
|
|
stats = {}
|
|
for rel, obj in parsed.items():
|
|
k = kind_of(rel)
|
|
if k in ("brace:weapon", "brace:shipsection", "brace:tech", "brace:combat", "brace:def", "brace:script"):
|
|
schema_walk(obj, k.split(":")[1], stats)
|
|
schema_json = {path: {"blocks": st["blocks"],
|
|
"keys": dict(st["keys"].most_common()),
|
|
"subblocks": dict(st["sub"].most_common())}
|
|
for path, st in sorted(stats.items())}
|
|
json.dump(schema_json, open(os.path.join(out, "schema_stats.json"), "w"), indent=1)
|
|
|
|
P("## Schema stats (key frequency per block type)")
|
|
P("")
|
|
P("Full table in `schema_stats.json`. Block paths with instance counts and the")
|
|
P("keys seen in them (count = number of block instances carrying the key):")
|
|
P("")
|
|
for path in ["weapon.weapon", "shipsection.shipsection", "tech.tech"]:
|
|
st = stats[path]
|
|
P(f"### `{path}` ({st['blocks']} instances)")
|
|
P("")
|
|
P("keys: " + ", ".join(f"{k}:{n}" for k, n in st["keys"].most_common()))
|
|
P("")
|
|
P("sub-blocks: " + ", ".join(f"{k}:{n}" for k, n in st["sub"].most_common()))
|
|
P("")
|
|
P("All block paths: " + ", ".join(f"`{p}`({st['blocks']})" for p, st in sorted(stats.items())))
|
|
P("")
|
|
|
|
# ---- 3. build catalogs -------------------------------------------------
|
|
techs = parsed["TechTree/MasterTechList.tech"]["tech"]
|
|
tech_by = {t["name"].lower(): t for t in techs}
|
|
groups = collections.defaultdict(list)
|
|
for t in techs:
|
|
if "group" in t:
|
|
groups[str(t["group"]).upper()].append(t["name"])
|
|
|
|
strings_rows = parsed["Locale/EN/Strings.csv"]
|
|
strings = {}
|
|
string_dups = []
|
|
for r in strings_rows:
|
|
k, v = r[0], (r[1] if len(r) > 1 else "")
|
|
if k in strings:
|
|
string_dups.append((k, strings[k], v))
|
|
strings[k] = v
|
|
strings_lc = {k.lower(): v for k, v in strings.items()}
|
|
|
|
def s(key):
|
|
return strings_lc.get(key.lower())
|
|
|
|
weapons = {} # stem -> record
|
|
for rel, obj in parsed.items():
|
|
if kind_of(rel) != "brace:weapon":
|
|
continue
|
|
stem = os.path.basename(rel)[:-7]
|
|
w = dict(obj["weapon"])
|
|
weapons[stem.lower()] = {"stem": stem, "file": rel,
|
|
"scope": "NPC" if rel.startswith("Species/_NPC") else "player",
|
|
"id": None, **w}
|
|
wman = parsed["Weapons/_weapons.txt"]
|
|
for i, name in wman.entries:
|
|
key = name.lower()[:-7]
|
|
if key in weapons and weapons[key]["scope"] == "player":
|
|
weapons[key]["id"] = i
|
|
|
|
sections = {} # (race, stem) -> record
|
|
for rel, obj in parsed.items():
|
|
if kind_of(rel) != "brace:shipsection":
|
|
continue
|
|
race = rel.split("/")[1]
|
|
stem = os.path.basename(rel)[:-12]
|
|
sections[(race, stem.lower())] = {"race": race, "stem": stem, "file": rel, "id": None, **obj["shipsection"]}
|
|
for race in ALL_RACE_DIRS:
|
|
m = parsed[f"Species/{race}/sections/_shipsections.txt"]
|
|
for i, name in m.entries:
|
|
key = (race, name.lower()[:-12])
|
|
if key in sections:
|
|
sections[key]["id"] = i
|
|
section_stems = collections.defaultdict(list) # stem.lower -> [races]
|
|
for (race, st), rec in sections.items():
|
|
section_stems[st].append(race)
|
|
|
|
# ---- 4. cross-links ----------------------------------------------------
|
|
X = {}
|
|
P("## Cross-link results")
|
|
P("")
|
|
|
|
def tech_exists(name):
|
|
n = name.lower()
|
|
if n in tech_by:
|
|
return True
|
|
if n.startswith("grp_") and n[4:].upper() in groups:
|
|
return True
|
|
return False
|
|
|
|
# weapon.requires -> tech (repeated `requires` lines = AND of techs)
|
|
dang = []
|
|
case_mismatch = []
|
|
nref = 0
|
|
multi = 0
|
|
for w in weapons.values():
|
|
reqs = get_list(w, "requires")
|
|
multi += len(reqs) > 1
|
|
for r in reqs:
|
|
nref += 1
|
|
r = str(r)
|
|
if not tech_exists(r):
|
|
dang.append((w["file"], r))
|
|
elif r.lower() in tech_by and tech_by[r.lower()]["name"] != r:
|
|
case_mismatch.append((w["file"], r))
|
|
no_req = [w["file"] for w in weapons.values() if "requires" not in w]
|
|
X["weapon_requires_dangling"] = dang
|
|
X["weapon_requires_case_mismatch"] = case_mismatch
|
|
X["weapon_without_requires"] = no_req
|
|
P(f"- weapon `requires` -> tech: {nref} refs in {len(weapons) - len(no_req)} weapons ({multi} weapons list 2+ techs), "
|
|
f"{len(dang)} dangling, {len(case_mismatch)} case-mismatched; {len(no_req)} weapons have no `requires` "
|
|
f"(NPC: {sum(1 for f in no_req if f.startswith('Species/_NPC'))}, player: "
|
|
f"{[os.path.basename(f) for f in no_req if not f.startswith('Species/_NPC')]}).")
|
|
for f, r in dang:
|
|
P(f" - DANGLING `{f}` requires `{r}`")
|
|
for f, r in case_mismatch:
|
|
P(f" - case: `{f}` requires `{r}` (tech is `{tech_by[r.lower()]['name']}`)")
|
|
|
|
# shipsection.requires / option -> tech
|
|
dang = []
|
|
case_mm = []
|
|
opt_dang = []
|
|
scalar_opts = []
|
|
nreq = 0
|
|
nopt = 0
|
|
for rec in sections.values():
|
|
for r in get_list(rec, "requires"):
|
|
nreq += 1
|
|
if not tech_exists(str(r)):
|
|
dang.append((rec["file"], r))
|
|
elif str(r).lower() in tech_by and tech_by[str(r).lower()]["name"] != r:
|
|
case_mm.append((rec["file"], r))
|
|
for blk_key in ("option", "optiondef"):
|
|
for blk in get_list(rec, blk_key):
|
|
# a few files write a bare `option TECH` at section level
|
|
# instead of wrapping it in option { }
|
|
opts = get_list(blk, "option") if isinstance(blk, dict) else [blk]
|
|
if not isinstance(blk, dict):
|
|
scalar_opts.append((rec["file"], blk))
|
|
for o in opts:
|
|
nopt += 1
|
|
if not tech_exists(str(o)):
|
|
opt_dang.append((rec["file"], o))
|
|
X["shipsection_requires_dangling"] = dang
|
|
X["shipsection_requires_case_mismatch"] = case_mm
|
|
X["shipsection_option_dangling"] = opt_dang
|
|
P(f"- shipsection `requires` -> tech: {nreq} refs, {len(dang)} dangling, {len(case_mm)} case-mismatched.")
|
|
for f, r in dang:
|
|
P(f" - DANGLING `{f}` requires `{r}`")
|
|
for f, r in case_mm:
|
|
P(f" - case: `{f}` requires `{r}`")
|
|
X["shipsection_scalar_option"] = scalar_opts
|
|
P(f"- shipsection `option{{option T}}`/`optiondef` -> tech: {nopt} refs, {len(opt_dang)} dangling. "
|
|
f"Two forms coexist: `option {{ option A option B }}` (a mutually-exclusive choice group) and a bare "
|
|
f"section-level `option T` ({len(scalar_opts)} occurrences in {len(set(f for f, _ in scalar_opts))} files, "
|
|
f"e.g. `option DRV_PlsmFoc` on engine sections) -- both merge under the key `option`, so consumers must "
|
|
f"accept str-or-dict list members.")
|
|
for f, r in sorted(set(opt_dang)):
|
|
P(f" - DANGLING `{f}` option `{r}`")
|
|
|
|
# tech.ship.section -> shipsection
|
|
dang = []
|
|
nsec = 0
|
|
for t in techs:
|
|
for blk in get_list(t, "ship"):
|
|
for sname in get_list(blk, "section"):
|
|
nsec += 1
|
|
if str(sname).lower() not in section_stems:
|
|
dang.append((t["name"], sname))
|
|
X["tech_ship_section_dangling"] = dang
|
|
P(f"- tech `ship{{section}}` -> shipsection: {nsec} refs, {len(dang)} dangling "
|
|
f"(matched against the union of all race catalogs, case-insensitive).")
|
|
for t, sname in dang:
|
|
P(f" - DANGLING tech `{t}` unlocks section `{sname}`")
|
|
|
|
# tech.weapon.filename -> file
|
|
disk = {rel.lower() for _, rel in walk(root)}
|
|
dang = [(t["name"], w["filename"]) for t in techs for w in get_list(t, "weapon") if w["filename"].lower() not in disk]
|
|
X["tech_weapon_filename_dangling"] = dang
|
|
nw = sum(len(get_list(t, "weapon")) for t in techs)
|
|
P(f"- tech `weapon{{filename}}` -> file: {nw} refs, {len(dang)} dangling.")
|
|
|
|
# tech.requires / allows -> tech
|
|
dang_req = [(t["name"], r) for t in techs for r in get_list(t, "requires") if not tech_exists(str(r))]
|
|
edges = []
|
|
dang_allow = []
|
|
bad_allow = []
|
|
for t in techs:
|
|
for a in get_list(t, "allows"):
|
|
child, rp, pct, extra = parse_allows(a)
|
|
if extra or rp is None:
|
|
bad_allow.append((t["name"], a))
|
|
if child.lower() not in tech_by:
|
|
dang_allow.append((t["name"], child))
|
|
edges.append({"from": t["name"], "to": child, "rp": rp, "pct": pct})
|
|
X["tech_requires_dangling"] = dang_req
|
|
X["tech_allows_dangling"] = dang_allow
|
|
X["tech_allows_unparsed"] = bad_allow
|
|
P(f"- tech `requires` -> tech/GRP_: {sum(len(get_list(t, 'requires')) for t in techs)} refs, {len(dang_req)} dangling. "
|
|
f"Groups: {dict((g, len(v)) for g, v in groups.items())}.")
|
|
for t, r in dang_req:
|
|
P(f" - DANGLING tech `{t}` requires `{r}`")
|
|
P(f"- tech `allows` edges: {len(edges)}, {len(dang_allow)} point at unknown techs, {len(bad_allow)} unparsable.")
|
|
for t, c in dang_allow:
|
|
P(f" - DANGLING tech `{t}` allows `{c}`")
|
|
roots = [t["name"] for t in techs if not any(e["to"].lower() == t["name"].lower() for e in edges)]
|
|
P(f"- techs never allowed by anything (roots/orphans): {len(roots)}: {', '.join(roots)}")
|
|
dup_names = [n for n, c in collections.Counter(t["name"].lower() for t in techs).items() if c > 1]
|
|
P(f"- duplicate tech names: {dup_names or 'none'}")
|
|
|
|
# manifests <-> files
|
|
P("- id manifests <-> files:")
|
|
man_rep = {}
|
|
wfiles = {os.path.basename(rel).lower() for rel in parsed if rel.startswith("Weapons/") and rel.endswith(".weapon")}
|
|
listed = {n.lower() for _, n in wman.entries}
|
|
man_rep["Weapons"] = {"ids": len(wman.entries), "deleted": wman.deleted,
|
|
"listed_but_no_file": sorted(listed - wfiles), "file_but_unlisted": sorted(wfiles - listed)}
|
|
for race in ALL_RACE_DIRS:
|
|
m = parsed[f"Species/{race}/sections/_shipsections.txt"]
|
|
files = {os.path.basename(rel).lower() for rel in parsed if rel.startswith(f"Species/{race}/sections/") and rel.endswith(".shipsection")}
|
|
listed = {n.lower() for _, n in m.entries}
|
|
man_rep[race] = {"ids": len(m.entries), "deleted": m.deleted,
|
|
"listed_but_no_file": sorted(listed - files), "file_but_unlisted": sorted(files - listed)}
|
|
X["manifests"] = man_rep
|
|
for k, v in man_rep.items():
|
|
P(f" - `{k}`: {v['ids']} ids (deleted {v['deleted'] or 'none'}); "
|
|
f"listed-but-no-file {len(v['listed_but_no_file'])}; file-but-unlisted {len(v['file_but_unlisted'])}")
|
|
for n in v["listed_but_no_file"]:
|
|
P(f" - MISSING FILE for id {[i for i, nn in (wman if k == 'Weapons' else parsed[f'Species/{k}/sections/_shipsections.txt']).entries if nn.lower() == n][0]}: `{n}`")
|
|
for n in v["file_but_unlisted"]:
|
|
P(f" - UNLISTED file `{n}` (no network/save id)")
|
|
npc_weapons_unlisted = sorted(w["file"] for w in weapons.values() if w["scope"] == "NPC")
|
|
P(f" - `Species/_NPC/weapons/*.weapon` ({len(npc_weapons_unlisted)} files) have no manifest at all; "
|
|
f"they are referenced by filename from `_NPC` shipsection `bank{{weapon}}` lines.")
|
|
|
|
# strings
|
|
P("- localization:")
|
|
P(f" - `Strings.csv`: {len(strings_rows)} data rows -> {len(strings)} keys. {len(string_dups)} keys occur twice "
|
|
f"because one copy carries a trailing space (parse_csv strips cells; the later row wins):")
|
|
for k, a, b in string_dups:
|
|
P(f" - `{k}`: {a!r} then {b!r}")
|
|
miss_tn = [t["name"] for t in techs if s("TECHNAME_" + t["name"]) is None]
|
|
miss_td = [t["name"] for t in techs if s("TECHDESC_" + t["name"]) is None]
|
|
P(f" - TECHNAME_/TECHDESC_ for {len(techs)} techs: {len(miss_tn)} / {len(miss_td)} missing. {miss_tn} {miss_td}")
|
|
stems = sorted(section_stems)
|
|
miss_sn = [st for st in stems if s("SECTIONNAME_" + st) is None]
|
|
miss_sd = [st for st in stems if s("SECTIONDESC_" + st) is None]
|
|
P(f" - SECTIONNAME_/SECTIONDESC_ for {len(stems)} distinct section stems: {len(miss_sn)} / {len(miss_sd)} missing.")
|
|
for label, miss in (("SECTIONNAME_", miss_sn), ("SECTIONDESC_", miss_sd)):
|
|
npc_only = [st for st in miss if section_stems[st] == ["_NPC"]]
|
|
other = [st for st in miss if st not in npc_only]
|
|
P(f" - missing {label}: {len(npc_only)} are `_NPC`-only stems (never shown in the design UI); "
|
|
f"player-race stems: {len(other)} {other}")
|
|
miss_wn = [(w["file"], w.get("name")) for w in weapons.values()
|
|
if isinstance(w.get("name"), str) and w["name"].startswith("@") and s(w["name"][1:]) is None]
|
|
unnamed = [w["file"] for w in weapons.values() if "name" not in w]
|
|
P(f" - weapon `name @TOKEN`: {len(miss_wn)} unresolved of {sum(1 for w in weapons.values() if 'name' in w)}; "
|
|
f"{len(unnamed)} weapons carry no `name`.")
|
|
for f, n in miss_wn:
|
|
P(f" - UNRESOLVED `{f}` name `{n}`")
|
|
# every @token anywhere in brace files
|
|
at_missing = collections.Counter()
|
|
at_total = 0
|
|
for rel, obj in parsed.items():
|
|
if not kind_of(rel).startswith("brace"):
|
|
continue
|
|
for tok in re.findall(r"@([A-Za-z0-9_]+)", manifest.read_text(os.path.join(root, rel))):
|
|
at_total += 1
|
|
if s(tok) is None:
|
|
at_missing[(rel, tok)] += 1
|
|
P(f" - all `@TOKEN` refs in brace-block files: {at_total} refs, {len(at_missing)} unresolved.")
|
|
for (rel, tok), n in sorted(at_missing.items()):
|
|
P(f" - UNRESOLVED `{rel}` `@{tok}`")
|
|
X["strings"] = {"missing_techname": miss_tn, "missing_techdesc": miss_td,
|
|
"missing_sectionname": miss_sn, "missing_sectiondesc": miss_sd,
|
|
"unresolved_weapon_name": miss_wn, "unresolved_at_tokens": sorted(f"{r}:@{t}" for r, t in at_missing)}
|
|
|
|
# turrets
|
|
turrets = parsed["Weapons/_turrets.txt"]
|
|
|
|
def last_lc(d, key):
|
|
v = get_list(d, key)
|
|
return str(v[-1]).lower() if v else None
|
|
|
|
tpairs = {(str(r[1]).lower(), str(r[2]).lower()) for r in turrets} # (weapon-size, class)
|
|
tslots = {(str(r[0]).lower(), str(r[2]).lower()) for r in turrets} # (mount size, class)
|
|
wpairs = collections.Counter((last_lc(w, "turretsize"), last_lc(w, "turretclass")) for w in weapons.values())
|
|
w_unfit = sorted((p, n) for p, n in wpairs.items() if p not in tpairs)
|
|
bpairs = collections.Counter()
|
|
nobank = 0
|
|
dupkeys = 0
|
|
for rec in sections.values():
|
|
for b in get_list(rec, "bank"):
|
|
if "turretsize" not in b:
|
|
nobank += 1
|
|
continue
|
|
if isinstance(b.get("turretsize"), list) or isinstance(b.get("turretclass"), list):
|
|
dupkeys += 1
|
|
bpairs[(last_lc(b, "turretsize"), last_lc(b, "turretclass"))] += 1
|
|
b_unfit = sorted((p, n) for p, n in bpairs.items() if p not in tslots)
|
|
X["turrets"] = {"turret_rows": len(turrets), "weapon_size_class_pairs_without_turret": w_unfit,
|
|
"bank_size_class_pairs_without_turret": b_unfit,
|
|
"banks_without_turretsize": nobank, "banks_with_repeated_size_or_class": dupkeys}
|
|
P(f"- `_turrets.txt` ({len(turrets)} rows; size/class values compared case-insensitively -- the data mixes "
|
|
f"`Large`/`large`, `Missile`/`missile`, `Standard`/`standard`):")
|
|
P(f" - weapon (turretsize,turretclass) pairs with no turret row: {w_unfit or 'none'}")
|
|
P(f" - section bank (turretsize,turretclass) pairs with no turret row: {b_unfit or 'none'}")
|
|
P(f" - banks with no turretsize at all (NPC fixed-weapon banks): {nobank}; banks that repeat "
|
|
f"turretsize/turretclass inside one bank{{}} (last value taken): {dupkeys}")
|
|
|
|
# NPC bank{weapon} refs
|
|
dang = []
|
|
n = 0
|
|
for rec in sections.values():
|
|
for b in get_list(rec, "bank"):
|
|
for wf in get_list(b, "weapon"):
|
|
n += 1
|
|
if str(wf).lower() not in disk:
|
|
dang.append((rec["file"], wf))
|
|
X["bank_weapon_dangling"] = dang
|
|
P(f"- shipsection `bank{{weapon <file>}}` -> file: {n} refs, {len(dang)} dangling.")
|
|
for f, w in dang:
|
|
P(f" - DANGLING `{f}` -> `{w}`")
|
|
|
|
# default weapons
|
|
dw = parsed["Weapons/_defaultweapons.txt"]
|
|
dang = [r for r in dw if ("weapons/" + str(r[2])).lower() not in disk]
|
|
P(f"- `_defaultweapons.txt`: {len(dw)} rows, {len(dang)} name a missing weapon file. {dang or ''}")
|
|
|
|
# AI tables
|
|
def csv_col(rel, col):
|
|
return [r[col] for r in parsed[rel] if len(r) > col and r[col]]
|
|
ai = {}
|
|
for rel in ("Data/Strategy/AI/aitechpri.csv", "Data/Strategy/AI/aitechgrp.csv", "Data/Strategy/AI/aitechmode.csv"):
|
|
rows = parsed[rel]
|
|
bad = [t for t in csv_col(rel, 0) if t.lower() not in tech_by]
|
|
ai[rel] = bad
|
|
if not rows:
|
|
P(f"- `{rel}`: 0 data rows -- the shipped file is a comment-only template (schema documented in its "
|
|
f"header, no entries); the AI's tech priorities must therefore come from code.")
|
|
else:
|
|
P(f"- `{rel}`: {len(rows)} rows; col0 not a tech: {bad or 'none'}")
|
|
bad = [x for x in csv_col("Data/Strategy/AI/affinity_section.csv", 0) if x.lower() not in section_stems]
|
|
ai["affinity_section_unknown"] = bad
|
|
P(f"- `AI/affinity_section.csv`: {len(parsed['Data/Strategy/AI/affinity_section.csv'])} rows; unknown sections: {bad or 'none'}")
|
|
bad = [x for x in csv_col("Data/Strategy/AI/raider_sections.csv", 0) if x.lower() not in section_stems]
|
|
P(f"- `AI/raider_sections.csv`: unknown sections: {bad or 'none'}")
|
|
wr = parsed["Data/Strategy/AI/weapon_replacements.csv"]
|
|
bad = [x for r in wr for x in r if x and x.lower() not in weapons]
|
|
ai["weapon_replacements_unknown"] = bad
|
|
P(f"- `AI/weapon_replacements.csv`: {len(wr)} rows; unknown weapon stems: {bad or 'none'}")
|
|
fams = collections.Counter(str(w.get("weaponfamily")) for w in weapons.values() if "weaponfamily" in w)
|
|
aw = csv_col("Data/Strategy/AI/affinity_weapon.csv", 0)
|
|
bad = [x for x in aw if x not in fams]
|
|
P(f"- `AI/affinity_weapon.csv`: families {sorted(set(aw))}; not a weaponfamily in any .weapon: {bad or 'none'}. "
|
|
f"weaponfamily values in data: {dict(fams)}")
|
|
# scenarios
|
|
for rel in sorted(parsed):
|
|
if rel.startswith("Scenarios/") and rel.endswith("Templates.csv"):
|
|
bad = [(r[0], x) for r in parsed[rel] for x in r[1:4] if x.lower() not in section_stems]
|
|
P(f"- `{rel}`: {len(parsed[rel])} rows; unknown sections: {bad or 'none'}")
|
|
if rel.startswith("Scenarios/") and rel.endswith("Techs.csv"):
|
|
bad = [x for x in csv_col(rel, 0) if x.lower() not in tech_by]
|
|
P(f"- `{rel}`: {len(parsed[rel])} rows; unknown techs: {bad or 'none'}")
|
|
X["ai"] = ai
|
|
json.dump(X, open(os.path.join(out, "crosslink.json"), "w"), indent=1)
|
|
P("")
|
|
|
|
# ---- 5. artifacts ------------------------------------------------------
|
|
nodes = []
|
|
for t in techs:
|
|
strat = get_list(t, "strategy")
|
|
inc = [x for b in strat for x in get_list(b, "inc")]
|
|
dec = [x for b in strat for x in get_list(b, "dec")]
|
|
nodes.append({
|
|
"name": t["name"],
|
|
"display_name": s("TECHNAME_" + t["name"]),
|
|
"description": s("TECHDESC_" + t["name"]),
|
|
"family": t.get("family"),
|
|
"family_inferred": t["name"].split("_", 1)[0].upper(),
|
|
"type": t.get("type"),
|
|
"threat": t.get("threat"),
|
|
"group": t.get("group"),
|
|
"option_cost": t.get("option_cost"),
|
|
"requires": [str(r) for r in get_list(t, "requires")],
|
|
"benefits_inc": inc,
|
|
"benefits_dec": dec,
|
|
"sections": [str(x) for b in get_list(t, "ship") for x in get_list(b, "section")],
|
|
"weapons": [w["filename"] for w in get_list(t, "weapon")],
|
|
"allows": [e["to"] for e in edges if e["from"] == t["name"]],
|
|
})
|
|
tech_tree = {
|
|
"_about": "SOTS1 MasterTechList.tech normalized. family is only written on ~half the nodes; "
|
|
"family_inferred is the name prefix (IND/WEP/DRV/...). edges[].pct: per-race availability % as written; "
|
|
"a race absent from pct has no override in the file (the engine default -- believed to be 100 -- "
|
|
"is code-owned, not asserted here). rp = research-point cost of the edge. "
|
|
"requires may name GRP_<group>, satisfied by any tech with group <group>.",
|
|
"races": RACES,
|
|
"groups": dict(groups),
|
|
"nodes": nodes,
|
|
"edges": edges,
|
|
}
|
|
json.dump(tech_tree, open(os.path.join(out, "tech_tree.json"), "w"), indent=1)
|
|
|
|
def norm_weapon(w):
|
|
d = dict(w)
|
|
d["display_name"] = s(w["name"][1:]) if isinstance(w.get("name"), str) and w["name"].startswith("@") else w.get("name")
|
|
return d
|
|
json.dump({"_about": "All *.weapon files (Weapons/ = player catalog with ids from _weapons.txt; Species/_NPC/weapons = NPC, no ids). "
|
|
"Keys lower-cased; repeated keys -> lists; bareword numbers typed.",
|
|
"weapons": [norm_weapon(w) for w in sorted(weapons.values(), key=lambda w: (w["scope"], w["stem"].lower()))]},
|
|
open(os.path.join(out, "weapons.json"), "w"), indent=1)
|
|
|
|
def norm_section(r):
|
|
d = dict(r)
|
|
d["display_name"] = s("SECTIONNAME_" + r["stem"])
|
|
d["description"] = s("SECTIONDESC_" + r["stem"])
|
|
d["unlocked_by"] = [t["name"] for t in techs for b in get_list(t, "ship") if r["stem"].lower() in [str(x).lower() for x in get_list(b, "section")]]
|
|
return d
|
|
json.dump({"_about": "All Species/<race>/sections/*.shipsection; id from the race's _shipsections.txt (null = unlisted). "
|
|
"Keys lower-cased; repeated keys (bank, option, thruster, requires) -> lists.",
|
|
"sections": [norm_section(r) for r in sorted(sections.values(), key=lambda r: (r["race"], r["stem"].lower()))]},
|
|
open(os.path.join(out, "shipsections.json"), "w"), indent=1)
|
|
|
|
json.dump(strings, open(os.path.join(out, "strings.json"), "w"), indent=1, ensure_ascii=False)
|
|
|
|
with open(os.path.join(out, "tech_tree.dot"), "w") as f:
|
|
f.write("digraph sots_tech {\n rankdir=LR; node [shape=box, fontsize=9];\n")
|
|
fam_color = {"IND": "#f4d03f", "NRG": "#e74c3c", "SLD": "#3498db", "DRV": "#9b59b6", "TRP": "#e67e22",
|
|
"WAR": "#c0392b", "BAL": "#7f8c8d", "BIO": "#2ecc71", "CCC": "#1abc9c", "DRN": "#95a5a6", "XNC": "#d35400"}
|
|
for n in nodes:
|
|
col = fam_color.get(str(n["family"]), "#ffffff")
|
|
label = n["display_name"] or n["name"]
|
|
f.write(f' "{n["name"]}" [label="{label}\\n{n["name"]}", style=filled, fillcolor="{col}"];\n')
|
|
for e in edges:
|
|
lab = f"{e['rp']}" if e["rp"] is not None else ""
|
|
if e["pct"]:
|
|
lab += "\\n" + " ".join(f"{r[:2]}{v}" for r, v in e["pct"].items())
|
|
f.write(f' "{e["from"]}" -> "{e["to"]}" [label="{lab}", fontsize=7];\n')
|
|
f.write("}\n")
|
|
|
|
P("## Artifacts")
|
|
P("")
|
|
for fn in ("tech_tree.json", "weapons.json", "shipsections.json", "strings.json", "schema_stats.json", "crosslink.json", "tech_tree.dot"):
|
|
P(f"- `{fn}` ({os.path.getsize(os.path.join(out, fn)) // 1024} KB)")
|
|
P(f"- tech_tree.json: {len(nodes)} nodes, {len(edges)} edges; weapons.json: {len(weapons)}; "
|
|
f"shipsections.json: {len(sections)}; strings.json: {len(strings)} keys")
|
|
print("\n".join(rep))
|
|
return 0 if not fails else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1], sys.argv[2]))
|