#!/usr/bin/env python3 """compare.py Compares the dump_catalog output with the reference catalogs produced by the RE repo's verify.py (tech_tree.json, weapons.json, shipsections.json, strings.json, crosslink.json), field by field. Canonicalisation (both sides): * raw bodies: exact, type-aware (bool != int != float != str), after the oracle's identity keys are removed. This proves the block snapshot. * typed fields: the oracle value under the same key, with these rules -- - repeated scalar keys -> the LAST value (the loader's rule); - repeatable list keys (requires, exclude, ...) -> always a list; - numbers compare numerically (int 50 == float 50.0); - bools accept the oracle's 0/1 ints; - a typed field that is absent (null / "" / []) matches a missing key; - a typed field that is null while the oracle holds a non-numeric value is an "untyped" divergence: the data carries a token the engine schema does not accept. These are listed and must match the documented set exactly. Exit 1 on any unexpected difference. """ import json import os import sys # Values in the shipped data that do not have the shape the typed schema # expects. The loader records a Problem and keeps the text in `raw`. KNOWN_UNTYPED = { ("weapon", "Weapons/las_red.weapon", "trackspeed_mod", "1.0f"), ("section", "Species/Hiver/sections/CRFusion.shipsection", "mount.min_inclination", "0-5"), ("section", "Species/Morrigi/sections/DNAIC.shipsection", "mount.max_inclination", "90\\"), } # `crew false` on the nine NPC hull sections for f in ("_CRRipperCommand", "_CRRipperEngine", "_CRRipperMission", "_DECommand", "_DEEngine", "_DEMission", "_VNMFrontSection", "_VNMMidSection", "_VNMRearSection"): KNOWN_UNTYPED.add(("section", f"Species/_NPC/sections/{f}.shipsection", "crew", False)) # `force_right o` (a letter o for zero) on 22 sections for race, stems in { "Hiver": ("CRDeflector", "CRDisruptor", "DEFireControl"), "Human": ("CRAbsorber", "CRAssault", "CRBattleBridge", "CRDeepScan", "CRFireControl"), "Liir": ("CRDeepScan", "CRFireControl", "DEDeepScan"), "Morrigi": ("CRAbsorber", "CRAssault", "CRBattleBridge", "CRDeepScan", "CRFireControl"), "Tarkas": ("CRCommand", "CRDeepScan"), "Zuul": ("CRAssault", "CRBattleBridge", "CRDeepScan", "CRFireControl"), }.items(): for st in stems: KNOWN_UNTYPED.add(("section", f"Species/{race}/sections/{st}.shipsection", "netforcelimits.force_right", "o")) MAX_DIFFS = 40 def L(v): """get_list: missing -> [], scalar -> [x], list -> list.""" if v is None: return [] return v if isinstance(v, list) else [v] def last(v): return v[-1] if isinstance(v, list) else v class Cmp: def __init__(self): self.diffs = [] self.untyped = set() self.checked = 0 def diff(self, path, msg): self.diffs.append(f"{path}: {msg}") # --- exact structural compare (raw bodies) -------------------------- def exact(self, a, b, path): self.checked += 1 if type(a) is not type(b): self.diff(path, f"type {type(a).__name__} != {type(b).__name__} ({a!r} vs {b!r})") return if isinstance(a, dict): for k in sorted(set(a) | set(b)): if k not in a: self.diff(f"{path}.{k}", "missing in ours") elif k not in b: self.diff(f"{path}.{k}", "extra in ours") else: self.exact(a[k], b[k], f"{path}.{k}") elif isinstance(a, list): if len(a) != len(b): self.diff(path, f"length {len(a)} != {len(b)}") for i, (x, y) in enumerate(zip(a, b)): self.exact(x, y, f"{path}[{i}]") elif a != b: self.diff(path, f"{a!r} != {b!r}") # --- typed field compare --------------------------------------------- def scalar(self, kind, file, key, ours, oracle, path): """ours: typed value (None/str/number/bool). oracle: raw oracle value (scalar or list).""" self.checked += 1 oracle = last(oracle) if ours is None or ours == "" or ours == []: if oracle is None: return if isinstance(oracle, (str, bool)) and ours is None: self.untyped.add((kind, file, key, oracle)) return if ours == "" and oracle == "": return self.diff(path, f"ours absent, oracle {oracle!r}") return if oracle is None: self.diff(path, f"ours {ours!r}, oracle missing") return if isinstance(ours, bool): if isinstance(oracle, bool): ok = ours == oracle elif isinstance(oracle, int): ok = ours == (oracle != 0) else: ok = False elif isinstance(ours, (int, float)): ok = isinstance(oracle, (int, float)) and not isinstance(oracle, bool) and float(ours) == float(oracle) elif isinstance(ours, str): if isinstance(oracle, str): ok = ours == oracle elif isinstance(oracle, bool): ok = ours.lower() == str(oracle).lower() else: try: ok = float(ours) == float(oracle) except ValueError: ok = False else: ok = ours == oracle if not ok: self.diff(path, f"{ours!r} != {oracle!r}") def strlist(self, ours, oracle, path): self.checked += 1 exp = [str(x) for x in L(oracle)] if list(ours) != exp: self.diff(path, f"{ours!r} != {exp!r}") WEAPON_TYPED_SCALARS = [ "name", "weaponclass", "weaponfamily", "weapondamagetype", "exclusive_species", "cost", "turretsize", "turretclass", "trackspeed_mod", "burst_volleys", "recharge_time", "volley_period", "volley_duration", "buildup_delay", "solution_tolerance", "range", "range_planet", "muzzle_speed", "hpbonus", "dam_est", "hidden", "pinpoint", "blindfire", "secondary_pd", "model1", "model2", "model3", "muzzle_effect", "muzzle_sound", "icon_file", "icon_rect", "fc_requires_los", "fc_requires_inrange", "fc_requires_enemycolony", "fc_manual_target", "fc_manual_toggle", "fc_manual_launch", "fc_controllable", "fc_holdsfire", "fc_explicit_target", "fc_exclusive_launch", "fc_targets_expire", "rating_frate", "rating_dam", "rating_acc", "rating_range", ] BEHAVIOR_BLOCKS = ["bolt", "beam", "torpedo", "rider", "missile", "chainlightning", "col", "mine", "disintegrator", "grapple", "projectedshield", "mirv", "nodecannon", "siege", "mesonprojector", "spyship", "wraith"] RANGE_KEYS = ["pb_range", "pb_range_dev", "pb_range_dam", "eff_range", "eff_range_dev", "eff_range_dam", "max_range", "max_range_dev", "max_range_dam"] BOLT_SCALARS = ["dam_pop", "dam_infra", "dam_terra", "mass", "beam_origin", "beam_length", "ricochet_mod", "effect", "impact_effect", "expire_effect"] SECTION_TYPED_SCALARS = [ "model", "dam_model", "section_type", "section_class", "design_class", "entity_class", "health", "mass", "cost", "cpoints", "crew", "command_cost", "maintenance_cost", "command_quota", "socket_fore", "socket_aft", "dam_socket_fore", "dam_socket_aft", "ftlspeed", "nodespeed", "range", "scanrange", "tacticalsensorrange", "engine_techera", "explicit_command_section", "explicit_engine_section", "explicit_section", "autonomous", "nodesign", ] MOUNT_KEYS = ["min_azimuth", "max_azimuth", "min_inclination", "max_inclination", "home_azimuth", "home_inclination"] NFL_KEYS = ["force_forward", "force_right", "force_up", "torque_yaw", "torque_pitch", "torque_roll", "speed", "rotspeed"] WEAPON_ID_KEYS = {"stem", "file", "scope", "id", "display_name"} SECTION_ID_KEYS = {"race", "stem", "file", "id", "display_name", "description", "unlocked_by"} def compare_weapons(c, ours, oracle): ob = {w["file"].lower(): w for w in oracle["weapons"]} if len(ours) != len(oracle["weapons"]): c.diff("weapons", f"count {len(ours)} != {len(oracle['weapons'])}") for w in ours: p = f"weapon[{w['file']}]" o = ob.get(w["file"].lower()) if o is None: c.diff(p, "not in oracle") continue for k in ("stem", "scope", "id", "display_name"): if w[k] != o[k]: c.diff(f"{p}.{k}", f"{w[k]!r} != {o[k]!r}") body = {k: v for k, v in o.items() if k not in WEAPON_ID_KEYS} c.exact(w["raw"], body, f"{p}.raw") t = w["typed"] for k in WEAPON_TYPED_SCALARS: c.scalar("weapon", w["file"], k, t[k], body.get(k), f"{p}.{k}") c.strlist(t["requires"], body.get("requires"), f"{p}.requires") c.strlist(t["compatible_section"], body.get("compatible_section"), f"{p}.compatible_section") kind = next((b for b in BEHAVIOR_BLOCKS if b in body), "") if t["behavior_kind"] != kind: c.diff(f"{p}.behavior_kind", f"{t['behavior_kind']!r} != {kind!r}") beh = body.get(kind, {}) if kind else {} for k in ("dam_pop", "dam_infra", "dam_terra"): c.scalar("weapon", w["file"], f"{kind}.{k}", t["planet_damage"][k], beh.get(k), f"{p}.planet_damage.{k}") rt = beh.get("rangetable") if (t["rangetable"] is None) != (rt is None): c.diff(f"{p}.rangetable", f"presence {t['rangetable'] is not None} != {rt is not None}") elif rt is not None: for k in RANGE_KEYS: c.scalar("weapon", w["file"], f"rangetable.{k}", t["rangetable"][k], rt.get(k), f"{p}.rangetable.{k}") if (t["bolt"] is None) != (kind != "bolt"): c.diff(f"{p}.bolt", f"presence mismatch (kind {kind})") elif t["bolt"] is not None: for k in BOLT_SCALARS: c.scalar("weapon", w["file"], f"bolt.{k}", t["bolt"][k], beh.get(k), f"{p}.bolt.{k}") for k in RANGE_KEYS: c.scalar("weapon", w["file"], f"bolt.rangetable.{k}", t["bolt"]["rangetable"][k], (rt or {}).get(k), f"{p}.bolt.rangetable.{k}") def norm_options(v): out = [] for e in L(v): if isinstance(e, dict): out.append({"members": [str(x) for x in L(e.get("option"))], "scalar": False}) else: out.append({"members": [str(e)], "scalar": True}) return out def compare_sections(c, ours, oracle): ob = {s["file"].lower(): s for s in oracle["sections"]} if len(ours) != len(oracle["sections"]): c.diff("sections", f"count {len(ours)} != {len(oracle['sections'])}") for s in ours: p = f"section[{s['file']}]" o = ob.get(s["file"].lower()) if o is None: c.diff(p, "not in oracle") continue for k in ("race", "stem", "id", "display_name", "description", "unlocked_by"): if s[k] != o[k]: c.diff(f"{p}.{k}", f"{s[k]!r} != {o[k]!r}") body = {k: v for k, v in o.items() if k not in SECTION_ID_KEYS} c.exact(s["raw"], body, f"{p}.raw") t = s["typed"] for k in SECTION_TYPED_SCALARS: c.scalar("section", s["file"], k, t[k], body.get(k), f"{p}.{k}") c.strlist(t["requires"], body.get("requires"), f"{p}.requires") c.strlist(t["exclude"], body.get("exclude"), f"{p}.exclude") exp_opts = norm_options(body.get("option")) if t["option"] != exp_opts: c.diff(f"{p}.option", f"{t['option']!r} != {exp_opts!r}") od = body.get("optiondef") exp_od = [str(x) for x in L(od.get("option"))] if isinstance(od, dict) else None if t["optiondef"] != exp_od: c.diff(f"{p}.optiondef", f"{t['optiondef']!r} != {exp_od!r}") banks = L(body.get("bank")) if len(t["bank"]) != len(banks): c.diff(f"{p}.bank", f"count {len(t['bank'])} != {len(banks)}") for i, (tb, ob_) in enumerate(zip(t["bank"], banks)): bp = f"{p}.bank[{i}]" c.scalar("section", s["file"], "bank.turretclass", tb["turretclass"], ob_.get("turretclass"), f"{bp}.turretclass") c.scalar("section", s["file"], "bank.turretsize", tb["turretsize"], ob_.get("turretsize"), f"{bp}.turretsize") c.scalar("section", s["file"], "bank.weapon", tb["weapon"], ob_.get("weapon"), f"{bp}.weapon") c.scalar("section", s["file"], "bank.showturrets", tb["showturrets"], ob_.get("showturrets"), f"{bp}.showturrets") c.scalar("section", s["file"], "bank.invincible", tb["invincible"], ob_.get("invincible"), f"{bp}.invincible") rep = len(L(ob_.get("turretsize"))) > 1 or len(L(ob_.get("turretclass"))) > 1 if tb["repeated_turret_spec"] != rep: c.diff(f"{bp}.repeated_turret_spec", f"{tb['repeated_turret_spec']} != {rep}") mounts = L(ob_.get("mount")) if len(tb["mount"]) != len(mounts): c.diff(f"{bp}.mount", f"count {len(tb['mount'])} != {len(mounts)}") for j, (tm, om) in enumerate(zip(tb["mount"], mounts)): mp = f"{bp}.mount[{j}]" c.scalar("section", s["file"], "mount.node", tm["node"], om.get("node"), f"{mp}.node") for k in MOUNT_KEYS: c.scalar("section", s["file"], f"mount.{k}", tm[k], om.get(k), f"{mp}.{k}") nfl = body.get("netforcelimits") if (t["netforcelimits"] is None) != (nfl is None): c.diff(f"{p}.netforcelimits", "presence mismatch") elif nfl is not None: for k in NFL_KEYS: c.scalar("section", s["file"], f"netforcelimits.{k}", t["netforcelimits"][k], last(nfl).get(k), f"{p}.netforcelimits.{k}") th = L(body.get("thruster")) if len(t["thruster"]) != len(th): c.diff(f"{p}.thruster", f"count {len(t['thruster'])} != {len(th)}") for i, (tt, ot) in enumerate(zip(t["thruster"], th)): for k in ("node", "effect", "idle_effect"): c.scalar("section", s["file"], f"thruster.{k}", tt[k], ot.get(k), f"{p}.thruster[{i}].{k}") def compare_tech(c, ours, oracle): on = {n["name"]: n for n in oracle["nodes"]} if len(ours["nodes"]) != len(oracle["nodes"]): c.diff("tech.nodes", f"count {len(ours['nodes'])} != {len(oracle['nodes'])}") for n in ours["nodes"]: p = f"tech[{n['name']}]" o = on.get(n["name"]) if o is None: c.diff(p, "not in oracle") continue for k in ("display_name", "description", "family", "family_inferred", "type", "threat", "group", "option_cost", "requires", "benefits_inc", "benefits_dec", "sections", "weapons", "allows"): c.checked += 1 a, b = n[k], o[k] if isinstance(a, (int, float)) and isinstance(b, (int, float)) and not isinstance(a, bool): ok = float(a) == float(b) else: ok = a == b if not ok: c.diff(f"{p}.{k}", f"{a!r} != {b!r}") if len(ours["edges"]) != len(oracle["edges"]): c.diff("tech.edges", f"count {len(ours['edges'])} != {len(oracle['edges'])}") for i, (a, b) in enumerate(zip(ours["edges"], oracle["edges"])): c.checked += 1 mine = {"from": a["from"], "to": a["to"], "rp": a["rp"], "pct": a["pct"]} if mine != b: c.diff(f"tech.edges[{i}]", f"{mine!r} != {b!r}") if a["unparsed"]: c.diff(f"tech.edges[{i}]", f"unparsed tokens {a['unparsed']}") for race, v in a["pct_effective"].items(): exp = b["pct"].get(race, 100) if v != exp: c.diff(f"tech.edges[{i}].pct_effective.{race}", f"{v} != {exp}") c.checked += 1 if ours["groups"] != oracle["groups"]: c.diff("tech.groups", f"{ours['groups']!r} != {oracle['groups']!r}") def compare_crosslink(c, ours, dump, oracle): x = ours def pairs(v): return sorted((a, b) for a, b in v) def eq(path, a, b): c.checked += 1 if a != b: c.diff(path, f"{a!r} != {b!r}") eq("crosslink.weapon_requires_dangling", pairs(x["weapon_requires_dangling"]), pairs(oracle["weapon_requires_dangling"])) eq("crosslink.weapon_requires_case_mismatch", pairs(x["weapon_requires_case_mismatch"]), pairs(oracle["weapon_requires_case_mismatch"])) eq("crosslink.weapon_without_requires", sorted(x["weapons_without_requires"]), sorted(oracle["weapon_without_requires"])) eq("crosslink.shipsection_requires_dangling", pairs(x["section_requires_dangling"]), pairs(oracle["shipsection_requires_dangling"])) eq("crosslink.shipsection_requires_case_mismatch", pairs(x["section_requires_case_mismatch"]), pairs(oracle["shipsection_requires_case_mismatch"])) eq("crosslink.shipsection_option_dangling", pairs(x["section_option_dangling"]), pairs(oracle["shipsection_option_dangling"])) scalar = sorted((s["file"], m) for s in dump["sections"] for g in s["typed"]["option"] if g["scalar"] for m in g["members"]) eq("crosslink.shipsection_scalar_option", scalar, pairs(oracle["shipsection_scalar_option"])) eq("crosslink.tech_ship_section_dangling", pairs(x["tech_ship_section_dangling"]), pairs(oracle["tech_ship_section_dangling"])) eq("crosslink.tech_weapon_filename_dangling", pairs(x["tech_weapon_file_dangling"]), pairs(oracle["tech_weapon_filename_dangling"])) eq("crosslink.tech_requires_dangling", pairs(x["tech_requires_dangling"]), pairs(oracle["tech_requires_dangling"])) eq("crosslink.tech_allows_dangling", pairs(x["tech_allows_dangling"]), pairs(oracle["tech_allows_dangling"])) eq("crosslink.tech_allows_unparsed", pairs(x["tech_allows_unparsed"]), pairs(oracle["tech_allows_unparsed"])) eq("crosslink.bank_weapon_dangling", pairs(x["bank_weapon_dangling"]), pairs(oracle["bank_weapon_dangling"])) for scope, om in oracle["manifests"].items(): m = dump["manifests"].get(scope) if m is None: c.diff(f"crosslink.manifests.{scope}", "missing in ours") continue eq(f"crosslink.manifests.{scope}.ids", len(m["ids"]), om["ids"]) eq(f"crosslink.manifests.{scope}.deleted", sorted(m["deleted"]), sorted(om["deleted"])) gaps = sorted(g["name"].lower() for g in x["manifest_ids_without_file"] if g["scope"] == scope) eq(f"crosslink.manifests.{scope}.listed_but_no_file", gaps, sorted(om["listed_but_no_file"])) unl = sorted(r for s_, r in x["files_without_manifest_id"] if s_ == scope) eq(f"crosslink.manifests.{scope}.file_but_unlisted", unl, sorted(om["file_but_unlisted"])) os_ = oracle["strings"] eq("crosslink.strings.missing_techname", sorted(x["missing_techname"]), sorted(os_["missing_techname"])) eq("crosslink.strings.missing_techdesc", sorted(x["missing_techdesc"]), sorted(os_["missing_techdesc"])) eq("crosslink.strings.missing_sectionname", sorted(x["missing_sectionname"]), sorted(os_["missing_sectionname"])) eq("crosslink.strings.missing_sectiondesc", sorted(x["missing_sectiondesc"]), sorted(os_["missing_sectiondesc"])) eq("crosslink.strings.unresolved_weapon_name", pairs(x["unresolved_weapon_names"]), pairs(os_["unresolved_weapon_name"])) ot = oracle["turrets"] eq("crosslink.turrets.turret_rows", len(dump["turrets"]), ot["turret_rows"]) eq("crosslink.turrets.weapon_pairs_without_turret", pairs(x["weapon_turret_pairs_without_row"]), pairs(ot["weapon_size_class_pairs_without_turret"])) eq("crosslink.turrets.bank_pairs_without_turret", pairs(x["bank_turret_pairs_without_row"]), pairs(ot["bank_size_class_pairs_without_turret"])) banks = [b for s in dump["sections"] for b in s["typed"]["bank"]] eq("crosslink.turrets.banks_without_turretsize", sum(1 for b in banks if b["turretsize"] == ""), ot["banks_without_turretsize"]) eq("crosslink.turrets.banks_with_repeated_size_or_class", sum(1 for b in banks if b["repeated_turret_spec"]), ot["banks_with_repeated_size_or_class"]) def compare_strings(c, dump, oracle): c.checked += 1 if dump["string_count"] != len(oracle): c.diff("strings.count", f"{dump['string_count']} != {len(oracle)}") # display names already compared per weapon / section / tech def main(argv): ours = json.load(open(argv[1], encoding="ascii")) odir = argv[2] oracle = {n: json.load(open(os.path.join(odir, n + ".json"), encoding="utf-8")) for n in ("tech_tree", "weapons", "shipsections", "strings", "crosslink")} c = Cmp() compare_weapons(c, ours["weapons"], oracle["weapons"]) compare_sections(c, ours["sections"], oracle["shipsections"]) compare_tech(c, ours["tech"], oracle["tech_tree"]) compare_strings(c, ours, oracle["strings"]) compare_crosslink(c, ours["cross_check"], ours, oracle["crosslink"]) print(f"compared {c.checked} values: {len(ours['weapons'])} weapons, {len(ours['sections'])} sections, " f"{len(ours['tech']['nodes'])} techs, {len(ours['tech']['edges'])} edges, crosslink") rc = 0 if c.untyped != KNOWN_UNTYPED: print("untyped values differ from the documented set:") for u in sorted(c.untyped - KNOWN_UNTYPED, key=str): print(" unexpected:", u) for u in sorted(KNOWN_UNTYPED - c.untyped, key=str): print(" missing: ", u) rc = 1 else: print(f"documented untyped values: {len(c.untyped)} (as expected)") if c.diffs: print(f"DIFF: {len(c.diffs)} differences") for d in c.diffs[:MAX_DIFFS]: print(" ", d) if len(c.diffs) > MAX_DIFFS: print(f" ... {len(c.diffs) - MAX_DIFFS} more") rc = 1 else: print("OK: 100% agreement with the oracle catalogs") return rc if __name__ == "__main__": sys.exit(main(sys.argv))