126 lines
4.3 KiB
Python
Executable file
126 lines
4.3 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Oracle dump for the mars_text cross-check.
|
|
|
|
dump.py list <gob-extract-dir> -> "<kind>\t<relpath>" per file handled by mars_text
|
|
dump.py <kind> <file> -> JSON of the reference reader's result
|
|
|
|
The reference readers live in the RE repo (sots-re/verify/parsers); point
|
|
SOTS_RE_PARSERS at that directory (default ~/sots-re/verify/parsers).
|
|
Strings are re-encoded to cp1252 bytes and emitted one \\u00XX escape per
|
|
byte so the JSON matches what dump_json.cpp writes for raw bytes.
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
sys.path.insert(0, os.environ.get("SOTS_RE_PARSERS", os.path.expanduser("~/sots-re/verify/parsers")))
|
|
import flat_kv # noqa: E402
|
|
import manifest # noqa: E402
|
|
|
|
# Same classification as the RE repo's verify.py: everything textual that is
|
|
# not brace-block, effect, shader or prose.
|
|
BRACE_TXT = {"Data/tutorial.txt", "Data/credits.txt", "Data/Strategy/systemnames.txt",
|
|
"Data/Combat/ctechvars.txt", "Data/Combat/shipai.txt",
|
|
"Models/Skysphere/skydefs.txt", "Models/Skysphere/NodeSpace-skydefs.txt"}
|
|
ROWS_TXT = {"Weapons/_turrets.txt", "Weapons/_defaultweapons.txt", "Data/Combat/damfx.txt",
|
|
"Data/Combat/damfx_levels.txt", "Data/Strategy/playercolors.txt",
|
|
"Badges/BadgeTable.txt", "Avatars/AvatarTable.txt", "GUI/WeaponIconPlacements.txt"}
|
|
PROSE = {"Locale/EN/ChatTrans.txt"}
|
|
|
|
|
|
def kind_of(rel):
|
|
ext = rel.rsplit(".", 1)[-1].lower()
|
|
base = os.path.basename(rel)
|
|
if ext == "csv":
|
|
return "csv"
|
|
if ext == "txt":
|
|
if base in ("_weapons.txt", "_shipsections.txt"):
|
|
return "manifest"
|
|
if rel.startswith("Scenarios/") or rel in BRACE_TXT:
|
|
return None
|
|
if rel in ROWS_TXT:
|
|
return "rows"
|
|
if rel.startswith("Locale/EN/Desc") or rel in PROSE:
|
|
return None
|
|
return "kv"
|
|
return None
|
|
|
|
|
|
def list_files(root):
|
|
out = []
|
|
for dirpath, _, names in os.walk(root):
|
|
for n in names:
|
|
rel = os.path.relpath(os.path.join(dirpath, n), root).replace(os.sep, "/")
|
|
k = kind_of(rel)
|
|
if k:
|
|
out.append((k, rel))
|
|
return sorted(out)
|
|
|
|
|
|
def bytes_str(s):
|
|
"""str decoded from cp1252 -> str whose code points are the original bytes."""
|
|
return s.encode("cp1252").decode("latin-1")
|
|
|
|
|
|
def norm(v):
|
|
if isinstance(v, str):
|
|
return bytes_str(v)
|
|
if isinstance(v, list):
|
|
return [norm(x) for x in v]
|
|
if isinstance(v, tuple):
|
|
return [norm(x) for x in v]
|
|
return v
|
|
|
|
|
|
_PROBLEM_RES = [
|
|
(re.compile(r"^line (\d+): unrecognised "), "unrecognised"),
|
|
(re.compile(r"^line (\d+): duplicate id (\d+) "), "duplicate_id"),
|
|
(re.compile(r"^id (\d+) is both DELETED and assigned"), "deleted_and_assigned"),
|
|
]
|
|
|
|
|
|
def problem_obj(msg):
|
|
for rx, kind in _PROBLEM_RES:
|
|
m = rx.match(msg)
|
|
if m:
|
|
if kind == "unrecognised":
|
|
return {"line": int(m.group(1)), "kind": kind, "id": None}
|
|
if kind == "duplicate_id":
|
|
return {"line": int(m.group(1)), "kind": kind, "id": int(m.group(2))}
|
|
return {"line": None, "kind": kind, "id": int(m.group(1))}
|
|
return {"line": None, "kind": "unknown:" + msg, "id": None}
|
|
|
|
|
|
def dump(kind, path):
|
|
text = manifest.read_text(path)
|
|
if kind == "kv":
|
|
d = flat_kv.parse_kv(text)
|
|
return {"values": {bytes_str(k): norm(v) for k, v in d.items()},
|
|
"duplicates": {bytes_str(k): v for k, v in flat_kv.duplicates(text).items()}}
|
|
if kind == "rows":
|
|
return {"rows": norm(flat_kv.parse_rows(text))}
|
|
if kind == "manifest":
|
|
m = manifest.parse_manifest(text)
|
|
return {"entries": [[i, bytes_str(n)] for i, n in m.entries],
|
|
"deleted": list(m.deleted),
|
|
"problems": [problem_obj(p) for p in m.problems]}
|
|
if kind == "csv":
|
|
header, rows = manifest.parse_csv_with_header(text)
|
|
return {"header": norm(header) if header is not None else None, "rows": norm(rows)}
|
|
raise SystemExit(f"unknown kind {kind}")
|
|
|
|
|
|
def main(argv):
|
|
if len(argv) != 3:
|
|
raise SystemExit(__doc__)
|
|
if argv[1] == "list":
|
|
for k, rel in list_files(argv[2]):
|
|
print(f"{k}\t{rel}")
|
|
return
|
|
json.dump(dump(argv[1], argv[2]), sys.stdout, ensure_ascii=True)
|
|
print()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv)
|