273 lines
10 KiB
Python
273 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""oracle_parsers.py -- run the proven Python data-file parsers (verify/parsers/)
|
|
over a list of files and emit the CANONICAL JSON form (TRACE_FORMAT.md section 7)
|
|
that a C++ reimplementation's test dump must reproduce byte-for-byte.
|
|
|
|
usage:
|
|
oracle_parsers.py --root EXTRACT_DIR [--out-dir DIR] [--jsonl OUT] [--kind KIND] FILE...
|
|
oracle_parsers.py --root EXTRACT_DIR --all [--out-dir DIR] [--jsonl OUT]
|
|
|
|
--root the .gob extract root; file kinds are chosen from the path relative
|
|
to it exactly as verify.py does (verify.kind_of)
|
|
--all walk --root and take every file with a known data kind
|
|
--out-dir write <out-dir>/<rel>.json per file (canonical text + "\\n")
|
|
--jsonl write one TRACE_FORMAT record per file (hook "parse:<kind>",
|
|
call_id = index in the sorted file list, ret = {"t":"json","v":...})
|
|
-- the golden log for `tracecmp.py --replay IMPL.jsonl OUT.jsonl`
|
|
--kind force a kind for every FILE (brace, effect, csv, manifest, rows, kv)
|
|
With neither --out-dir nor --jsonl, the canonical text of each file goes to stdout.
|
|
|
|
Canonical form (what a C++ dump must produce):
|
|
* JSON object keys sorted by code point; no whitespace; arrays in parse order
|
|
* strings byte-preserving: the raw cp1252 byte b -> code point b; then escaped
|
|
as json.dumps(ensure_ascii=True) does: \\" \\\\ \\n \\r \\t \\b \\f, every other
|
|
unit < 0x20 or >= 0x7f as \\u00xx (lowercase)
|
|
* ints decimal; bools true/false; None -> null; tuples -> arrays;
|
|
Manifest -> {"deleted":[...],"entries":[[id,name],...]}
|
|
* floats: round to float32, print %.9g, append ".0" if no '.', 'e' or 'n';
|
|
non-finite -> "nan" / "inf" / "-inf" (strings)
|
|
* text ends with one "\\n"
|
|
* a file the parser rejects canonicalizes to {"_error":true}
|
|
|
|
Result shapes are the parsers' own (see their docstrings): mars_data dict with
|
|
repeated keys -> lists, "_items" for bare items; effect_txt ordered [key, value]
|
|
pairs; flat_kv dict / rows; manifest as above; csv list of rows.
|
|
|
|
Stdlib only.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import dataclasses
|
|
import json
|
|
import math
|
|
import os
|
|
import struct
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
PARSERS = os.path.normpath(os.path.join(HERE, "..", "..", "parsers"))
|
|
sys.path.insert(0, PARSERS)
|
|
import effect_txt # noqa: E402
|
|
import flat_kv # noqa: E402
|
|
import manifest # noqa: E402
|
|
import mars_data # noqa: E402
|
|
import verify as _verify # noqa: E402 (kind_of; main is __main__-guarded)
|
|
|
|
kind_of = _verify.kind_of
|
|
KINDS = ("brace", "effect", "csv", "manifest", "rows", "kv")
|
|
SKIP_KINDS = ("hlsl", "prose", "other")
|
|
|
|
|
|
# --- canonical form --------------------------------------------------------------------
|
|
|
|
def canon_float(x: float) -> str:
|
|
if math.isnan(x):
|
|
return '"nan"'
|
|
if math.isinf(x):
|
|
return '"inf"' if x > 0 else '"-inf"'
|
|
x = struct.unpack("<f", struct.pack("<f", x))[0]
|
|
s = "%.9g" % x
|
|
if not any(c in s for c in ".en"):
|
|
s += ".0"
|
|
return s
|
|
|
|
|
|
def canon_str(s: str) -> str:
|
|
"""Byte-preserving: re-encode the parser's cp1252 text so each byte is one
|
|
code point, then escape like json.dumps(ensure_ascii=True)."""
|
|
try:
|
|
b = s.encode("cp1252")
|
|
except UnicodeEncodeError:
|
|
b = s.encode("latin-1", "replace")
|
|
return json.dumps(b.decode("latin-1"), ensure_ascii=True)
|
|
|
|
|
|
def canonical(obj):
|
|
"""Normalize a parser result to plain JSON-able Python (floats stay float,
|
|
strings become byte-mapped, tuples -> lists, dataclasses -> dicts)."""
|
|
if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
|
|
if isinstance(obj, manifest.Manifest):
|
|
return {"entries": [[i, canonical(n)] for i, n in obj.entries],
|
|
"deleted": list(obj.deleted)}
|
|
return canonical(dataclasses.asdict(obj))
|
|
if isinstance(obj, dict):
|
|
return {str(k): canonical(v) for k, v in obj.items()}
|
|
if isinstance(obj, (list, tuple)):
|
|
return [canonical(v) for v in obj]
|
|
if isinstance(obj, bool) or obj is None or isinstance(obj, int):
|
|
return obj
|
|
if isinstance(obj, float):
|
|
if math.isnan(obj):
|
|
return "nan"
|
|
if math.isinf(obj):
|
|
return "inf" if obj > 0 else "-inf"
|
|
return struct.unpack("<f", struct.pack("<f", obj))[0]
|
|
if isinstance(obj, str):
|
|
try:
|
|
return obj.encode("cp1252").decode("latin-1")
|
|
except UnicodeEncodeError:
|
|
return obj.encode("latin-1", "replace").decode("latin-1")
|
|
raise TypeError(f"cannot canonicalize {type(obj).__name__}")
|
|
|
|
|
|
def dumps(obj) -> str:
|
|
"""Canonical text (no trailing newline) of an already-canonical() object."""
|
|
out: list[str] = []
|
|
_emit(obj, out)
|
|
return "".join(out)
|
|
|
|
|
|
def _emit(x, out: list[str]) -> None:
|
|
if x is None:
|
|
out.append("null")
|
|
elif isinstance(x, bool):
|
|
out.append("true" if x else "false")
|
|
elif isinstance(x, int):
|
|
out.append("%d" % x)
|
|
elif isinstance(x, float):
|
|
out.append(canon_float(x))
|
|
elif isinstance(x, str):
|
|
out.append(json.dumps(x, ensure_ascii=True))
|
|
elif isinstance(x, list):
|
|
out.append("[")
|
|
for i, e in enumerate(x):
|
|
if i:
|
|
out.append(",")
|
|
_emit(e, out)
|
|
out.append("]")
|
|
elif isinstance(x, dict):
|
|
out.append("{")
|
|
for i, k in enumerate(sorted(x)):
|
|
if i:
|
|
out.append(",")
|
|
out.append(json.dumps(k, ensure_ascii=True))
|
|
out.append(":")
|
|
_emit(x[k], out)
|
|
out.append("}")
|
|
else:
|
|
raise TypeError(type(x).__name__)
|
|
|
|
|
|
# --- parsing dispatch ---------------------------------------------------------------------
|
|
|
|
def parse_kind(path: str, kind: str) -> tuple[object, list[str]]:
|
|
"""-> (result, notes). Raises on parse failure."""
|
|
notes: list[str] = []
|
|
base = kind.split(":")[0]
|
|
if base == "brace":
|
|
w: list[str] = []
|
|
obj = mars_data.parse_file(path, warnings=w)
|
|
notes += w
|
|
elif base == "effect":
|
|
obj = effect_txt.parse_file(path)
|
|
elif base == "csv":
|
|
obj = manifest.parse_csv_file(path)
|
|
elif base == "manifest":
|
|
obj = manifest.parse_manifest_file(path)
|
|
if obj.problems:
|
|
raise ValueError("; ".join(obj.problems))
|
|
elif base == "rows":
|
|
obj = flat_kv.parse_rows_file(path)
|
|
elif base == "kv":
|
|
txt = manifest.read_text(path)
|
|
obj = flat_kv.parse_kv(txt)
|
|
d = flat_kv.duplicates(txt)
|
|
if d:
|
|
notes.append(f"duplicate keys {d}")
|
|
else:
|
|
raise ValueError(f"no parser for kind {kind!r}")
|
|
return obj, notes
|
|
|
|
|
|
def oracle_file(path: str, kind: str) -> tuple[dict | list, list[str], str | None]:
|
|
"""-> (canonical object, notes, error). Never raises for parse errors."""
|
|
try:
|
|
obj, notes = parse_kind(path, kind)
|
|
except Exception as e: # noqa: BLE001
|
|
return {"_error": True}, [], f"{type(e).__name__}: {e}"
|
|
return canonical(obj), notes, None
|
|
|
|
|
|
def collect(root: str, files: list[str], all_files: bool, forced_kind: str | None) -> list[tuple[str, str, str]]:
|
|
"""-> sorted [(abs path, rel, kind)] with data kinds only."""
|
|
items = []
|
|
if all_files:
|
|
for dp, _, fn in os.walk(root):
|
|
for f in fn:
|
|
items.append(os.path.join(dp, f))
|
|
items += files
|
|
out = []
|
|
for p in items:
|
|
p = os.path.abspath(p)
|
|
rel = os.path.relpath(p, root).replace(os.sep, "/") if root else os.path.basename(p)
|
|
k = forced_kind or kind_of(rel)
|
|
if k.split(":")[0] in SKIP_KINDS:
|
|
continue
|
|
out.append((p, rel, k))
|
|
out.sort(key=lambda t: t[1])
|
|
return out
|
|
|
|
|
|
def jsonl_record(idx: int, rel: str, kind: str, canon, notes: list[str], err: str | None) -> str:
|
|
"""One TRACE_FORMAT record (json.dumps is spec-conformant for our values)."""
|
|
rec = {"ts": idx, "hook": "parse:" + kind.split(":")[0], "mode": "trace", "call_id": idx, "thread": 0,
|
|
"args": [{"t": "str", "v": rel.encode("cp1252", "replace").decode("latin-1"), "n": "path"}],
|
|
"ret": {"t": "json", "v": canon}, "side": {}}
|
|
if err:
|
|
rec["err"] = err
|
|
if notes:
|
|
rec["note"] = "; ".join(notes)
|
|
return json.dumps(rec, ensure_ascii=True, separators=(",", ":"), allow_nan=False) + "\n"
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("files", nargs="*", metavar="FILE")
|
|
ap.add_argument("--root", default=None, help=".gob extract root (kinds from relative path)")
|
|
ap.add_argument("--all", action="store_true", help="walk --root for every data file")
|
|
ap.add_argument("--out-dir")
|
|
ap.add_argument("--jsonl")
|
|
ap.add_argument("--kind", choices=KINDS)
|
|
a = ap.parse_args(argv)
|
|
if a.all and not a.root:
|
|
ap.error("--all needs --root")
|
|
if not a.all and not a.files:
|
|
ap.error("no FILE given (or use --all)")
|
|
root = os.path.abspath(a.root) if a.root else None
|
|
items = collect(root, a.files, a.all, a.kind)
|
|
if not items:
|
|
print("error: no data files selected", file=sys.stderr)
|
|
return 2
|
|
|
|
jf = None
|
|
if a.jsonl:
|
|
os.makedirs(os.path.dirname(os.path.abspath(a.jsonl)), exist_ok=True)
|
|
jf = open(a.jsonl, "w", encoding="utf-8", newline="\n")
|
|
jf.write(json.dumps({"meta": {"format": 1, "build": "oracle_parsers", "started": "", "inline_max": 0,
|
|
"hooks": {}, "root": root or ""}}, separators=(",", ":")) + "\n")
|
|
errors = 0
|
|
for idx, (p, rel, k) in enumerate(items):
|
|
canon, notes, err = oracle_file(p, k)
|
|
if err:
|
|
errors += 1
|
|
print(f"error: {rel}: {err}", file=sys.stderr)
|
|
text = dumps(canon) + "\n"
|
|
if a.out_dir:
|
|
op = os.path.join(a.out_dir, rel + ".json")
|
|
os.makedirs(os.path.dirname(op), exist_ok=True)
|
|
with open(op, "w", encoding="utf-8", newline="\n") as f:
|
|
f.write(text)
|
|
if jf:
|
|
jf.write(jsonl_record(idx, rel, k, canon, notes, err))
|
|
if not a.out_dir and not jf:
|
|
sys.stdout.write(text)
|
|
if jf:
|
|
jf.close()
|
|
print(f"{len(items)} file(s), {errors} parse error(s)"
|
|
+ (f" -> {a.out_dir}" if a.out_dir else "") + (f" -> {a.jsonl}" if a.jsonl else ""), file=sys.stderr)
|
|
return 0 if not errors else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|