120 lines
4.3 KiB
Python
120 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Emit the reference (sots-re) parse of every brace-block and .effect data
|
|
file as canonical JSON, one file per input, for tests/mars_parse/oracle_check.
|
|
|
|
dump.py <data-dir> <out-dir> [--parsers <dir>]
|
|
|
|
<data-dir> an extracted sots.gob tree (the owner's copy, never committed)
|
|
<out-dir> receives <rel>.json per input plus index.txt:
|
|
<kind> TAB <warning-count> TAB <rel-path>
|
|
--parsers directory holding the reference readers mars_data.py /
|
|
effect_txt.py / flat_kv.py / verify.py (default:
|
|
$SOTS_RE_PARSERS, else ~/sots-re/verify/parsers)
|
|
|
|
Canonical form (mirrored by tests/mars_parse/canon.cpp):
|
|
* brace files: the reader's dict, json.dumps(sort_keys=True, compact,
|
|
ensure_ascii=True); floats rendered as "\\x01" + "%.17g" strings so both
|
|
sides format through the same printf rules.
|
|
* effect files: the reader's ordered [[key, value], ...] list, same rules.
|
|
* strings are the file's cp1252 bytes, one \\u00xx per byte >= 0x7f.
|
|
|
|
Exit status 3 when the reference parsers cannot be found (the caller then runs
|
|
the C++ side in count-only mode).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
|
|
def find_parsers(explicit: str | None) -> str | None:
|
|
cands = [explicit, os.environ.get("SOTS_RE_PARSERS"),
|
|
os.path.expanduser("~/sots-re/verify/parsers")]
|
|
for c in cands:
|
|
if c and os.path.isfile(os.path.join(c, "mars_data.py")):
|
|
return c
|
|
return None
|
|
|
|
|
|
class Canon(json.JSONEncoder):
|
|
"""Floats -> marker string; str -> cp1252 bytes as latin-1 code points."""
|
|
|
|
def default(self, o): # pragma: no cover - not reached for our types
|
|
return super().default(o)
|
|
|
|
def iterencode(self, o, _one_shot=False):
|
|
return super().iterencode(self._fix(o), _one_shot)
|
|
|
|
def _fix(self, o):
|
|
if isinstance(o, bool):
|
|
return o
|
|
if isinstance(o, float):
|
|
return "\x01" + ("%.17g" % o)
|
|
if isinstance(o, str):
|
|
return o.encode("cp1252").decode("latin-1")
|
|
if isinstance(o, dict):
|
|
return {self._fix(k): self._fix(v) for k, v in o.items()}
|
|
if isinstance(o, (list, tuple)):
|
|
return [self._fix(v) for v in o]
|
|
return o
|
|
|
|
|
|
def dumps(obj) -> str:
|
|
return json.dumps(obj, cls=Canon, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
args = [a for a in argv[1:]]
|
|
parsers = None
|
|
if "--parsers" in args:
|
|
i = args.index("--parsers")
|
|
parsers = args[i + 1]
|
|
del args[i:i + 2]
|
|
if len(args) != 2:
|
|
print(__doc__, file=sys.stderr)
|
|
return 2
|
|
data_dir, out_dir = args
|
|
pdir = find_parsers(parsers)
|
|
if pdir is None:
|
|
print("dump.py: reference parsers not found (set SOTS_RE_PARSERS)", file=sys.stderr)
|
|
return 3
|
|
sys.path.insert(0, pdir)
|
|
import effect_txt # noqa: E402
|
|
import mars_data # noqa: E402
|
|
import verify # noqa: E402 (for kind_of: which .txt files are brace-form)
|
|
|
|
index = []
|
|
counts: dict[str, int] = {}
|
|
for dp, _, fns in os.walk(data_dir):
|
|
for fn in sorted(fns):
|
|
path = os.path.join(dp, fn)
|
|
rel = os.path.relpath(path, data_dir).replace(os.sep, "/")
|
|
kind = verify.kind_of(rel)
|
|
if kind.startswith("brace:"):
|
|
warnings: list = []
|
|
obj = mars_data.parse_file(path, warnings=warnings)
|
|
nwarn = len(warnings)
|
|
elif kind == "effect":
|
|
obj = effect_txt.parse_file(path)
|
|
nwarn = 0
|
|
else:
|
|
continue
|
|
out_path = os.path.join(out_dir, rel + ".json")
|
|
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
|
with open(out_path, "w", encoding="ascii", newline="\n") as f:
|
|
f.write(dumps(obj))
|
|
index.append(f"{kind}\t{nwarn}\t{rel}")
|
|
counts[kind] = counts.get(kind, 0) + 1
|
|
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
with open(os.path.join(out_dir, "index.txt"), "w", encoding="utf-8", newline="\n") as f:
|
|
f.write("\n".join(index) + ("\n" if index else ""))
|
|
for k in sorted(counts):
|
|
print(f"oracle {k}: {counts[k]}")
|
|
print(f"oracle total: {sum(counts.values())} files -> {out_dir}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|