sots-re/tools/serializers_ghidra.py

#!/usr/bin/env python3
"""Push the recovered layouts into Ghidra so later lanes inherit them.

Reads `objects/layouts.json` + `objects/layouts.h` (produced by
`tools/serializers.py all`) and, through `tools/reva_call.py`:

  * `parse-c-structure` for every emittable struct, dependencies first
  * `create-label`      for every serializer `Write` / `Read` entry point

Idempotent: re-running replaces the structures and re-applies the labels.
Progress is written to `objects/.ghidra_pushed` so an interrupted run resumes.

    uv run python3 tools/serializers_ghidra.py structs
    uv run python3 tools/serializers_ghidra.py labels
"""
import json
import os
import re
import subprocess
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(HERE)
OUT = os.path.join(REPO, "objects")
PROG = "/Sword of the Stars.exe"
STATE = os.path.join(OUT, ".ghidra_pushed")


def reva(tool, payload):
    r = subprocess.run(
        ["uv", "run", "python3", os.path.join(HERE, "reva_call.py"), tool,
         json.dumps(payload)],
        cwd=REPO, capture_output=True, text=True, timeout=180)
    return r.returncode, (r.stdout or r.stderr).strip()


def done_set():
    if not os.path.exists(STATE):
        return set()
    return set(open(STATE).read().split("\n"))


def mark(key):
    with open(STATE, "a") as fh:
        fh.write(key + "\n")


def structs():
    text = open(os.path.join(OUT, "layouts.h")).read()
    defs = re.findall(r"struct \w+ \{.*?\};", text, re.S)
    have = done_set()
    ok = fail = skip = 0
    for d in defs:
        name = d.split()[1]
        key = "struct:" + name
        if key in have:
            skip += 1
            continue
        rc, out = reva("parse-c-structure",
                       {"programPath": PROG, "cDefinition": d})
        if rc == 0 and '"message"' in out:
            ok += 1
            mark(key)
        else:
            fail += 1
            print(f"  FAIL {name}: {out[:160]}")
        if (ok + fail) % 25 == 0:
            print(f"  {ok} ok, {fail} failed, {skip} already there")
    print(f"structures: {ok} pushed, {fail} failed, {skip} skipped")


def labels():
    lay = json.load(open(os.path.join(OUT, "layouts.json")))
    have = done_set()
    ok = fail = skip = 0
    for c, L in sorted(lay.items()):
        if L["grade"] not in ("verified", "clean"):
            continue
        nm = c.replace("::", "_")
        for kind in ("write", "read"):
            va = L.get(kind)
            if not va:
                continue
            label = f"{nm}_{kind.capitalize()}"
            key = f"label:{label}:{va:x}"
            if key in have:
                skip += 1
                continue
            rc, out = reva("create-label",
                           {"programPath": PROG, "labelName": label,
                            "address": f"0x{va:08x}"})
            if rc == 0:
                ok += 1
                mark(key)
            else:
                fail += 1
                print(f"  FAIL {label}: {out[:120]}")
    print(f"labels: {ok} created, {fail} failed, {skip} skipped")


def addresses():
    """Merge the verified-tier serializers and corroborated sizeofs into
    ghidra/addresses.json (then run tools/gen_addresses.py -- never hand-edit
    the header)."""
    path = os.path.join(REPO, "ghidra", "addresses.json")
    j = json.load(open(path))
    base = int(j["image_base"], 16)
    have = {e["name"] for e in j["entries"]}
    lay = json.load(open(os.path.join(OUT, "layouts.json")))
    added = 0
    for c, L in sorted(lay.items()):
        if L["grade"] != "verified":
            continue
        nm = c.replace("::", "_")
        cr = f"{L['read_agree']}/{L['read_comparable']} field offsets agree " \
             f"between Read and Write"
        for kind in ("write", "read"):
            va = L.get(kind)
            name = f"{nm}_{kind.capitalize()}"
            if not va or name in have or not (base <= va < base + 0x1000000):
                continue
            j["entries"].append({
                "name": name, "addr": f"0x{va:08x}", "convention": "thiscall",
                "prototype": f"void ({nm}* this, Mars::Stream* s)  "
                             f"/* IStreamable slot {2 if kind == 'write' else 1}"
                             f", vftable 0x{L['vftable']:08x}, COL offset "
                             f"+0x{L['col_offset']:x}; {len(L['fields'])} "
                             f"member fields; {cr} */",
                "status": "verified",
                "source": "findings/objects/serializer-struct-recovery.md"})
            have.add(name)
            added += 1
        if L["sizeof"]:
            name = f"sizeof_{nm}"
            if name not in have:
                j["entries"].append({
                    "name": name, "offset": f"0x{L['sizeof']:x}",
                    "convention": "layout",
                    "prototype": f"sizeof({c}) -- {L['sizeof_by']}",
                    "status": "verified",
                    "source": "findings/objects/serializer-struct-recovery.md"})
                have.add(name)
                added += 1
    with open(path, "w") as fh:
        json.dump(j, fh, indent=1, ensure_ascii=False)
        fh.write("\n")
    print(f"addresses.json: +{added} entries, {len(j['entries'])} total")


if __name__ == "__main__":
    cmd = sys.argv[1] if len(sys.argv) > 1 else "structs"
    {"structs": structs, "labels": labels, "addresses": addresses}[cmd]()