#!/usr/bin/env python3
"""Automated struct recovery from the `Mars::IStreamable` serializers.

The idea
--------
Lane S's rule: never size a struct member from the offsets the code *touches* --
size it from an **enumeration**.  `std::allocator<char>` is an empty class; it
occupies a word and is never loaded or stored, so a touch-scan undercounts every
`std::string` by exactly 4 and every `std::vector` by exactly 4.  An enumeration
can show *absence*; a touch-scan cannot.

Every serializable class in this exe carries such an enumeration: its
`Write(Stream&)`, which walks the class's fields in order, each with a 4-char
name tag.  `ObservedTech::Write` @0x00817cf0 gave the complete field list, tags
and all, and it matched `save_reader.py`'s on-disk order exactly.  This tool does
that mechanically for every serializer in the binary.

The idioms it decodes
---------------------
Writer side, two calling conventions, one shape:

    push 0xff                       ; default-value argument
    <member expression>             ; lea r,[this+D] | mov r,[this+D] | movzx ...
    push r
    push <tag>                      ; -> .rdata "otch", "pswd", ...
    push <stream>  /  mov ecx,<stream>
    call WriteString / WriteBool / ... | call [streamvft+slot]

so the push immediately before the tag push is always the member.  Wrapper
helpers are direct `call rel32`; the compiler also inlines them as
`call [[stream]+slot]`, and the slot is what names the type:

    +0x18 string   +0x1c bool   +0x20 float   +0x24 int   +0x28 nested
    +0x30 raw bytes (n=8 -> int64)

Nested members go through a `StreamableHelper<T>` / `VectorHelper<T>` built on
the stack as `{vptr, 0, T*}`; the vptr is an RTTI'd vftable, so the *type* of the
nested member is recovered by name, not guessed.

Sizes are then assigned from the kind, never from the touched offsets:
    bool 1 | int16 2 | int/float/enum/handle 4 | int64 8
    std::string 0x1c  (allocator-last, invisible to a touch scan)
    std::vector 0x10  (allocator-last, likewise)
    nested T    sizeof(T), resolved recursively when T is itself serializable

Usage
-----
    uv run python3 tools/serializers.py find          # enumerate serializers
    uv run python3 tools/serializers.py dump 0x00817cf0
    uv run python3 tools/serializers.py validate      # known layouts, first
    uv run python3 tools/serializers.py all           # -> objects/layouts.json
"""
import bisect
import json
import os
import struct
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(HERE)
sys.path.insert(0, HERE)
import x86disp                                              # noqa: E402
from x86disp import decode, Desync, EXE, FUNCS              # noqa: E402
from rtti_map import Image, load as load_rtti               # noqa: E402

OUT = os.path.join(REPO, "objects")

R32 = ["eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi"]
PREFIX = {0x66, 0x67, 0xF0, 0xF2, 0xF3, 0x2E, 0x36, 0x3E, 0x26, 0x64, 0x65}

# Stream vftable slots (derived by disassembling the wrappers below, not assumed)
SLOT_KIND = {0x18: "string", 0x1c: "bool", 0x20: "float", 0x24: "int",
             0x28: "nested", 0x2c: "nested2", 0x30: "raw"}

# Wrapper helpers: VA -> (kind, fixed width or None)
HELPERS = {
    0x008b9d70: ("string", 0x1c), 0x008b9d90: ("string", 0x1c),
    0x008b9c20: ("bool", 1),      0x008b9c00: ("bool", 1),
    0x008b9be0: ("float", 4),     0x008b9bc0: ("float", 4),
    0x008b9d50: ("int", 4),       0x008b9d20: ("int", 4),
    0x008b9d00: ("int16", 2),     0x008b9cd0: ("int16", 2),
    0x008b9c60: ("int64", 8),     0x008b9c40: ("int64", 8),
    0x00816490: ("handle", 4),    0x008164d0: ("handle", 4),
}
READ_HELPERS = {0x008b9d90, 0x008b9c00, 0x008b9bc0, 0x008b9d20,
                0x008b9cd0, 0x008b9c40, 0x008164d0}
KIND_SIZE = {"string": 0x1c, "bool": 1, "float": 4, "int": 4, "int16": 2,
             "int64": 8, "handle": 4, "enum": 4, "vector": 0x10}


# ------------------------------------------------------------------ anatomy
def _has_modrm(raw):
    """Return (op2, opcode, modrm_index or None) for one instruction's bytes."""
    j = 0
    while j < len(raw) and raw[j] in PREFIX:
        j += 1
    if j >= len(raw):
        return False, None, None
    op2 = False
    op = raw[j]
    j += 1
    if op == 0x0F:
        op2 = True
        op = raw[j]
        j += 1
        ent = x86disp._TWO.get(op)
    else:
        ent = x86disp._ONE.get(op)
    if ent is None:
        return op2, op, None
    return op2, op, (j if ent[0] else None)


class Insn:
    __slots__ = ("va", "raw", "info", "op", "op2", "modrm", "mod", "reg", "rm")

    def __init__(self, va, raw, info):
        self.va, self.raw, self.info = va, raw, info
        self.op2, self.op, mi = _has_modrm(raw)
        if mi is not None and mi < len(raw):
            self.modrm = raw[mi]
            self.mod = self.modrm >> 6
            self.reg = (self.modrm >> 3) & 7
            self.rm = self.modrm & 7
            # x86disp only reports operands that carry a *displacement*, so a
            # `mov eax,[edi]` -- member at offset 0 -- is invisible to it.  For
            # a plain struct (`.?AU...`, no vptr) offset 0 is a real field, so
            # synthesise the mod=0 case here.
            if self.info is None and self.mod == 0 and self.rm != 5:
                if self.rm == 4:
                    sib = raw[mi + 1] if mi + 1 < len(raw) else None
                    if sib is not None and (sib & 7) != 5:
                        idx = (sib >> 3) & 7
                        self.info = {"base": R32[sib & 7],
                                     "index": None if idx == 4 else R32[idx],
                                     "scale": 1 << (sib >> 6), "disp": 0,
                                     "dispsize": 0, "reg": R32[self.reg],
                                     "lea": (not self.op2 and self.op == 0x8D),
                                     "mnem": ""}
                else:
                    self.info = {"base": R32[self.rm], "index": None,
                                 "scale": 1, "disp": 0, "dispsize": 0,
                                 "reg": R32[self.reg],
                                 "lea": (not self.op2 and self.op == 0x8D),
                                 "mnem": ""}
        else:
            self.modrm = self.mod = self.reg = self.rm = None


def body(img, funcs_sorted, starts, fva):
    """Decode a function body, sweeping to the next function start."""
    j = bisect.bisect_right(starts, fva)
    limit = starts[j] if j < len(starts) else None
    buf, off, _ = img.find(fva)
    if buf is None:
        return []
    end = len(buf) if limit is None else min(len(buf), off + (limit - fva))
    out, i = [], off
    while i < end:
        try:
            ln, info = decode(buf, i, end)
        except Desync:
            break
        out.append(Insn(fva + (i - off), buf[i:i + ln], info))
        i += ln
        if len(out) > 60000:
            break
    return out


# -------------------------------------------------------------- extraction
class Extractor:
    def __init__(self, img, rtti_vfts, fnames, funcs_sorted, starts):
        self.img = img
        self.vfts = rtti_vfts
        self.fnames = fnames
        self.funcs = funcs_sorted
        self.starts = starts
        self.extra = {}          # discovered forwarding sub-writers

    # -- tag strings ------------------------------------------------------
    def tag(self, va):
        if va == 0:
            return "."
        buf, o, sec = self.img.find(va)
        if buf is None or sec not in (".rdata", ".data"):
            return None
        e = buf.find(b"\0", o, o + 24)
        if e < 0:
            return None
        s = buf[o:e]
        if not (1 <= len(s) <= 16):
            return None
        try:
            t = s.decode("ascii")
        except UnicodeDecodeError:
            return None
        return t if t.isprintable() and not t.isspace() else None

    # -- one serializer ---------------------------------------------------
    def run(self, fva, debug=False):
        ins = body(self.img, self.funcs, self.starts, fva)
        if not ins:
            return None
        n = len(ins)

        this_regs = {"ecx"}
        this_spill = set()
        regdef = {}          # reg -> record
        stkdef = {}          # (base, disp) -> record
        prov = {}            # reg -> (this-base, disp, insn idx)
        pushes = []          # records since the last call
        side = [0, 0]        # [write-side calls, read-side calls]
        containers = {}      # (base, disp) -> element stride
        fields = []
        calls = []           # non-helper calls (base-class Write, etc.)
        notes = []

        def mem_rec(info, width, kind):
            b = info["base"]
            return {"k": kind, "base": b, "disp": info["disp"], "width": width,
                    "this": b in this_regs, "index": info["index"]}

        for k, x in enumerate(ins):
            info, raw = x.info, x.raw
            op, op2, mod = x.op, x.op2, x.mod

            # ---- calls -------------------------------------------------
            is_call = (op == 0xE8 and not op2) or \
                      (op == 0xFF and not op2 and x.reg == 2)
            if is_call:
                self._call(x, k, ins, pushes, regdef, stkdef, fields, calls,
                           this_regs, notes, debug, prov, k, side)
                pushes = []
                for r in ("eax", "ecx", "edx"):
                    regdef.pop(r, None)
                    this_regs.discard(r)
                    prov.pop(r, None)
                continue

            # ---- pushes ------------------------------------------------
            if not op2 and op == 0x68 and len(raw) == 5:
                pushes.append({"k": "imm",
                               "v": struct.unpack_from("<I", raw, 1)[0]})
                continue
            if not op2 and op == 0x6A:
                pushes.append({"k": "imm",
                               "v": struct.unpack_from("<b", raw, 1)[0] & 0xFFFFFFFF})
                continue
            if not op2 and 0x50 <= op <= 0x57 and len(raw) == 1:
                r = R32[op - 0x50]
                if r in this_regs:
                    # the member is at offset 0, so the compiler pushed `this`
                    # itself rather than emitting `lea r,[r+0]`
                    pushes.append({"k": "addr", "base": r, "disp": 0,
                                   "this": True, "width": None, "reg": r})
                else:
                    pushes.append(dict(regdef.get(r, {"k": "?", "reg": r}),
                                       reg=r))
                continue
            if not op2 and op == 0xFF and x.reg == 6:
                if info:
                    key = (info["base"], info["disp"])
                    if info["base"] in ("ebp", "esp") and key in stkdef:
                        pushes.append(dict(stkdef[key]))
                    else:
                        pushes.append(mem_rec(info, 4, "load"))
                else:
                    pushes.append({"k": "?"})
                continue
            if not op2 and op == 0x8D and info:            # lea
                dst = R32[x.reg]
                regdef[dst] = mem_rec(info, None, "addr")
                if info["base"] in prov and info["base"] not in this_regs:
                    regdef[dst]["prov"] = prov[info["base"]][:2]
                if dst != info["base"]:
                    prov.pop(dst, None)
                this_regs.discard(dst)
                continue

            # ---- register / stack definitions ---------------------------
            if not op2 and op == 0x8B:                     # mov r, rm
                dst = R32[x.reg]
                if mod == 3:
                    src = R32[x.rm]
                    if src in this_regs:
                        this_regs.add(dst)
                    else:
                        this_regs.discard(dst)
                    if src in regdef:
                        regdef[dst] = regdef[src]
                    else:
                        regdef.pop(dst, None)
                elif info:
                    key = (info["base"], info["disp"])
                    # `mov edi,[edi+0x1c0]` overwrites the this-register with a
                    # pointer member -- so the base's this-ness has to be read
                    # BEFORE the destination is retired, or the field is lost.
                    base_this = info["base"] in this_regs
                    base_prov = prov.get(info["base"])
                    if key in this_spill:
                        this_regs.add(dst)
                    else:
                        this_regs.discard(dst)
                    if info["base"] in ("ebp", "esp") and key in stkdef:
                        regdef[dst] = stkdef[key]
                    else:
                        regdef[dst] = mem_rec(info, 4, "load")
                        regdef[dst]["this"] = base_this
                        if base_prov and not base_this:
                            # loading *through* a pointer member: the value
                            # serialised is `member->id` / `member->name`, so
                            # the member is the pointer at the remembered
                            # this-offset.  This is the `NetworkObject id @+4`
                            # handle idiom, and it is why a plain "which offset
                            # was pushed" read misses every handle field.
                            regdef[dst]["prov"] = base_prov[:2]
                    r = regdef.get(dst, {})
                    if r.get("k") == "load" and r.get("this"):
                        prov[dst] = (r["base"], r["disp"], k)
                    elif "prov" not in r:
                        prov.pop(dst, None)
                else:
                    regdef.pop(dst, None)
                    this_regs.discard(dst)
                continue
            if op2 and op in (0xB6, 0xB7, 0xBE, 0xBF) and info:   # movzx/movsx
                dst = R32[x.reg]
                w = 1 if op in (0xB6, 0xBE) else 2
                regdef[dst] = mem_rec(info, w, "load")
                regdef[dst]["signed"] = op in (0xBE, 0xBF)
                this_regs.discard(dst)
                continue
            if not op2 and 0xB8 <= op <= 0xBF and len(raw) >= 5:  # mov r,imm32
                dst = R32[op - 0xB8]
                regdef[dst] = {"k": "imm",
                               "v": struct.unpack_from("<I", raw, len(raw) - 4)[0]}
                this_regs.discard(dst)
                continue
            if not op2 and op == 0x33 and mod == 3 and x.reg == x.rm:  # xor r,r
                dst = R32[x.reg]
                regdef[dst] = {"k": "imm", "v": 0}
                this_regs.discard(dst)
                continue
            if not op2 and op == 0x89:                     # mov rm, r
                src = R32[x.reg]
                if mod == 3:
                    dst = R32[x.rm]
                    if src in this_regs:
                        this_regs.add(dst)
                    else:
                        this_regs.discard(dst)
                    if src in regdef:
                        regdef[dst] = regdef[src]
                    else:
                        regdef.pop(dst, None)
                elif info:
                    key = (info["base"], info["disp"])
                    if info["base"] in ("ebp", "esp"):
                        stkdef[key] = dict(regdef.get(src, {"k": "?"}))
                        if src in this_regs:
                            this_spill.add(key)
                        else:
                            this_spill.discard(key)
                continue
            if not op2 and op == 0xC7 and info and mod != 3:   # mov rm, imm32
                key = (info["base"], info["disp"])
                if info["base"] in ("ebp", "esp") and len(raw) >= 4:
                    stkdef[key] = {"k": "imm",
                                   "v": struct.unpack_from("<I", raw, len(raw) - 4)[0]}
                continue

            # ---- container size: (_Mylast - _Myfirst) / stride -----------
            # A count argument is never a member; it is computed from the
            # container.  Recovering the *container's* offset from the span
            # subtraction is what turns "NumFlts" into a real vector member.
            if not op2 and op == 0x2B:                     # sub r32, rm32
                dst = R32[x.reg]
                d0 = regdef.get(dst)
                span = None
                if mod == 3:
                    d1 = regdef.get(R32[x.rm])
                    if d0 and d1 and d0.get("k") == "load" and \
                            d1.get("k") == "load" and d0.get("base") and \
                            d0["base"] == d1["base"] and \
                            d0["disp"] == d1["disp"] + 4:
                        span = (d1["base"], d1["disp"], d1.get("this"))
                elif info and d0 and d0.get("k") == "load" and \
                        d0.get("base") == info["base"] and \
                        d0.get("disp") == info["disp"] + 4:
                    span = (info["base"], info["disp"], info["base"] in this_regs)
                if span:
                    regdef[dst] = {"k": "vecspan", "base": span[0],
                                   "disp": span[1], "this": span[2],
                                   "width": 0x10, "magic": None, "shift": None}
                else:
                    regdef.pop(dst, None)
                this_regs.discard(dst)
                continue
            if not op2 and op == 0xF7 and mod == 3 and x.reg in (4, 5):
                # mul/imul rm -> edx:eax.  The MSVC magic-number divide by a
                # non-power-of-two stride runs the span through this.
                src = regdef.get(R32[x.rm], {})
                if src.get("k") == "vecspan":
                    m = regdef.get("eax", {})
                    src = dict(src)
                    if m.get("k") == "imm":
                        src["magic"] = m["v"]
                    regdef["edx"] = dict(src)
                    regdef["eax"] = dict(src)
                else:
                    regdef.pop("edx", None)
                    regdef.pop("eax", None)
                this_regs.discard("eax")
                this_regs.discard("edx")
                continue
            # add/sub reg,imm on a this-pointer: a sub-object adjustment
            if not op2 and op in (0x83, 0x81) and mod == 3 and x.reg in (0, 5):
                dst = R32[x.rm]
                imm = struct.unpack_from("<b" if op == 0x83 else "<i",
                                         raw, len(raw) - (1 if op == 0x83 else 4))[0]
                if x.reg == 5:
                    imm = -imm
                if dst in this_regs:
                    regdef[dst] = {"k": "addr", "base": dst, "disp": imm,
                                   "this": True, "width": None}
                    this_regs.discard(dst)
                    continue
                d0 = regdef.get(dst)
                if d0 and d0.get("k") == "addr" and d0.get("this"):
                    regdef[dst] = dict(d0, disp=d0["disp"] + imm)
                    continue
                if d0 and d0.get("k") == "vecspan":
                    continue
                regdef.pop(dst, None)
                continue
            if (not op2 and op in (0xC1, 0xD1) and mod == 3) or \
                    (not op2 and op in (0x69, 0x6B)) or (op2 and op == 0xAF):
                dst = R32[x.reg] if (op in (0x69, 0x6B) or op2) else R32[x.rm]
                d0 = regdef.get(dst, {})
                if d0.get("k") == "vecspan":
                    # `sar reg,k` completes the divide: with a preceding magic
                    # multiply the divisor is the element stride, without one
                    # the stride is 1<<k.  Either way this is the container's
                    # element size, read off the code rather than guessed.
                    if x.reg == 7 and d0.get("shift") is None:
                        k = raw[-1] if op == 0xC1 else 1
                        d0["shift"] = k
                        d0["stride"] = (stride_from_magic(d0["magic"], k)
                                        if d0.get("magic") else (1 << k))
                        containers[(d0["base"], d0["disp"],
                                    bool(d0.get("this")))] = d0.get("stride")
                    continue                                # span survives
                regdef.pop(dst, None)
                this_regs.discard(dst)
                continue

            # ---- anything else that writes a register --------------------
            if not op2 and op in (0x01, 0x03, 0x29, 0x2B, 0x31, 0x21, 0x09,
                                  0x0B, 0x23, 0x19, 0x1B, 0x11, 0x13) \
                    and mod == 3:
                dst = R32[x.rm] if op in (0x01, 0x29, 0x31, 0x21, 0x09, 0x19,
                                          0x11) else R32[x.reg]
                regdef.pop(dst, None)
                this_regs.discard(dst)
            elif not op2 and 0x40 <= op <= 0x4F:
                r = R32[(op - 0x40) & 7]
                regdef.pop(r, None)
                this_regs.discard(r)
        return {"va": fva, "fields": fields, "calls": calls, "notes": notes,
                "insns": n, "side": side,
                "strides": {d: v for (b, d, t), v in containers.items()
                            if v and t},
                "containers": [{"base": b, "disp": d, "this": t, "stride": v}
                               for (b, d, t), v in containers.items() if v]}

    # -- one call site ----------------------------------------------------
    def _call(self, x, k, ins, pushes, regdef, stkdef, fields, calls,
              this_regs, notes, debug, prov=None, idx=0, side=None):
        # target
        tgt = None
        slot = None
        if x.op == 0xE8:
            tgt = (x.va + len(x.raw) +
                   struct.unpack_from("<i", x.raw, 1)[0]) & 0xFFFFFFFF
        else:
            if x.mod == 3:                       # call reg
                r = R32[x.rm]
                d = regdef.get(r)
                if d and d.get("k") == "load" and d.get("disp") in SLOT_KIND:
                    slot = d["disp"]
            elif x.info and x.info["disp"] in SLOT_KIND:   # call [reg+slot]
                slot = x.info["disp"]

        kind = width = None
        if tgt is not None and tgt in HELPERS:
            kind, width = HELPERS[tgt]
            if side is not None:
                side[1 if tgt in READ_HELPERS else 0] += 1
        elif tgt is not None and tgt in self.extra:
            kind, width = self.extra[tgt]
        elif slot is not None:
            kind = SLOT_KIND[slot]
            if side is not None:
                side[0] += 1
        elif tgt is not None:
            # not a stream primitive: could be a base-class Write, called
            # thiscall with ecx = this (+ a fixed sub-object adjustment).
            adj = None
            if "ecx" in this_regs:
                adj = 0
            else:
                d = regdef.get("ecx")
                if d and d.get("k") == "addr" and d.get("this"):
                    adj = d["disp"]
            calls.append({"tgt": tgt, "va": x.va, "adj": adj})
            return
        else:
            return

        # tag = the last pushed .rdata short string; member = the push before it
        ti = None
        for i in range(len(pushes) - 1, -1, -1):
            p = pushes[i]
            if p.get("k") == "imm" and self.tag(p["v"]) is not None:
                ti = i
                break
        if ti is None or ti == 0:
            # A *forwarding* writer -- `f(stream, name, member)` that passes the
            # caller's name straight through to a stream primitive -- has no
            # constant tag of its own.  Spotting them is what lets the caller's
            # `push "PlayerIDs"; call f` be read as a field at all.
            if any(p.get("k") == "load" and p.get("base") == "ebp" and
                   p.get("disp") == 0xC for p in pushes):
                notes.append("FORWARDER")
            else:
                notes.append(f"0x{x.va:08x} {kind}: no tag/member pair")
            return
        tag = self.tag(pushes[ti]["v"])
        m = pushes[ti - 1]

        rec = {"va": x.va, "tag": tag, "kind": kind, "slot": slot,
               "helper": tgt if tgt in HELPERS else None}

        # raw-bytes writes carry their length as the push before the pointer
        if kind == "raw":
            nb = pushes[ti - 2] if ti >= 2 else None
            if nb and nb.get("k") == "imm":
                rec["bytes"] = nb["v"]
                if nb["v"] == 8:
                    kind = rec["kind"] = "int64"
            m = pushes[ti - 1]

        # `WriteInt`/`WriteFloat`/... take a *pointer*, so a computed value (a
        # container count, a NULL-test gate) is spilled to a stack local whose
        # address is pushed -- sometimes stored after the push, before the call.
        if m.get("k") == "addr" and m.get("base") in ("ebp", "esp") \
                and kind not in ("nested", "nested2"):
            sp = stkdef.get((m["base"], m["disp"]))
            if sp is not None:
                m = sp

        if kind in ("nested", "nested2"):
            self._nested(rec, m, stkdef)
        elif m.get("k") == "vecspan":
            # the argument was a container *count*, computed as
            # (_Mylast - _Myfirst)/stride -- so the member is the container
            rec["kind"] = "vector"
            rec["base"] = m.get("base")
            rec["off"] = m.get("disp")
            rec["this"] = bool(m.get("this"))
            rec["size"] = 0x10
            rec["count_of"] = True
        elif m.get("prov") and not m.get("this"):
            # a pointer member, serialised through what it points at
            rec["kind"] = "handle"
            rec["base"] = m["prov"][0]
            rec["off"] = m["prov"][1]
            rec["this"] = True
            rec["size"] = 4
            rec["via"] = "deref"
        elif m.get("k") in ("addr", "load"):
            rec["base"] = m.get("base")
            rec["off"] = m.get("disp")
            rec["this"] = bool(m.get("this"))
            if kind == "int" and m.get("k") == "load" and m.get("width"):
                w = m["width"]
                if w == 2:
                    rec["kind"] = "int16"
                elif w == 1:
                    rec["kind"] = "int8"
            rec["size"] = KIND_SIZE.get(rec["kind"], 4)
        elif m.get("k") == "imm":
            rec["kind"] = "const"
            rec["const"] = m["v"]
        else:
            rec["kind"] = rec["kind"] + "?"
            rec["unresolved"] = True
        fields.append(rec)

    def _nested(self, rec, m, stkdef):
        """A nested member goes through {vptr, 0, T*} built on the stack."""
        if m.get("k") != "addr" or m.get("base") not in ("ebp", "esp"):
            rec["unresolved"] = True
            return
        b, d = m["base"], m["disp"]
        vp = stkdef.get((b, d))
        tp = stkdef.get((b, d + 8))
        if vp and vp.get("k") == "imm" and vp["v"] in self.vfts:
            hv = self.vfts[vp["v"]]
            rec["helper_vftable"] = vp["v"]
            rec["type"] = hv["class"]
            inner, isvec = helper_inner(hv["mangled"])
            rec["inner"] = inner
            if isvec:
                rec["kind"], rec["size"] = "vector", 0x10
            elif inner.startswith("enum:"):
                rec["kind"], rec["size"] = "enum", 4
            else:
                rec["kind"] = "object"
        if tp and tp.get("k") == "addr":
            rec["base"] = tp.get("base")
            rec["off"] = tp.get("disp")
            rec["this"] = bool(tp.get("this"))
        elif tp and tp.get("k") == "load":
            rec["base"] = tp.get("base")
            rec["off"] = tp.get("disp")
            rec["this"] = bool(tp.get("this"))
            rec["ptr"] = True
            rec["size"] = 4
        else:
            rec["unresolved"] = True


def stride_from_magic(magic, shift):
    """MSVC emits `n / s` as `(n * ceil(2^(32+k)/s)) >> (32+k)`; invert it.

    This is the container-stride enumeration: it reads `sizeof(element)` off
    the divide the compiler emitted, so an element's size comes from the code
    rather than from adding up the offsets the code happens to touch.
    """
    if not magic:
        return None
    for s in range(1, 8192):
        n = 1 << (32 + shift)
        if -(-n // s) == magic:
            return s
    return None


def helper_inner(mangled):
    """`.?AU?$StreamableHelper@VObservedTech@Game@@@Mars@@` -> (Game::ObservedTech, False)
       `?$VectorHelper@V...`                              -> (T, True)"""
    m = mangled
    for p in (".?AV", ".?AU", ".?AW", ".?AT"):
        if m.startswith(p):
            m = m[len(p):]
    isvec = m.startswith("?$VectorHelper@")
    if isvec:
        m = m[len("?$VectorHelper@"):]
    elif m.startswith("?$StreamableHelper@"):
        m = m[len("?$StreamableHelper@"):]
    else:
        return mangled, False
    if m.endswith("@Mars@@@Mars@@"):
        m = m[:-len("@Mars@@@Mars@@")]
    elif m.endswith("@Mars@@"):
        m = m[:-len("@Mars@@")]
    if m[:1] in ("V", "U", "W", "T") and m[1:2] == "?":
        m = m[1:]
    if m.startswith("?$StreamableEnum@"):
        inner = m[len("?$StreamableEnum@"):].rstrip("@")
        return "enum:" + BUILTIN.get(inner, inner.split("@")[0].lstrip("W4V")), \
            isvec
    # MSVC builtin type codes (a template argument that is not a class)
    if m in BUILTIN:
        return BUILTIN[m], isvec
    if m[:1] in ("V", "U", "W", "T") and not m.startswith("?$"):
        m = m[1:]
    parts = [p for p in m.split("@") if p]
    return "::".join(reversed(parts)), isvec


BUILTIN = {"C": "int8", "D": "char", "E": "uint8", "F": "int16", "G": "uint16",
           "H": "int", "I": "uint", "J": "long", "K": "ulong", "M": "float",
           "N": "double", "_N": "bool", "X": "void", "PAX": "void*",
           "PAD": "char*"}


# ------------------------------------------------------------------ driver
class Lab:
    def __init__(self):
        self.img = Image()
        _, _, self.vfts = load_rtti()
        raw = json.load(open(FUNCS))
        self.funcs = sorted((int(a, 16), n, sz) for a, (n, sz) in raw.items())
        self.starts = [f[0] for f in self.funcs]
        self.fnames = {f[0]: f[1] for f in self.funcs}
        self.ex = Extractor(self.img, self.vfts, self.fnames, self.funcs,
                            self.starts)

        self._cache = {}
        self.writers = self.writer_set()
        self.forwarders = self.find_forwarders()
        self.ex.extra = self.forwarders
        self._cache = {}                    # re-extract knowing the forwarders
        self.infos = self.classes()
        for i in self.infos.values():       # base-class splicing needs these
            self.writers.setdefault(i["write"], []).append(
                (i["class"], i["col_offset"], i["vftable"]))

    def find_forwarders(self):
        """Sub-writers `f(stream, name, member)` that pass the tag through.

        Without these, every field they write is invisible: the caller's
        `push "PlayerIDs"; call f` looks like an ordinary call.  Discovered, not
        listed by hand -- a hand list is exactly the kind of thing that silently
        under-reports on the 1,600 classes nobody has read.
        """
        cand = set()
        for wva in self.writers:
            r = self.raw(wva)
            if r:
                cand.update(c["tgt"] for c in r["calls"])
        out = {}
        for va in sorted(cand):
            if va in HELPERS or va in self.writers:
                continue
            r = self.raw(va)
            if r and "FORWARDER" in r["notes"]:
                out[va] = ("opaque", None)
        return out

    # -- the serializer set ----------------------------------------------
    def serializer_vftables(self):
        """IStreamable sub-vftables: 3 slots {dtor, Read, Write}, on a class
        whose RTTI base list actually contains `Mars::IStreamable`.

        The 3-slot shape alone is not a test: TacAISquadRule_*, the CSV row
        parsers and ~100 other classes have three virtuals and no relation to
        the stream at all.  The class hierarchy descriptor settles it."""
        return [(va, v) for va, v in self.vfts.items()
                if len(v["slots"]) == 3 and
                ("Mars::IStreamable" in v.get("bases", []) or
                 v["class"] == "Mars::IStreamable")]

    def writer_set(self):
        """VA -> [class names] for every slot-2 function of a 3-slot vftable."""
        w = {}
        for va, v in self.serializer_vftables():
            w.setdefault(v["slots"][2], []).append((v["class"], v["offset"], va))
        return w

    def thunk_target(self, va):
        """`push ebp; mov ebp,esp; mov ecx,[ecx+8]; pop ebp; jmp writer`.

        A POD type (Vector3, OutputRates, ...) has no vftable of its own; its
        serializer is reached only through this specialised StreamableHelper
        thunk, so without resolving it those classes are invisible.
        """
        buf, o, _ = self.img.find(va)
        if buf is None or o + 12 > len(buf):
            return None
        # Write thunk loads the helper's T* from +8, the Read thunk from +4.
        if buf[o:o + 4] != b"\x55\x8b\xec\x8b" or buf[o + 4] != 0x49 or \
                buf[o + 5] not in (4, 8) or buf[o + 6] != 0x5D or \
                buf[o + 7] != 0xE9:
            return None
        return (va + 12 + struct.unpack_from("<i", buf, o + 8)[0]) & 0xFFFFFFFF

    def helper_writers(self):
        """inner type name -> writer VA, for the POD helper thunks."""
        out = {}
        for va, v in self.vfts.items():
            if len(v["slots"]) != 3 or \
                    "?$StreamableHelper@" not in v["mangled"]:
                continue
            t = self.thunk_target(v["slots"][2])
            if t is None:
                continue
            inner, isvec = helper_inner(v["mangled"])
            if isvec:
                continue
            out[inner] = {"write": t, "read": self.thunk_target(v["slots"][1]),
                          "helper_vftable": va}
        return out

    def raw(self, fva):
        if fva not in self._cache:
            self._cache[fva] = self.ex.run(fva)
        return self._cache[fva]

    def layout(self, fva, depth=0, seen=None):
        """Fields of one serializer, with base-class serializers spliced in."""
        seen = seen or set()
        if fva in seen or depth > 6:
            return []
        seen = seen | {fva}
        r = self.raw(fva)
        if r is None:
            return []
        # A base-class Write is emitted at its call site, so splice in program
        # order -- that order is the on-disk order.
        ev = [(f["va"], 0, f) for f in r["fields"]]
        for c in r["calls"]:
            if c["adj"] is None or c["tgt"] == fva:
                continue
            # (a) a base-class Write, reached through its own vftable, or
            # (b) a private sub-writer of the same class -- e.g. StrategyServer
            #     writes its six id lists in FUN_00794cd0, and without this the
            #     six `PlayerIDs`-style fields are simply absent.
            if c["tgt"] in self.writers or \
                    (self.raw(c["tgt"]) and self.raw(c["tgt"])["fields"]):
                ev.append((c["va"], 1, c))
        out = []
        for va, kind, item in sorted(ev, key=lambda e: e[0]):
            if kind == 0:
                out.append(item)
                continue
            for f in self.layout(item["tgt"], depth + 1, seen):
                g = dict(f)
                if g.get("this") and isinstance(g.get("off"), int):
                    g["off"] += item["adj"]
                g["from"] = item["tgt"]
                out.append(g)
        return out


    # -- sizeof oracle ----------------------------------------------------
    def sizeof_from_vectorhelper(self):
        """sizeof(T) read off `VectorHelper<T>::Write`'s element divide.

        This is the container-stride enumeration applied binary-wide: the
        helper iterates `(_Mylast - _Myfirst)/sizeof(T)`, so the divisor MSVC
        compiled in *is* sizeof(T) -- independent of which offsets any code
        touches, and therefore able to see the trailing allocator words that a
        touch-scan cannot.
        """
        out = {}
        for va, v in self.vfts.items():
            if len(v["slots"]) != 3 or "?$VectorHelper@" not in v["mangled"]:
                continue
            inner, _ = helper_inner(v["mangled"])
            r = self.raw(v["slots"][2])
            if not r:
                continue
            st = {c["stride"] for c in r["containers"]}
            if len(st) == 1:
                s = st.pop()
                if inner in out and out[inner] != s:
                    out[inner] = None            # contradiction: report it
                elif inner not in out:
                    out[inner] = s
        return {k: v for k, v in out.items() if v}

    # -- assembled layouts -------------------------------------------------
    def classes(self):
        """class name -> {write, read, vftable, col_offset}. One entry per
        (class, IStreamable sub-object)."""
        out = {}
        for va, v in self.serializer_vftables():
            if v["mangled"].startswith(".?AU?$") or \
                    v["mangled"].startswith(".?AV?$"):
                continue                        # template adaptors, not classes
            cur = out.get(v["class"])
            if cur and cur["write"] == v["slots"][2]:
                continue
            r = self.raw(v["slots"][2])
            nf = len(r["fields"]) if r else 0
            if cur and cur["nfields"] >= nf:
                continue
            out[v["class"]] = {"class": v["class"], "vftable": va,
                               "col_offset": v["offset"],
                               "dtor": v["slots"][0], "read": v["slots"][1],
                               "write": v["slots"][2], "nfields": nf}
        # When IStreamable is the FIRST base the compiler merges its three
        # slots into the class's primary vftable, so there is no 3-slot table
        # to find and ~170 classes would silently go missing.  Take slots
        # [1]/[2] there, but only after confirming that [2] really is
        # write-side and [1] read-side -- guessing the slot is how you publish
        # a layout built from the wrong function.
        for va, v in self.vfts.items():
            c = v["class"]
            if c in out or c.startswith("?$") or len(v["slots"]) < 3:
                continue
            if "Mars::IStreamable" not in v.get("bases", []):
                continue
            w, rd = self.raw(v["slots"][2]), self.raw(v["slots"][1])
            if not w or w["side"][0] == 0 or w["side"][0] <= w["side"][1]:
                continue
            out[c] = {"class": c, "vftable": va, "col_offset": v["offset"],
                      "dtor": v["slots"][0], "read": v["slots"][1],
                      "write": v["slots"][2], "nfields": len(w["fields"]),
                      "merged_vftable": True,
                      "read_side_ok": bool(rd and rd["side"][1] >= rd["side"][0])}
        return out

    def build(self, info, sizes):
        """One class -> a layout record, with fields at absolute offsets."""
        adj = info["col_offset"]
        fields, elems, extern = [], [], []
        for f in self.layout(info["write"]):
            g = dict(f)
            if g.get("this") and isinstance(g.get("off"), int):
                g["off_abs"] = g["off"] + adj
                if g.get("kind") in ("object",) and not g.get("ptr"):
                    g["size"] = sizes.get(g.get("inner"))
                fields.append(g)
            elif isinstance(g.get("off"), int) and \
                    g.get("base") not in ("ebp", "esp"):
                elems.append(g)                 # container element / iterator
            else:
                extern.append(g)
        fields.sort(key=lambda x: (x["off_abs"], x["va"]))
        # merge duplicate offsets (a field written twice, e.g. legacy tags)
        seen, uniq = {}, []
        for f in fields:
            k = f["off_abs"]
            if k in seen:
                seen[k].setdefault("alt_tags", []).append(f["tag"])
                continue
            seen[k] = f
            uniq.append(f)
        lower = 0
        unknown = 0
        for f in uniq:
            if isinstance(f.get("size"), int):
                lower = max(lower, f["off_abs"] + f["size"])
            else:
                unknown += 1
        gaps = []
        for a, b in zip(uniq, uniq[1:]):
            if isinstance(a.get("size"), int):
                d = b["off_abs"] - (a["off_abs"] + a["size"])
                if d > 3:
                    gaps.append({"after": a["tag"], "at": a["off_abs"],
                                 "bytes": d})
        # Independent cross-check: the class's Read must land on the same
        # member addresses as its Write.  Read reaches by-value ints through a
        # stack temporary, so only the pointer-passed fields are comparable --
        # but where they are comparable, disagreement means the layout is wrong
        # or the wrong function was taken for the Write.
        ragree = rtotal = 0
        if info.get("read"):
            # Only *named* tags are cross-checkable.  A NULL writer name is
            # "." on disk and carries no identity, so matching those between
            # Read and Write would be positional -- and one side resolving a
            # field the other did not then shifts every later position and
            # manufactures disagreements that are not real.
            rl = {}
            for f in self.layout(info["read"]):
                if f.get("this") and isinstance(f.get("off"), int) \
                        and f["tag"] != ".":
                    rl.setdefault(f["tag"], set()).add(f["off"] + adj)
            dup = {t for t in [f["tag"] for f in uniq]
                   if [f["tag"] for f in uniq].count(t) > 1}
            for f in uniq:
                # `ServerPlayer` writes two different members as `Team`; a tag
                # used twice cannot be matched by name, so skip it rather than
                # report a conflict that is really a name collision.
                if f["tag"] != "." and f["tag"] in rl and f["tag"] not in dup:
                    rtotal += 1
                    ragree += (f["off_abs"] in rl[f["tag"]])
        return {"class": info["class"], "write": info["write"],
                "read_agree": ragree, "read_comparable": rtotal,
                "read": info["read"], "vftable": info["vftable"],
                "col_offset": adj, "fields": uniq, "elements": elems,
                "computed": extern, "sizeof_lower": lower,
                "sizeof_stride": sizes.get(info["class"]),
                "unsized_fields": unknown, "gaps": gaps,
                "strides": self.raw(info["write"])["strides"]}


    # -- the whole binary -------------------------------------------------
    def run_all(self):
        sizes = dict(self.sizeof_from_vectorhelper())
        infos = dict(self.infos)
        for t, h in self.helper_writers().items():
            if t in infos or not h["write"]:
                continue
            infos[t] = {"class": t, "vftable": h["helper_vftable"],
                        "col_offset": 0, "dtor": None, "read": h["read"],
                        "write": h["write"], "nfields": 0, "pod": True}
        # fixpoint: nested object sizes feed each other's lower bounds
        lay = {}
        for _ in range(6):
            lay = {c: self.build(i, sizes) for c, i in infos.items()}
            changed = False
            for c, L in lay.items():
                if c not in sizes and L["sizeof_lower"] and \
                        not L["unsized_fields"]:
                    sizes[c] = L["sizeof_lower"]
                    changed = True
            if not changed:
                break
        # upper bounds from embeddings: an object at +D followed by a field at
        # +D' cannot be larger than D'-D
        ub = {}
        for L in lay.values():
            fs = L["fields"]
            for a, b in zip(fs, fs[1:]):
                t = a.get("inner")
                if a["kind"] == "object" and not a.get("ptr") and t:
                    d = b["off_abs"] - a["off_abs"]
                    if d > 0:
                        ub[t] = min(ub.get(t, 1 << 30), d)
        for c, L in lay.items():
            L["sizeof_upper"] = ub.get(c)
            L["sizeof_vectorstride"] = self.sizeof_from_vectorhelper().get(c)
            lo, up = L["sizeof_lower"], L["sizeof_upper"]
            st = L["sizeof_vectorstride"]
            if st:
                L["sizeof"], L["sizeof_by"] = st, "container stride"
            elif lo and up and lo <= up < lo + 4 and not L["unsized_fields"]:
                L["sizeof"], L["sizeof_by"] = up, "enumeration meets embedding"
            else:
                L["sizeof"], L["sizeof_by"] = None, None
            L["grade"], L["why"] = grade(L)
        return lay


def grade(L):
    """Tier + why.  The failure classes are the point of this function."""
    fs = L["fields"]
    if not fs and not L["elements"] and not L["computed"]:
        return "empty", "serializer writes no tagged field (default/stub Write)"
    if not fs:
        return "partial", "only container elements / computed values recovered"
    bad = [f for f in fs if f.get("unresolved")]
    unsized = [f for f in fs if not isinstance(f.get("size"), int)]
    opaque = [f for f in fs if f["kind"] == "opaque"]
    if bad:
        return "partial", f"{len(bad)} field(s) unresolved (value built across " \
                          f"a branch, or a non-stack helper)"
    if opaque:
        return "partial", f"{len(opaque)} field(s) via an untyped sub-writer"
    if unsized:
        return "partial", f"{len(unsized)} nested field(s) of unknown size"
    if L["read_comparable"] and L["read_agree"] < L["read_comparable"]:
        return "conflict", (f"Read disagrees on "
                            f"{L['read_comparable'] - L['read_agree']} of "
                            f"{L['read_comparable']} comparable field offset(s)")
    anon = sum(1 for f in fs if f["tag"] == ".")
    if anon > len(fs) // 2:
        return "unnamed", ("offsets and types recovered; the writer passes a "
                           "NULL name so the fields have no on-disk names and "
                           "cannot be matched against Read by name")
    if L["read_comparable"] and L["read_agree"] == L["read_comparable"]:
        return "verified", (f"fields exact; Read agrees on all "
                            f"{L['read_comparable']} comparable offsets"
                            + ("; sizeof corroborated" if L["sizeof"] else ""))
    if L["sizeof"]:
        return "clean", "fields exact; sizeof corroborated"
    return "clean", "fields exact; sizeof is a lower bound only"


def fmt_field(f):
    off = f.get("off")
    o = f"+0x{off:x}" if isinstance(off, int) and off >= 0 else \
        (f"-0x{-off:x}" if isinstance(off, int) else "?")
    sz = f.get("size")
    s = f"0x{sz:x}" if isinstance(sz, int) else "?"
    extra = ""
    if f.get("inner"):
        extra = f" <{f['inner']}>"
    if f.get("ptr"):
        extra += " (ptr)"
    if not f.get("this", True):
        extra += f" [base={f.get('base')}]"
    if f.get("unresolved"):
        extra += " UNRESOLVED"
    return f"  {o:>8} {s:>5}  {f['kind']:<10} {f['tag']!r}{extra}"


def validate(lab, verbose=False):
    """Reproduce, exactly, the layouts the campaign already had.

    Nothing new is claimed until this passes: a recovered layout that disagrees
    with `save_reader.py` or with `struct-recovery.md` is a finding to
    investigate, not a number to publish.
    """
    from serializers_golden import GOLDEN, GOLDEN_SIZEOF, DISK_ORDER
    sizes = lab.sizeof_from_vectorhelper()
    hw = lab.helper_writers()
    ok = miss = wrong = 0
    print("=" * 72)
    print("A. field offsets and kinds vs struct-recovery.md / observedtech-append.md")
    print("=" * 72)
    for cls, g in sorted(GOLDEN.items()):
        info = {"class": cls, "write": g["write"], "read": 0, "vftable": 0,
                "col_offset": None}
        vf = [v for v in lab.vfts.values()
              if len(v["slots"]) == 3 and v["slots"][2] == g["write"]]
        if vf:
            info["col_offset"] = vf[0]["offset"]
        elif cls in hw:
            info["col_offset"] = 0          # POD helper thunk: no sub-object
        else:
            print(f"  {cls}: no 3-slot vftable for 0x{g['write']:08x}")
            continue
        lay = lab.build(info, sizes)
        got = {}
        for f in lay["fields"]:
            got.setdefault(f["tag"], []).append(f)
        c_ok = c_miss = c_wrong = 0
        bad = []
        for tag, off, kind in g["fields"]:
            cands = got.get(tag, [])
            hit = [f for f in cands if f["off_abs"] == off]
            if hit:
                k = hit[0]["kind"]
                if k == kind or (kind == "object" and k in ("object", "vector")) \
                        or (kind == "handle" and k in ("handle", "int")) \
                        or (kind == "int" and k in ("int", "handle", "enum")):
                    c_ok += 1
                else:
                    c_wrong += 1
                    bad.append(f"    {tag} +0x{off:x}: kind {k} != {kind}")
            elif cands:
                c_wrong += 1
                bad.append(f"    {tag}: at +0x{cands[0]['off_abs']:x}, "
                           f"expected +0x{off:x}")
            else:
                c_miss += 1
                bad.append(f"    {tag} +0x{off:x}: not recovered")
        ok, miss, wrong = ok + c_ok, miss + c_miss, wrong + c_wrong
        n = len(g["fields"])
        flag = "OK  " if c_wrong == 0 and c_miss == 0 else \
               ("MISS" if c_wrong == 0 else "FAIL")
        print(f"  [{flag}] {cls:38s} {c_ok:3d}/{n:<3d} exact"
              f"{'' if not c_miss else f'  {c_miss} missing'}"
              f"{'' if not c_wrong else f'  {c_wrong} WRONG'}")
        if bad and (verbose or c_wrong):
            print("\n".join(bad))
    tot = ok + miss + wrong
    print(f"\n  fields: {ok}/{tot} exact, {miss} not recovered, {wrong} WRONG")

    print()
    print("=" * 72)
    print("B. sizeof, from the container-stride divides")
    print("=" * 72)
    for cls, want in sorted(GOLDEN_SIZEOF.items()):
        got = sizes.get(cls)
        print(f"  [{'OK  ' if got == want else 'FAIL'}] {cls:38s} "
              f"got 0x{got:x}" if got else
              f"  [--  ] {cls:38s} no VectorHelper stride "
              f"(expected 0x{want:x})")

    print()
    print("=" * 72)
    print("C. on-disk tag order vs save_reader.py")
    print("=" * 72)
    exp = disk_expectations()
    agree = disagree = 0
    for va, name in sorted(DISK_ORDER.items()):
        want = exp.get(name)
        if not want:
            continue
        r = lab.raw(va)
        if not r:
            continue
        got = [f["tag"] for f in lab.layout(va)]
        i, missing = 0, []
        for t in want:
            j = got.index(t, i) if t in got[i:] else -1
            if j < 0:
                missing.append(t)
            else:
                i = j + 1
        if missing:
            disagree += 1
            print(f"  [FAIL] {name:22s} 0x{va:08x}  {len(want)-len(missing)}"
                  f"/{len(want)} in order; not found in order: "
                  f"{missing[:8]}{'...' if len(missing) > 8 else ''}")
        else:
            agree += 1
            print(f"  [OK  ] {name:22s} 0x{va:08x}  {len(want)} tags, "
                  f"same relative order as the save oracle")
    print(f"\n  {agree} shapes agree, {disagree} disagree")
    return 0 if (wrong == 0 and disagree == 0) else 1


def disk_expectations():
    """Top-level tag sequence of each save_reader.py Shape."""
    sys.path.insert(0, os.path.join(REPO, "verify", "save-reader"))
    import save_reader as SR
    out = {}

    def tags(shape):
        seq = []
        fields = shape.fields if isinstance(shape, (SR.Shape, SR.Seq)) else []
        for f in fields:
            nm = getattr(f, "name", None)
            cls = type(f).__name__
            if cls in ("Opt", "If", "Rest", "Repeat"):
                continue            # legacy / conditional / open-ended
            if not getattr(f, "auth", False):
                continue            # R(): community field name, not a disk tag
            if nm and nm != ".":
                seq.append(nm)
        return seq

    for nm in dir(SR):
        v = getattr(SR, nm)
        if isinstance(v, SR.Shape):
            out[nm] = tags(v)
    return out


CNAME = {"bool": "bool", "int": "int", "int16": "short", "int8": "char",
         "float": "float", "int64": "longlong", "enum": "int"}


C_KEYWORDS = {
    "auto", "break", "case", "char", "const", "continue", "default", "do",
    "double", "else", "enum", "extern", "float", "for", "goto", "if", "inline",
    "int", "long", "register", "restrict", "return", "short", "signed",
    "sizeof", "static", "struct", "switch", "typedef", "union", "unsigned",
    "void", "volatile", "while", "bool", "class", "new", "delete", "this",
    "operator", "template", "typename", "namespace", "public", "private"}


def cident(s, used):
    out = "".join(c if (c.isalnum() or c == "_") else "_" for c in s)
    if not out or out[0].isdigit():
        out = "f_" + out
    if out in C_KEYWORDS:
        out += "_"
    base, n = out, 1
    while out in used:
        out = f"{base}_{n}"
        n += 1
    used.add(out)
    return out


VALID_ID = __import__("re").compile(r"^[A-Za-z_]\w*$")


def emit_c(cls, L, sizes, emitted):
    """One recovered layout as a C struct, gaps made explicit as padding."""
    nm = cls.replace("::", "_")
    if not VALID_ID.match(nm):
        return None
    lines, used, cur, k = [], set(), 0, 0
    for f in L["fields"]:
        off, kind = f["off_abs"], f["kind"]
        sz = f.get("size")
        if kind in ("object", "vector") and f.get("ptr"):
            ctype, sz = "void *", 4
        elif kind == "vector":
            ctype, sz = "Mars_vector", 0x10
        elif kind == "string":
            ctype, sz = "Mars_string", 0x1C
        elif kind == "handle":
            ctype, sz = "void *", 4
        elif kind == "object":
            t = (f.get("inner") or "").replace("::", "_")
            if t in emitted and VALID_ID.match(t) and sizes.get(f.get("inner")):
                ctype, sz = t, sizes[f["inner"]]
            elif isinstance(sz, int) and sz > 0:
                ctype = f"char[{sz}]"
            else:
                return None                     # unsized nested member
        elif kind in CNAME:
            ctype = CNAME[kind]
        else:
            return None
        if not isinstance(sz, int) or sz <= 0 or off < cur:
            return None
        if off > cur:
            lines.append(f"    char _pad{k}[{off - cur}];")
            k += 1
        fname = cident(f["tag"] if f["tag"] != "." else f"f{off:x}", used)
        if ctype.endswith("]"):
            base, n = ctype.split("[")
            lines.append(f"    {base} {fname}[{n};")
        else:
            lines.append(f"    {ctype} {fname};")
        cur = off + sz
    if not lines:
        return None
    total = L["sizeof"] or L["sizeof_lower"]
    if total and total > cur:
        lines.append(f"    char _pad{k}[{total - cur}];")
    return f"struct {nm} {{\n" + "\n".join(lines) + "\n};"


def build_c_header(lay, sizes):
    """All emittable layouts, nested types first."""
    order, emitted, out = [], set(), []
    todo = [c for c, L in lay.items()
            if L["grade"] in ("verified", "clean", "unnamed")]
    for _ in range(8):
        progress = False
        for c in list(todo):
            L = lay[c]
            deps = {(f.get("inner") or "") for f in L["fields"]
                    if f["kind"] == "object" and not f.get("ptr")}
            if any(d in lay and d.replace("::", "_") not in emitted and d != c
                   for d in deps):
                continue
            s = emit_c(c, L, sizes, emitted)
            todo.remove(c)
            if s:
                emitted.add(c.replace("::", "_"))
                order.append(c)
                out.append(s)
                progress = True
        if not progress:
            break
    return order, out


def write_markdown(lay):
    p = os.path.join(OUT, "layouts.md")
    with open(p, "w") as fh:
        fh.write("# Recovered class layouts (`tools/serializers.py all`)\n\n"
                 "Generated -- do not hand-edit.  Offsets are absolute (object "
                 "base).  `sizeof` is quoted only where a second, independent "
                 "line of evidence agrees with the field enumeration; "
                 "otherwise a lower bound is given.\n\n")
        for c in sorted(lay):
            L = lay[c]
            if L["grade"] == "empty":
                continue
            sz = (f"0x{L['sizeof']:x} ({L['sizeof_by']})" if L["sizeof"]
                  else f">= 0x{L['sizeof_lower']:x} (lower bound)")
            fh.write(f"## `{c}`\n\n"
                     f"Write `0x{L['write']:08x}`, Read "
                     f"`{('0x%08x' % L['read']) if L['read'] else '-'}`, "
                     f"IStreamable vftable `0x{L['vftable']:08x}` "
                     f"(COL offset +0x{L['col_offset']:x}). "
                     f"sizeof {sz}. **{L['grade']}** -- {L['why']}\n\n")
            if L["fields"]:
                fh.write("| offset | size | kind | tag | type |\n"
                         "|---|---|---|---|---|\n")
                for f in L["fields"]:
                    s = f"0x{f['size']:x}" if isinstance(f.get("size"), int) \
                        else "?"
                    fh.write(f"| 0x{f['off_abs']:x} | {s} | {f['kind']}"
                             f"{' (ptr)' if f.get('ptr') else ''} | "
                             f"`{f['tag']}` | {f.get('inner', '')} |\n")
                fh.write("\n")
            if L["elements"]:
                fh.write("Container-element fields (offsets are within the "
                         "element / map node, not the object): "
                         + ", ".join(f"`{f['tag']}`@+0x{f['off']:x}"
                                     for f in L["elements"]) + "\n\n")
    print(f"wrote {p}")


def main():
    cmd = sys.argv[1] if len(sys.argv) > 1 else "find"
    lab = Lab()
    if cmd == "validate":
        return validate(lab, "-v" in sys.argv)
    if cmd == "dump":
        va = int(sys.argv[2], 0)
        owners = lab.writers.get(va, [])
        print(f"0x{va:08x} {lab.fnames.get(va,'')}  "
              f"{[o[0] for o in owners]}")
        for f in lab.layout(va):
            print(fmt_field(f))
        r = lab.raw(va)
        for n in r["notes"]:
            print("  note: " + n)
    elif cmd == "all":
        lay = lab.run_all()
        os.makedirs(OUT, exist_ok=True)
        with open(os.path.join(OUT, "layouts.json"), "w") as fh:
            json.dump(lay, fh, indent=1, sort_keys=True)
        by = {}
        for c, L in lay.items():
            by.setdefault(L["grade"], []).append(c)
        nf = sum(len(L["fields"]) for L in lay.values())
        print(f"serializable classes with a Write : {len(lay)}")
        for g in ("verified", "clean", "unnamed", "partial", "conflict",
                  "empty"):
            print(f"  {g:8s}: {len(by.get(g, []))}")
        print(f"member fields recovered           : {nf}")
        print(f"classes with a corroborated sizeof: "
              f"{sum(1 for L in lay.values() if L['sizeof'])}")
        ra = sum(L["read_agree"] for L in lay.values())
        rt = sum(L["read_comparable"] for L in lay.values())
        bad = [c for c, L in lay.items()
               if L["read_comparable"] and L["read_agree"] < L["read_comparable"]]
        print(f"Read/Write cross-check            : {ra}/{rt} fields agree "
              f"({len(bad)} class(es) with any disagreement)")
        for c in bad[:12]:
            L = lay[c]
            print(f"    {c}: {L['read_agree']}/{L['read_comparable']}")
        why = {}
        for L in lay.values():
            if L["grade"] not in ("verified", "clean"):
                why[L["why"]] = why.get(L["why"], 0) + 1
        print("\nwhy not clean:")
        for w, n in sorted(why.items(), key=lambda x: -x[1]):
            print(f"  {n:4d}  {w}")
        write_markdown(lay)
        sizes = {c: (L["sizeof"] or L["sizeof_lower"]) for c, L in lay.items()}
        order, defs = build_c_header(lay, sizes)
        hdr = ("// GENERATED by tools/serializers.py -- do not hand-edit.\n"
               "// Layouts recovered from the Mars::IStreamable serializers.\n"
               "// `Mars_string` is 0x1c and `Mars_vector` 0x10 -- allocator-LAST\n"
               "// in this build's STL, which is why a touch-scan undercounts\n"
               "// both by exactly 4 (findings/objects/struct-recovery.md 0).\n"
               "struct Mars_string { char _bx[16]; unsigned int size; "
               "unsigned int res; void* alval; };\n"
               "struct Mars_vector { void* first; void* last; void* end; "
               "void* alval; };\n\n" + "\n\n".join(defs) + "\n")
        with open(os.path.join(OUT, "layouts.h"), "w") as fh:
            fh.write(hdr)
        print(f"wrote {OUT}/layouts.h ({len(defs)} structs)")
    elif cmd == "find":
        vs = lab.serializer_vftables()
        print(f"3-slot vftables       : {len(vs)}")
        print(f"distinct Write slots  : {len(lab.writers)}")
        classes = {c for o in lab.writers.values() for c, _, _ in o}
        print(f"distinct classes      : {len(classes)}")
    return 0


if __name__ == "__main__":
    sys.exit(main() or 0)