d7ea0a048c
Every serializable class carries an enumeration of its own fields -- its
Write(Stream&), walking the members in order with a 4-char tag. This decodes
that idiom mechanically for the whole binary in 0.35 s.
Validation first (tools/serializers.py validate), against answers the campaign
already had before the tool existed:
A 305/307 field offsets+kinds exact across 17 classes, 0 WRONG, vs
struct-recovery.md 1-4 and observedtech-append.md
B sizeof from the container-stride divides: ObservedTech 0x2c, MoraleEvent
0x50, PlayerReport 0x30, DiplomacyStats 0x24 -- all matching
C 22 of save_reader.py's shapes, tag order identical (Sys 78 tags,
Player 104, CreateParams 25, Ship 22): 22 agree, 0 disagree
D Read/Write cross-check on every class: 437/437 field offsets agree
At scale: 386 classes with a Write, 1,682 member fields.
verified 87 (542 fields) | clean 77 (328) | unnamed 176 (471)
partial 31 (341) | empty 15
58 classes with a sizeof corroborated by a second line of evidence
(45 container stride, 13 enumeration meeting the embedding bound); the rest
report a lower bound and say so.
Four things each worth 10-170 classes: the RTTI class hierarchy descriptor as
the only honest "is this an IStreamable" test (a 3-slot vftable also matches
TacAISquadRule_* and the row parsers); mod=0 memory operands, which x86disp.py
cannot index and which hide every field at offset 0; the member->id pointer
idiom behind every handle field; and sub-writers, both base-class and private
(StrategyServer's six id lists live in FUN_00794cd0).
Failure classes are enumerated in the finding -- 176 anonymous-tag classes are
a hard limit on names but not on layout, and the other 64 are bounded
mechanical fixes. Two fields lost to a value assembled across a branch were
left unrecovered rather than patched with an unverifiable heuristic.
Write-back: 288 structures + 328 labels into Ghidra (0 failures), +201
addresses.json entries, header regenerated with tools/gen_addresses.py.
Note: ghidra/addresses.json also carries lane V's already-written live
confirmation text on ObservedTech_sizeof and ServerPlayer_off_ObservedTechs --
their edit, swept in only because we share the file.
1425 lines
61 KiB
Python
1425 lines
61 KiB
Python
#!/usr/bin/env python3
|
|
"""Automated struct recovery from the `Mars::IStreamable` serializers.
|
|
|
|
The idea
|
|
--------
|
|
Lane S's rule: never size a struct member from the offsets the code *touches* --
|
|
size it from an **enumeration**. `std::allocator<char>` is an empty class; it
|
|
occupies a word and is never loaded or stored, so a touch-scan undercounts every
|
|
`std::string` by exactly 4 and every `std::vector` by exactly 4. An enumeration
|
|
can show *absence*; a touch-scan cannot.
|
|
|
|
Every serializable class in this exe carries such an enumeration: its
|
|
`Write(Stream&)`, which walks the class's fields in order, each with a 4-char
|
|
name tag. `ObservedTech::Write` @0x00817cf0 gave the complete field list, tags
|
|
and all, and it matched `save_reader.py`'s on-disk order exactly. This tool does
|
|
that mechanically for every serializer in the binary.
|
|
|
|
The idioms it decodes
|
|
---------------------
|
|
Writer side, two calling conventions, one shape:
|
|
|
|
push 0xff ; default-value argument
|
|
<member expression> ; lea r,[this+D] | mov r,[this+D] | movzx ...
|
|
push r
|
|
push <tag> ; -> .rdata "otch", "pswd", ...
|
|
push <stream> / mov ecx,<stream>
|
|
call WriteString / WriteBool / ... | call [streamvft+slot]
|
|
|
|
so the push immediately before the tag push is always the member. Wrapper
|
|
helpers are direct `call rel32`; the compiler also inlines them as
|
|
`call [[stream]+slot]`, and the slot is what names the type:
|
|
|
|
+0x18 string +0x1c bool +0x20 float +0x24 int +0x28 nested
|
|
+0x30 raw bytes (n=8 -> int64)
|
|
|
|
Nested members go through a `StreamableHelper<T>` / `VectorHelper<T>` built on
|
|
the stack as `{vptr, 0, T*}`; the vptr is an RTTI'd vftable, so the *type* of the
|
|
nested member is recovered by name, not guessed.
|
|
|
|
Sizes are then assigned from the kind, never from the touched offsets:
|
|
bool 1 | int16 2 | int/float/enum/handle 4 | int64 8
|
|
std::string 0x1c (allocator-last, invisible to a touch scan)
|
|
std::vector 0x10 (allocator-last, likewise)
|
|
nested T sizeof(T), resolved recursively when T is itself serializable
|
|
|
|
Usage
|
|
-----
|
|
uv run python3 tools/serializers.py find # enumerate serializers
|
|
uv run python3 tools/serializers.py dump 0x00817cf0
|
|
uv run python3 tools/serializers.py validate # known layouts, first
|
|
uv run python3 tools/serializers.py all # -> objects/layouts.json
|
|
"""
|
|
import bisect
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
REPO = os.path.dirname(HERE)
|
|
sys.path.insert(0, HERE)
|
|
import x86disp # noqa: E402
|
|
from x86disp import decode, Desync, EXE, FUNCS # noqa: E402
|
|
from rtti_map import Image, load as load_rtti # noqa: E402
|
|
|
|
OUT = os.path.join(REPO, "objects")
|
|
|
|
R32 = ["eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi"]
|
|
PREFIX = {0x66, 0x67, 0xF0, 0xF2, 0xF3, 0x2E, 0x36, 0x3E, 0x26, 0x64, 0x65}
|
|
|
|
# Stream vftable slots (derived by disassembling the wrappers below, not assumed)
|
|
SLOT_KIND = {0x18: "string", 0x1c: "bool", 0x20: "float", 0x24: "int",
|
|
0x28: "nested", 0x2c: "nested2", 0x30: "raw"}
|
|
|
|
# Wrapper helpers: VA -> (kind, fixed width or None)
|
|
HELPERS = {
|
|
0x008b9d70: ("string", 0x1c), 0x008b9d90: ("string", 0x1c),
|
|
0x008b9c20: ("bool", 1), 0x008b9c00: ("bool", 1),
|
|
0x008b9be0: ("float", 4), 0x008b9bc0: ("float", 4),
|
|
0x008b9d50: ("int", 4), 0x008b9d20: ("int", 4),
|
|
0x008b9d00: ("int16", 2), 0x008b9cd0: ("int16", 2),
|
|
0x008b9c60: ("int64", 8), 0x008b9c40: ("int64", 8),
|
|
0x00816490: ("handle", 4), 0x008164d0: ("handle", 4),
|
|
}
|
|
READ_HELPERS = {0x008b9d90, 0x008b9c00, 0x008b9bc0, 0x008b9d20,
|
|
0x008b9cd0, 0x008b9c40, 0x008164d0}
|
|
KIND_SIZE = {"string": 0x1c, "bool": 1, "float": 4, "int": 4, "int16": 2,
|
|
"int64": 8, "handle": 4, "enum": 4, "vector": 0x10}
|
|
|
|
|
|
# ------------------------------------------------------------------ anatomy
|
|
def _has_modrm(raw):
|
|
"""Return (op2, opcode, modrm_index or None) for one instruction's bytes."""
|
|
j = 0
|
|
while j < len(raw) and raw[j] in PREFIX:
|
|
j += 1
|
|
if j >= len(raw):
|
|
return False, None, None
|
|
op2 = False
|
|
op = raw[j]
|
|
j += 1
|
|
if op == 0x0F:
|
|
op2 = True
|
|
op = raw[j]
|
|
j += 1
|
|
ent = x86disp._TWO.get(op)
|
|
else:
|
|
ent = x86disp._ONE.get(op)
|
|
if ent is None:
|
|
return op2, op, None
|
|
return op2, op, (j if ent[0] else None)
|
|
|
|
|
|
class Insn:
|
|
__slots__ = ("va", "raw", "info", "op", "op2", "modrm", "mod", "reg", "rm")
|
|
|
|
def __init__(self, va, raw, info):
|
|
self.va, self.raw, self.info = va, raw, info
|
|
self.op2, self.op, mi = _has_modrm(raw)
|
|
if mi is not None and mi < len(raw):
|
|
self.modrm = raw[mi]
|
|
self.mod = self.modrm >> 6
|
|
self.reg = (self.modrm >> 3) & 7
|
|
self.rm = self.modrm & 7
|
|
# x86disp only reports operands that carry a *displacement*, so a
|
|
# `mov eax,[edi]` -- member at offset 0 -- is invisible to it. For
|
|
# a plain struct (`.?AU...`, no vptr) offset 0 is a real field, so
|
|
# synthesise the mod=0 case here.
|
|
if self.info is None and self.mod == 0 and self.rm != 5:
|
|
if self.rm == 4:
|
|
sib = raw[mi + 1] if mi + 1 < len(raw) else None
|
|
if sib is not None and (sib & 7) != 5:
|
|
idx = (sib >> 3) & 7
|
|
self.info = {"base": R32[sib & 7],
|
|
"index": None if idx == 4 else R32[idx],
|
|
"scale": 1 << (sib >> 6), "disp": 0,
|
|
"dispsize": 0, "reg": R32[self.reg],
|
|
"lea": (not self.op2 and self.op == 0x8D),
|
|
"mnem": ""}
|
|
else:
|
|
self.info = {"base": R32[self.rm], "index": None,
|
|
"scale": 1, "disp": 0, "dispsize": 0,
|
|
"reg": R32[self.reg],
|
|
"lea": (not self.op2 and self.op == 0x8D),
|
|
"mnem": ""}
|
|
else:
|
|
self.modrm = self.mod = self.reg = self.rm = None
|
|
|
|
|
|
def body(img, funcs_sorted, starts, fva):
|
|
"""Decode a function body, sweeping to the next function start."""
|
|
j = bisect.bisect_right(starts, fva)
|
|
limit = starts[j] if j < len(starts) else None
|
|
buf, off, _ = img.find(fva)
|
|
if buf is None:
|
|
return []
|
|
end = len(buf) if limit is None else min(len(buf), off + (limit - fva))
|
|
out, i = [], off
|
|
while i < end:
|
|
try:
|
|
ln, info = decode(buf, i, end)
|
|
except Desync:
|
|
break
|
|
out.append(Insn(fva + (i - off), buf[i:i + ln], info))
|
|
i += ln
|
|
if len(out) > 60000:
|
|
break
|
|
return out
|
|
|
|
|
|
# -------------------------------------------------------------- extraction
|
|
class Extractor:
|
|
def __init__(self, img, rtti_vfts, fnames, funcs_sorted, starts):
|
|
self.img = img
|
|
self.vfts = rtti_vfts
|
|
self.fnames = fnames
|
|
self.funcs = funcs_sorted
|
|
self.starts = starts
|
|
self.extra = {} # discovered forwarding sub-writers
|
|
|
|
# -- tag strings ------------------------------------------------------
|
|
def tag(self, va):
|
|
if va == 0:
|
|
return "."
|
|
buf, o, sec = self.img.find(va)
|
|
if buf is None or sec not in (".rdata", ".data"):
|
|
return None
|
|
e = buf.find(b"\0", o, o + 24)
|
|
if e < 0:
|
|
return None
|
|
s = buf[o:e]
|
|
if not (1 <= len(s) <= 16):
|
|
return None
|
|
try:
|
|
t = s.decode("ascii")
|
|
except UnicodeDecodeError:
|
|
return None
|
|
return t if t.isprintable() and not t.isspace() else None
|
|
|
|
# -- one serializer ---------------------------------------------------
|
|
def run(self, fva, debug=False):
|
|
ins = body(self.img, self.funcs, self.starts, fva)
|
|
if not ins:
|
|
return None
|
|
n = len(ins)
|
|
|
|
this_regs = {"ecx"}
|
|
this_spill = set()
|
|
regdef = {} # reg -> record
|
|
stkdef = {} # (base, disp) -> record
|
|
prov = {} # reg -> (this-base, disp, insn idx)
|
|
pushes = [] # records since the last call
|
|
side = [0, 0] # [write-side calls, read-side calls]
|
|
containers = {} # (base, disp) -> element stride
|
|
fields = []
|
|
calls = [] # non-helper calls (base-class Write, etc.)
|
|
notes = []
|
|
|
|
def mem_rec(info, width, kind):
|
|
b = info["base"]
|
|
return {"k": kind, "base": b, "disp": info["disp"], "width": width,
|
|
"this": b in this_regs, "index": info["index"]}
|
|
|
|
for k, x in enumerate(ins):
|
|
info, raw = x.info, x.raw
|
|
op, op2, mod = x.op, x.op2, x.mod
|
|
|
|
# ---- calls -------------------------------------------------
|
|
is_call = (op == 0xE8 and not op2) or \
|
|
(op == 0xFF and not op2 and x.reg == 2)
|
|
if is_call:
|
|
self._call(x, k, ins, pushes, regdef, stkdef, fields, calls,
|
|
this_regs, notes, debug, prov, k, side)
|
|
pushes = []
|
|
for r in ("eax", "ecx", "edx"):
|
|
regdef.pop(r, None)
|
|
this_regs.discard(r)
|
|
prov.pop(r, None)
|
|
continue
|
|
|
|
# ---- pushes ------------------------------------------------
|
|
if not op2 and op == 0x68 and len(raw) == 5:
|
|
pushes.append({"k": "imm",
|
|
"v": struct.unpack_from("<I", raw, 1)[0]})
|
|
continue
|
|
if not op2 and op == 0x6A:
|
|
pushes.append({"k": "imm",
|
|
"v": struct.unpack_from("<b", raw, 1)[0] & 0xFFFFFFFF})
|
|
continue
|
|
if not op2 and 0x50 <= op <= 0x57 and len(raw) == 1:
|
|
r = R32[op - 0x50]
|
|
if r in this_regs:
|
|
# the member is at offset 0, so the compiler pushed `this`
|
|
# itself rather than emitting `lea r,[r+0]`
|
|
pushes.append({"k": "addr", "base": r, "disp": 0,
|
|
"this": True, "width": None, "reg": r})
|
|
else:
|
|
pushes.append(dict(regdef.get(r, {"k": "?", "reg": r}),
|
|
reg=r))
|
|
continue
|
|
if not op2 and op == 0xFF and x.reg == 6:
|
|
if info:
|
|
key = (info["base"], info["disp"])
|
|
if info["base"] in ("ebp", "esp") and key in stkdef:
|
|
pushes.append(dict(stkdef[key]))
|
|
else:
|
|
pushes.append(mem_rec(info, 4, "load"))
|
|
else:
|
|
pushes.append({"k": "?"})
|
|
continue
|
|
if not op2 and op == 0x8D and info: # lea
|
|
dst = R32[x.reg]
|
|
regdef[dst] = mem_rec(info, None, "addr")
|
|
if info["base"] in prov and info["base"] not in this_regs:
|
|
regdef[dst]["prov"] = prov[info["base"]][:2]
|
|
if dst != info["base"]:
|
|
prov.pop(dst, None)
|
|
this_regs.discard(dst)
|
|
continue
|
|
|
|
# ---- register / stack definitions ---------------------------
|
|
if not op2 and op == 0x8B: # mov r, rm
|
|
dst = R32[x.reg]
|
|
if mod == 3:
|
|
src = R32[x.rm]
|
|
if src in this_regs:
|
|
this_regs.add(dst)
|
|
else:
|
|
this_regs.discard(dst)
|
|
if src in regdef:
|
|
regdef[dst] = regdef[src]
|
|
else:
|
|
regdef.pop(dst, None)
|
|
elif info:
|
|
key = (info["base"], info["disp"])
|
|
# `mov edi,[edi+0x1c0]` overwrites the this-register with a
|
|
# pointer member -- so the base's this-ness has to be read
|
|
# BEFORE the destination is retired, or the field is lost.
|
|
base_this = info["base"] in this_regs
|
|
base_prov = prov.get(info["base"])
|
|
if key in this_spill:
|
|
this_regs.add(dst)
|
|
else:
|
|
this_regs.discard(dst)
|
|
if info["base"] in ("ebp", "esp") and key in stkdef:
|
|
regdef[dst] = stkdef[key]
|
|
else:
|
|
regdef[dst] = mem_rec(info, 4, "load")
|
|
regdef[dst]["this"] = base_this
|
|
if base_prov and not base_this:
|
|
# loading *through* a pointer member: the value
|
|
# serialised is `member->id` / `member->name`, so
|
|
# the member is the pointer at the remembered
|
|
# this-offset. This is the `NetworkObject id @+4`
|
|
# handle idiom, and it is why a plain "which offset
|
|
# was pushed" read misses every handle field.
|
|
regdef[dst]["prov"] = base_prov[:2]
|
|
r = regdef.get(dst, {})
|
|
if r.get("k") == "load" and r.get("this"):
|
|
prov[dst] = (r["base"], r["disp"], k)
|
|
elif "prov" not in r:
|
|
prov.pop(dst, None)
|
|
else:
|
|
regdef.pop(dst, None)
|
|
this_regs.discard(dst)
|
|
continue
|
|
if op2 and op in (0xB6, 0xB7, 0xBE, 0xBF) and info: # movzx/movsx
|
|
dst = R32[x.reg]
|
|
w = 1 if op in (0xB6, 0xBE) else 2
|
|
regdef[dst] = mem_rec(info, w, "load")
|
|
regdef[dst]["signed"] = op in (0xBE, 0xBF)
|
|
this_regs.discard(dst)
|
|
continue
|
|
if not op2 and 0xB8 <= op <= 0xBF and len(raw) >= 5: # mov r,imm32
|
|
dst = R32[op - 0xB8]
|
|
regdef[dst] = {"k": "imm",
|
|
"v": struct.unpack_from("<I", raw, len(raw) - 4)[0]}
|
|
this_regs.discard(dst)
|
|
continue
|
|
if not op2 and op == 0x33 and mod == 3 and x.reg == x.rm: # xor r,r
|
|
dst = R32[x.reg]
|
|
regdef[dst] = {"k": "imm", "v": 0}
|
|
this_regs.discard(dst)
|
|
continue
|
|
if not op2 and op == 0x89: # mov rm, r
|
|
src = R32[x.reg]
|
|
if mod == 3:
|
|
dst = R32[x.rm]
|
|
if src in this_regs:
|
|
this_regs.add(dst)
|
|
else:
|
|
this_regs.discard(dst)
|
|
if src in regdef:
|
|
regdef[dst] = regdef[src]
|
|
else:
|
|
regdef.pop(dst, None)
|
|
elif info:
|
|
key = (info["base"], info["disp"])
|
|
if info["base"] in ("ebp", "esp"):
|
|
stkdef[key] = dict(regdef.get(src, {"k": "?"}))
|
|
if src in this_regs:
|
|
this_spill.add(key)
|
|
else:
|
|
this_spill.discard(key)
|
|
continue
|
|
if not op2 and op == 0xC7 and info and mod != 3: # mov rm, imm32
|
|
key = (info["base"], info["disp"])
|
|
if info["base"] in ("ebp", "esp") and len(raw) >= 4:
|
|
stkdef[key] = {"k": "imm",
|
|
"v": struct.unpack_from("<I", raw, len(raw) - 4)[0]}
|
|
continue
|
|
|
|
# ---- container size: (_Mylast - _Myfirst) / stride -----------
|
|
# A count argument is never a member; it is computed from the
|
|
# container. Recovering the *container's* offset from the span
|
|
# subtraction is what turns "NumFlts" into a real vector member.
|
|
if not op2 and op == 0x2B: # sub r32, rm32
|
|
dst = R32[x.reg]
|
|
d0 = regdef.get(dst)
|
|
span = None
|
|
if mod == 3:
|
|
d1 = regdef.get(R32[x.rm])
|
|
if d0 and d1 and d0.get("k") == "load" and \
|
|
d1.get("k") == "load" and d0.get("base") and \
|
|
d0["base"] == d1["base"] and \
|
|
d0["disp"] == d1["disp"] + 4:
|
|
span = (d1["base"], d1["disp"], d1.get("this"))
|
|
elif info and d0 and d0.get("k") == "load" and \
|
|
d0.get("base") == info["base"] and \
|
|
d0.get("disp") == info["disp"] + 4:
|
|
span = (info["base"], info["disp"], info["base"] in this_regs)
|
|
if span:
|
|
regdef[dst] = {"k": "vecspan", "base": span[0],
|
|
"disp": span[1], "this": span[2],
|
|
"width": 0x10, "magic": None, "shift": None}
|
|
else:
|
|
regdef.pop(dst, None)
|
|
this_regs.discard(dst)
|
|
continue
|
|
if not op2 and op == 0xF7 and mod == 3 and x.reg in (4, 5):
|
|
# mul/imul rm -> edx:eax. The MSVC magic-number divide by a
|
|
# non-power-of-two stride runs the span through this.
|
|
src = regdef.get(R32[x.rm], {})
|
|
if src.get("k") == "vecspan":
|
|
m = regdef.get("eax", {})
|
|
src = dict(src)
|
|
if m.get("k") == "imm":
|
|
src["magic"] = m["v"]
|
|
regdef["edx"] = dict(src)
|
|
regdef["eax"] = dict(src)
|
|
else:
|
|
regdef.pop("edx", None)
|
|
regdef.pop("eax", None)
|
|
this_regs.discard("eax")
|
|
this_regs.discard("edx")
|
|
continue
|
|
# add/sub reg,imm on a this-pointer: a sub-object adjustment
|
|
if not op2 and op in (0x83, 0x81) and mod == 3 and x.reg in (0, 5):
|
|
dst = R32[x.rm]
|
|
imm = struct.unpack_from("<b" if op == 0x83 else "<i",
|
|
raw, len(raw) - (1 if op == 0x83 else 4))[0]
|
|
if x.reg == 5:
|
|
imm = -imm
|
|
if dst in this_regs:
|
|
regdef[dst] = {"k": "addr", "base": dst, "disp": imm,
|
|
"this": True, "width": None}
|
|
this_regs.discard(dst)
|
|
continue
|
|
d0 = regdef.get(dst)
|
|
if d0 and d0.get("k") == "addr" and d0.get("this"):
|
|
regdef[dst] = dict(d0, disp=d0["disp"] + imm)
|
|
continue
|
|
if d0 and d0.get("k") == "vecspan":
|
|
continue
|
|
regdef.pop(dst, None)
|
|
continue
|
|
if (not op2 and op in (0xC1, 0xD1) and mod == 3) or \
|
|
(not op2 and op in (0x69, 0x6B)) or (op2 and op == 0xAF):
|
|
dst = R32[x.reg] if (op in (0x69, 0x6B) or op2) else R32[x.rm]
|
|
d0 = regdef.get(dst, {})
|
|
if d0.get("k") == "vecspan":
|
|
# `sar reg,k` completes the divide: with a preceding magic
|
|
# multiply the divisor is the element stride, without one
|
|
# the stride is 1<<k. Either way this is the container's
|
|
# element size, read off the code rather than guessed.
|
|
if x.reg == 7 and d0.get("shift") is None:
|
|
k = raw[-1] if op == 0xC1 else 1
|
|
d0["shift"] = k
|
|
d0["stride"] = (stride_from_magic(d0["magic"], k)
|
|
if d0.get("magic") else (1 << k))
|
|
containers[(d0["base"], d0["disp"],
|
|
bool(d0.get("this")))] = d0.get("stride")
|
|
continue # span survives
|
|
regdef.pop(dst, None)
|
|
this_regs.discard(dst)
|
|
continue
|
|
|
|
# ---- anything else that writes a register --------------------
|
|
if not op2 and op in (0x01, 0x03, 0x29, 0x2B, 0x31, 0x21, 0x09,
|
|
0x0B, 0x23, 0x19, 0x1B, 0x11, 0x13) \
|
|
and mod == 3:
|
|
dst = R32[x.rm] if op in (0x01, 0x29, 0x31, 0x21, 0x09, 0x19,
|
|
0x11) else R32[x.reg]
|
|
regdef.pop(dst, None)
|
|
this_regs.discard(dst)
|
|
elif not op2 and 0x40 <= op <= 0x4F:
|
|
r = R32[(op - 0x40) & 7]
|
|
regdef.pop(r, None)
|
|
this_regs.discard(r)
|
|
return {"va": fva, "fields": fields, "calls": calls, "notes": notes,
|
|
"insns": n, "side": side,
|
|
"strides": {d: v for (b, d, t), v in containers.items()
|
|
if v and t},
|
|
"containers": [{"base": b, "disp": d, "this": t, "stride": v}
|
|
for (b, d, t), v in containers.items() if v]}
|
|
|
|
# -- one call site ----------------------------------------------------
|
|
def _call(self, x, k, ins, pushes, regdef, stkdef, fields, calls,
|
|
this_regs, notes, debug, prov=None, idx=0, side=None):
|
|
# target
|
|
tgt = None
|
|
slot = None
|
|
if x.op == 0xE8:
|
|
tgt = (x.va + len(x.raw) +
|
|
struct.unpack_from("<i", x.raw, 1)[0]) & 0xFFFFFFFF
|
|
else:
|
|
if x.mod == 3: # call reg
|
|
r = R32[x.rm]
|
|
d = regdef.get(r)
|
|
if d and d.get("k") == "load" and d.get("disp") in SLOT_KIND:
|
|
slot = d["disp"]
|
|
elif x.info and x.info["disp"] in SLOT_KIND: # call [reg+slot]
|
|
slot = x.info["disp"]
|
|
|
|
kind = width = None
|
|
if tgt is not None and tgt in HELPERS:
|
|
kind, width = HELPERS[tgt]
|
|
if side is not None:
|
|
side[1 if tgt in READ_HELPERS else 0] += 1
|
|
elif tgt is not None and tgt in self.extra:
|
|
kind, width = self.extra[tgt]
|
|
elif slot is not None:
|
|
kind = SLOT_KIND[slot]
|
|
if side is not None:
|
|
side[0] += 1
|
|
elif tgt is not None:
|
|
# not a stream primitive: could be a base-class Write, called
|
|
# thiscall with ecx = this (+ a fixed sub-object adjustment).
|
|
adj = None
|
|
if "ecx" in this_regs:
|
|
adj = 0
|
|
else:
|
|
d = regdef.get("ecx")
|
|
if d and d.get("k") == "addr" and d.get("this"):
|
|
adj = d["disp"]
|
|
calls.append({"tgt": tgt, "va": x.va, "adj": adj})
|
|
return
|
|
else:
|
|
return
|
|
|
|
# tag = the last pushed .rdata short string; member = the push before it
|
|
ti = None
|
|
for i in range(len(pushes) - 1, -1, -1):
|
|
p = pushes[i]
|
|
if p.get("k") == "imm" and self.tag(p["v"]) is not None:
|
|
ti = i
|
|
break
|
|
if ti is None or ti == 0:
|
|
# A *forwarding* writer -- `f(stream, name, member)` that passes the
|
|
# caller's name straight through to a stream primitive -- has no
|
|
# constant tag of its own. Spotting them is what lets the caller's
|
|
# `push "PlayerIDs"; call f` be read as a field at all.
|
|
if any(p.get("k") == "load" and p.get("base") == "ebp" and
|
|
p.get("disp") == 0xC for p in pushes):
|
|
notes.append("FORWARDER")
|
|
else:
|
|
notes.append(f"0x{x.va:08x} {kind}: no tag/member pair")
|
|
return
|
|
tag = self.tag(pushes[ti]["v"])
|
|
m = pushes[ti - 1]
|
|
|
|
rec = {"va": x.va, "tag": tag, "kind": kind, "slot": slot,
|
|
"helper": tgt if tgt in HELPERS else None}
|
|
|
|
# raw-bytes writes carry their length as the push before the pointer
|
|
if kind == "raw":
|
|
nb = pushes[ti - 2] if ti >= 2 else None
|
|
if nb and nb.get("k") == "imm":
|
|
rec["bytes"] = nb["v"]
|
|
if nb["v"] == 8:
|
|
kind = rec["kind"] = "int64"
|
|
m = pushes[ti - 1]
|
|
|
|
# `WriteInt`/`WriteFloat`/... take a *pointer*, so a computed value (a
|
|
# container count, a NULL-test gate) is spilled to a stack local whose
|
|
# address is pushed -- sometimes stored after the push, before the call.
|
|
if m.get("k") == "addr" and m.get("base") in ("ebp", "esp") \
|
|
and kind not in ("nested", "nested2"):
|
|
sp = stkdef.get((m["base"], m["disp"]))
|
|
if sp is not None:
|
|
m = sp
|
|
|
|
if kind in ("nested", "nested2"):
|
|
self._nested(rec, m, stkdef)
|
|
elif m.get("k") == "vecspan":
|
|
# the argument was a container *count*, computed as
|
|
# (_Mylast - _Myfirst)/stride -- so the member is the container
|
|
rec["kind"] = "vector"
|
|
rec["base"] = m.get("base")
|
|
rec["off"] = m.get("disp")
|
|
rec["this"] = bool(m.get("this"))
|
|
rec["size"] = 0x10
|
|
rec["count_of"] = True
|
|
elif m.get("prov") and not m.get("this"):
|
|
# a pointer member, serialised through what it points at
|
|
rec["kind"] = "handle"
|
|
rec["base"] = m["prov"][0]
|
|
rec["off"] = m["prov"][1]
|
|
rec["this"] = True
|
|
rec["size"] = 4
|
|
rec["via"] = "deref"
|
|
elif m.get("k") in ("addr", "load"):
|
|
rec["base"] = m.get("base")
|
|
rec["off"] = m.get("disp")
|
|
rec["this"] = bool(m.get("this"))
|
|
if kind == "int" and m.get("k") == "load" and m.get("width"):
|
|
w = m["width"]
|
|
if w == 2:
|
|
rec["kind"] = "int16"
|
|
elif w == 1:
|
|
rec["kind"] = "int8"
|
|
rec["size"] = KIND_SIZE.get(rec["kind"], 4)
|
|
elif m.get("k") == "imm":
|
|
rec["kind"] = "const"
|
|
rec["const"] = m["v"]
|
|
else:
|
|
rec["kind"] = rec["kind"] + "?"
|
|
rec["unresolved"] = True
|
|
fields.append(rec)
|
|
|
|
def _nested(self, rec, m, stkdef):
|
|
"""A nested member goes through {vptr, 0, T*} built on the stack."""
|
|
if m.get("k") != "addr" or m.get("base") not in ("ebp", "esp"):
|
|
rec["unresolved"] = True
|
|
return
|
|
b, d = m["base"], m["disp"]
|
|
vp = stkdef.get((b, d))
|
|
tp = stkdef.get((b, d + 8))
|
|
if vp and vp.get("k") == "imm" and vp["v"] in self.vfts:
|
|
hv = self.vfts[vp["v"]]
|
|
rec["helper_vftable"] = vp["v"]
|
|
rec["type"] = hv["class"]
|
|
inner, isvec = helper_inner(hv["mangled"])
|
|
rec["inner"] = inner
|
|
if isvec:
|
|
rec["kind"], rec["size"] = "vector", 0x10
|
|
elif inner.startswith("enum:"):
|
|
rec["kind"], rec["size"] = "enum", 4
|
|
else:
|
|
rec["kind"] = "object"
|
|
if tp and tp.get("k") == "addr":
|
|
rec["base"] = tp.get("base")
|
|
rec["off"] = tp.get("disp")
|
|
rec["this"] = bool(tp.get("this"))
|
|
elif tp and tp.get("k") == "load":
|
|
rec["base"] = tp.get("base")
|
|
rec["off"] = tp.get("disp")
|
|
rec["this"] = bool(tp.get("this"))
|
|
rec["ptr"] = True
|
|
rec["size"] = 4
|
|
else:
|
|
rec["unresolved"] = True
|
|
|
|
|
|
def stride_from_magic(magic, shift):
|
|
"""MSVC emits `n / s` as `(n * ceil(2^(32+k)/s)) >> (32+k)`; invert it.
|
|
|
|
This is the container-stride enumeration: it reads `sizeof(element)` off
|
|
the divide the compiler emitted, so an element's size comes from the code
|
|
rather than from adding up the offsets the code happens to touch.
|
|
"""
|
|
if not magic:
|
|
return None
|
|
for s in range(1, 8192):
|
|
n = 1 << (32 + shift)
|
|
if -(-n // s) == magic:
|
|
return s
|
|
return None
|
|
|
|
|
|
def helper_inner(mangled):
|
|
"""`.?AU?$StreamableHelper@VObservedTech@Game@@@Mars@@` -> (Game::ObservedTech, False)
|
|
`?$VectorHelper@V...` -> (T, True)"""
|
|
m = mangled
|
|
for p in (".?AV", ".?AU", ".?AW", ".?AT"):
|
|
if m.startswith(p):
|
|
m = m[len(p):]
|
|
isvec = m.startswith("?$VectorHelper@")
|
|
if isvec:
|
|
m = m[len("?$VectorHelper@"):]
|
|
elif m.startswith("?$StreamableHelper@"):
|
|
m = m[len("?$StreamableHelper@"):]
|
|
else:
|
|
return mangled, False
|
|
if m.endswith("@Mars@@@Mars@@"):
|
|
m = m[:-len("@Mars@@@Mars@@")]
|
|
elif m.endswith("@Mars@@"):
|
|
m = m[:-len("@Mars@@")]
|
|
if m[:1] in ("V", "U", "W", "T") and m[1:2] == "?":
|
|
m = m[1:]
|
|
if m.startswith("?$StreamableEnum@"):
|
|
inner = m[len("?$StreamableEnum@"):].rstrip("@")
|
|
return "enum:" + BUILTIN.get(inner, inner.split("@")[0].lstrip("W4V")), \
|
|
isvec
|
|
# MSVC builtin type codes (a template argument that is not a class)
|
|
if m in BUILTIN:
|
|
return BUILTIN[m], isvec
|
|
if m[:1] in ("V", "U", "W", "T") and not m.startswith("?$"):
|
|
m = m[1:]
|
|
parts = [p for p in m.split("@") if p]
|
|
return "::".join(reversed(parts)), isvec
|
|
|
|
|
|
BUILTIN = {"C": "int8", "D": "char", "E": "uint8", "F": "int16", "G": "uint16",
|
|
"H": "int", "I": "uint", "J": "long", "K": "ulong", "M": "float",
|
|
"N": "double", "_N": "bool", "X": "void", "PAX": "void*",
|
|
"PAD": "char*"}
|
|
|
|
|
|
# ------------------------------------------------------------------ driver
|
|
class Lab:
|
|
def __init__(self):
|
|
self.img = Image()
|
|
_, _, self.vfts = load_rtti()
|
|
raw = json.load(open(FUNCS))
|
|
self.funcs = sorted((int(a, 16), n, sz) for a, (n, sz) in raw.items())
|
|
self.starts = [f[0] for f in self.funcs]
|
|
self.fnames = {f[0]: f[1] for f in self.funcs}
|
|
self.ex = Extractor(self.img, self.vfts, self.fnames, self.funcs,
|
|
self.starts)
|
|
|
|
self._cache = {}
|
|
self.writers = self.writer_set()
|
|
self.forwarders = self.find_forwarders()
|
|
self.ex.extra = self.forwarders
|
|
self._cache = {} # re-extract knowing the forwarders
|
|
self.infos = self.classes()
|
|
for i in self.infos.values(): # base-class splicing needs these
|
|
self.writers.setdefault(i["write"], []).append(
|
|
(i["class"], i["col_offset"], i["vftable"]))
|
|
|
|
def find_forwarders(self):
|
|
"""Sub-writers `f(stream, name, member)` that pass the tag through.
|
|
|
|
Without these, every field they write is invisible: the caller's
|
|
`push "PlayerIDs"; call f` looks like an ordinary call. Discovered, not
|
|
listed by hand -- a hand list is exactly the kind of thing that silently
|
|
under-reports on the 1,600 classes nobody has read.
|
|
"""
|
|
cand = set()
|
|
for wva in self.writers:
|
|
r = self.raw(wva)
|
|
if r:
|
|
cand.update(c["tgt"] for c in r["calls"])
|
|
out = {}
|
|
for va in sorted(cand):
|
|
if va in HELPERS or va in self.writers:
|
|
continue
|
|
r = self.raw(va)
|
|
if r and "FORWARDER" in r["notes"]:
|
|
out[va] = ("opaque", None)
|
|
return out
|
|
|
|
# -- the serializer set ----------------------------------------------
|
|
def serializer_vftables(self):
|
|
"""IStreamable sub-vftables: 3 slots {dtor, Read, Write}, on a class
|
|
whose RTTI base list actually contains `Mars::IStreamable`.
|
|
|
|
The 3-slot shape alone is not a test: TacAISquadRule_*, the CSV row
|
|
parsers and ~100 other classes have three virtuals and no relation to
|
|
the stream at all. The class hierarchy descriptor settles it."""
|
|
return [(va, v) for va, v in self.vfts.items()
|
|
if len(v["slots"]) == 3 and
|
|
("Mars::IStreamable" in v.get("bases", []) or
|
|
v["class"] == "Mars::IStreamable")]
|
|
|
|
def writer_set(self):
|
|
"""VA -> [class names] for every slot-2 function of a 3-slot vftable."""
|
|
w = {}
|
|
for va, v in self.serializer_vftables():
|
|
w.setdefault(v["slots"][2], []).append((v["class"], v["offset"], va))
|
|
return w
|
|
|
|
def thunk_target(self, va):
|
|
"""`push ebp; mov ebp,esp; mov ecx,[ecx+8]; pop ebp; jmp writer`.
|
|
|
|
A POD type (Vector3, OutputRates, ...) has no vftable of its own; its
|
|
serializer is reached only through this specialised StreamableHelper
|
|
thunk, so without resolving it those classes are invisible.
|
|
"""
|
|
buf, o, _ = self.img.find(va)
|
|
if buf is None or o + 12 > len(buf):
|
|
return None
|
|
# Write thunk loads the helper's T* from +8, the Read thunk from +4.
|
|
if buf[o:o + 4] != b"\x55\x8b\xec\x8b" or buf[o + 4] != 0x49 or \
|
|
buf[o + 5] not in (4, 8) or buf[o + 6] != 0x5D or \
|
|
buf[o + 7] != 0xE9:
|
|
return None
|
|
return (va + 12 + struct.unpack_from("<i", buf, o + 8)[0]) & 0xFFFFFFFF
|
|
|
|
def helper_writers(self):
|
|
"""inner type name -> writer VA, for the POD helper thunks."""
|
|
out = {}
|
|
for va, v in self.vfts.items():
|
|
if len(v["slots"]) != 3 or \
|
|
"?$StreamableHelper@" not in v["mangled"]:
|
|
continue
|
|
t = self.thunk_target(v["slots"][2])
|
|
if t is None:
|
|
continue
|
|
inner, isvec = helper_inner(v["mangled"])
|
|
if isvec:
|
|
continue
|
|
out[inner] = {"write": t, "read": self.thunk_target(v["slots"][1]),
|
|
"helper_vftable": va}
|
|
return out
|
|
|
|
def raw(self, fva):
|
|
if fva not in self._cache:
|
|
self._cache[fva] = self.ex.run(fva)
|
|
return self._cache[fva]
|
|
|
|
def layout(self, fva, depth=0, seen=None):
|
|
"""Fields of one serializer, with base-class serializers spliced in."""
|
|
seen = seen or set()
|
|
if fva in seen or depth > 6:
|
|
return []
|
|
seen = seen | {fva}
|
|
r = self.raw(fva)
|
|
if r is None:
|
|
return []
|
|
# A base-class Write is emitted at its call site, so splice in program
|
|
# order -- that order is the on-disk order.
|
|
ev = [(f["va"], 0, f) for f in r["fields"]]
|
|
for c in r["calls"]:
|
|
if c["adj"] is None or c["tgt"] == fva:
|
|
continue
|
|
# (a) a base-class Write, reached through its own vftable, or
|
|
# (b) a private sub-writer of the same class -- e.g. StrategyServer
|
|
# writes its six id lists in FUN_00794cd0, and without this the
|
|
# six `PlayerIDs`-style fields are simply absent.
|
|
if c["tgt"] in self.writers or \
|
|
(self.raw(c["tgt"]) and self.raw(c["tgt"])["fields"]):
|
|
ev.append((c["va"], 1, c))
|
|
out = []
|
|
for va, kind, item in sorted(ev, key=lambda e: e[0]):
|
|
if kind == 0:
|
|
out.append(item)
|
|
continue
|
|
for f in self.layout(item["tgt"], depth + 1, seen):
|
|
g = dict(f)
|
|
if g.get("this") and isinstance(g.get("off"), int):
|
|
g["off"] += item["adj"]
|
|
g["from"] = item["tgt"]
|
|
out.append(g)
|
|
return out
|
|
|
|
|
|
# -- sizeof oracle ----------------------------------------------------
|
|
def sizeof_from_vectorhelper(self):
|
|
"""sizeof(T) read off `VectorHelper<T>::Write`'s element divide.
|
|
|
|
This is the container-stride enumeration applied binary-wide: the
|
|
helper iterates `(_Mylast - _Myfirst)/sizeof(T)`, so the divisor MSVC
|
|
compiled in *is* sizeof(T) -- independent of which offsets any code
|
|
touches, and therefore able to see the trailing allocator words that a
|
|
touch-scan cannot.
|
|
"""
|
|
out = {}
|
|
for va, v in self.vfts.items():
|
|
if len(v["slots"]) != 3 or "?$VectorHelper@" not in v["mangled"]:
|
|
continue
|
|
inner, _ = helper_inner(v["mangled"])
|
|
r = self.raw(v["slots"][2])
|
|
if not r:
|
|
continue
|
|
st = {c["stride"] for c in r["containers"]}
|
|
if len(st) == 1:
|
|
s = st.pop()
|
|
if inner in out and out[inner] != s:
|
|
out[inner] = None # contradiction: report it
|
|
elif inner not in out:
|
|
out[inner] = s
|
|
return {k: v for k, v in out.items() if v}
|
|
|
|
# -- assembled layouts -------------------------------------------------
|
|
def classes(self):
|
|
"""class name -> {write, read, vftable, col_offset}. One entry per
|
|
(class, IStreamable sub-object)."""
|
|
out = {}
|
|
for va, v in self.serializer_vftables():
|
|
if v["mangled"].startswith(".?AU?$") or \
|
|
v["mangled"].startswith(".?AV?$"):
|
|
continue # template adaptors, not classes
|
|
cur = out.get(v["class"])
|
|
if cur and cur["write"] == v["slots"][2]:
|
|
continue
|
|
r = self.raw(v["slots"][2])
|
|
nf = len(r["fields"]) if r else 0
|
|
if cur and cur["nfields"] >= nf:
|
|
continue
|
|
out[v["class"]] = {"class": v["class"], "vftable": va,
|
|
"col_offset": v["offset"],
|
|
"dtor": v["slots"][0], "read": v["slots"][1],
|
|
"write": v["slots"][2], "nfields": nf}
|
|
# When IStreamable is the FIRST base the compiler merges its three
|
|
# slots into the class's primary vftable, so there is no 3-slot table
|
|
# to find and ~170 classes would silently go missing. Take slots
|
|
# [1]/[2] there, but only after confirming that [2] really is
|
|
# write-side and [1] read-side -- guessing the slot is how you publish
|
|
# a layout built from the wrong function.
|
|
for va, v in self.vfts.items():
|
|
c = v["class"]
|
|
if c in out or c.startswith("?$") or len(v["slots"]) < 3:
|
|
continue
|
|
if "Mars::IStreamable" not in v.get("bases", []):
|
|
continue
|
|
w, rd = self.raw(v["slots"][2]), self.raw(v["slots"][1])
|
|
if not w or w["side"][0] == 0 or w["side"][0] <= w["side"][1]:
|
|
continue
|
|
out[c] = {"class": c, "vftable": va, "col_offset": v["offset"],
|
|
"dtor": v["slots"][0], "read": v["slots"][1],
|
|
"write": v["slots"][2], "nfields": len(w["fields"]),
|
|
"merged_vftable": True,
|
|
"read_side_ok": bool(rd and rd["side"][1] >= rd["side"][0])}
|
|
return out
|
|
|
|
def build(self, info, sizes):
|
|
"""One class -> a layout record, with fields at absolute offsets."""
|
|
adj = info["col_offset"]
|
|
fields, elems, extern = [], [], []
|
|
for f in self.layout(info["write"]):
|
|
g = dict(f)
|
|
if g.get("this") and isinstance(g.get("off"), int):
|
|
g["off_abs"] = g["off"] + adj
|
|
if g.get("kind") in ("object",) and not g.get("ptr"):
|
|
g["size"] = sizes.get(g.get("inner"))
|
|
fields.append(g)
|
|
elif isinstance(g.get("off"), int) and \
|
|
g.get("base") not in ("ebp", "esp"):
|
|
elems.append(g) # container element / iterator
|
|
else:
|
|
extern.append(g)
|
|
fields.sort(key=lambda x: (x["off_abs"], x["va"]))
|
|
# merge duplicate offsets (a field written twice, e.g. legacy tags)
|
|
seen, uniq = {}, []
|
|
for f in fields:
|
|
k = f["off_abs"]
|
|
if k in seen:
|
|
seen[k].setdefault("alt_tags", []).append(f["tag"])
|
|
continue
|
|
seen[k] = f
|
|
uniq.append(f)
|
|
lower = 0
|
|
unknown = 0
|
|
for f in uniq:
|
|
if isinstance(f.get("size"), int):
|
|
lower = max(lower, f["off_abs"] + f["size"])
|
|
else:
|
|
unknown += 1
|
|
gaps = []
|
|
for a, b in zip(uniq, uniq[1:]):
|
|
if isinstance(a.get("size"), int):
|
|
d = b["off_abs"] - (a["off_abs"] + a["size"])
|
|
if d > 3:
|
|
gaps.append({"after": a["tag"], "at": a["off_abs"],
|
|
"bytes": d})
|
|
# Independent cross-check: the class's Read must land on the same
|
|
# member addresses as its Write. Read reaches by-value ints through a
|
|
# stack temporary, so only the pointer-passed fields are comparable --
|
|
# but where they are comparable, disagreement means the layout is wrong
|
|
# or the wrong function was taken for the Write.
|
|
ragree = rtotal = 0
|
|
if info.get("read"):
|
|
# Only *named* tags are cross-checkable. A NULL writer name is
|
|
# "." on disk and carries no identity, so matching those between
|
|
# Read and Write would be positional -- and one side resolving a
|
|
# field the other did not then shifts every later position and
|
|
# manufactures disagreements that are not real.
|
|
rl = {}
|
|
for f in self.layout(info["read"]):
|
|
if f.get("this") and isinstance(f.get("off"), int) \
|
|
and f["tag"] != ".":
|
|
rl.setdefault(f["tag"], set()).add(f["off"] + adj)
|
|
dup = {t for t in [f["tag"] for f in uniq]
|
|
if [f["tag"] for f in uniq].count(t) > 1}
|
|
for f in uniq:
|
|
# `ServerPlayer` writes two different members as `Team`; a tag
|
|
# used twice cannot be matched by name, so skip it rather than
|
|
# report a conflict that is really a name collision.
|
|
if f["tag"] != "." and f["tag"] in rl and f["tag"] not in dup:
|
|
rtotal += 1
|
|
ragree += (f["off_abs"] in rl[f["tag"]])
|
|
return {"class": info["class"], "write": info["write"],
|
|
"read_agree": ragree, "read_comparable": rtotal,
|
|
"read": info["read"], "vftable": info["vftable"],
|
|
"col_offset": adj, "fields": uniq, "elements": elems,
|
|
"computed": extern, "sizeof_lower": lower,
|
|
"sizeof_stride": sizes.get(info["class"]),
|
|
"unsized_fields": unknown, "gaps": gaps,
|
|
"strides": self.raw(info["write"])["strides"]}
|
|
|
|
|
|
# -- the whole binary -------------------------------------------------
|
|
def run_all(self):
|
|
sizes = dict(self.sizeof_from_vectorhelper())
|
|
infos = dict(self.infos)
|
|
for t, h in self.helper_writers().items():
|
|
if t in infos or not h["write"]:
|
|
continue
|
|
infos[t] = {"class": t, "vftable": h["helper_vftable"],
|
|
"col_offset": 0, "dtor": None, "read": h["read"],
|
|
"write": h["write"], "nfields": 0, "pod": True}
|
|
# fixpoint: nested object sizes feed each other's lower bounds
|
|
lay = {}
|
|
for _ in range(6):
|
|
lay = {c: self.build(i, sizes) for c, i in infos.items()}
|
|
changed = False
|
|
for c, L in lay.items():
|
|
if c not in sizes and L["sizeof_lower"] and \
|
|
not L["unsized_fields"]:
|
|
sizes[c] = L["sizeof_lower"]
|
|
changed = True
|
|
if not changed:
|
|
break
|
|
# upper bounds from embeddings: an object at +D followed by a field at
|
|
# +D' cannot be larger than D'-D
|
|
ub = {}
|
|
for L in lay.values():
|
|
fs = L["fields"]
|
|
for a, b in zip(fs, fs[1:]):
|
|
t = a.get("inner")
|
|
if a["kind"] == "object" and not a.get("ptr") and t:
|
|
d = b["off_abs"] - a["off_abs"]
|
|
if d > 0:
|
|
ub[t] = min(ub.get(t, 1 << 30), d)
|
|
for c, L in lay.items():
|
|
L["sizeof_upper"] = ub.get(c)
|
|
L["sizeof_vectorstride"] = self.sizeof_from_vectorhelper().get(c)
|
|
lo, up = L["sizeof_lower"], L["sizeof_upper"]
|
|
st = L["sizeof_vectorstride"]
|
|
if st:
|
|
L["sizeof"], L["sizeof_by"] = st, "container stride"
|
|
elif lo and up and lo <= up < lo + 4 and not L["unsized_fields"]:
|
|
L["sizeof"], L["sizeof_by"] = up, "enumeration meets embedding"
|
|
else:
|
|
L["sizeof"], L["sizeof_by"] = None, None
|
|
L["grade"], L["why"] = grade(L)
|
|
return lay
|
|
|
|
|
|
def grade(L):
|
|
"""Tier + why. The failure classes are the point of this function."""
|
|
fs = L["fields"]
|
|
if not fs and not L["elements"] and not L["computed"]:
|
|
return "empty", "serializer writes no tagged field (default/stub Write)"
|
|
if not fs:
|
|
return "partial", "only container elements / computed values recovered"
|
|
bad = [f for f in fs if f.get("unresolved")]
|
|
unsized = [f for f in fs if not isinstance(f.get("size"), int)]
|
|
opaque = [f for f in fs if f["kind"] == "opaque"]
|
|
if bad:
|
|
return "partial", f"{len(bad)} field(s) unresolved (value built across " \
|
|
f"a branch, or a non-stack helper)"
|
|
if opaque:
|
|
return "partial", f"{len(opaque)} field(s) via an untyped sub-writer"
|
|
if unsized:
|
|
return "partial", f"{len(unsized)} nested field(s) of unknown size"
|
|
if L["read_comparable"] and L["read_agree"] < L["read_comparable"]:
|
|
return "conflict", (f"Read disagrees on "
|
|
f"{L['read_comparable'] - L['read_agree']} of "
|
|
f"{L['read_comparable']} comparable field offset(s)")
|
|
anon = sum(1 for f in fs if f["tag"] == ".")
|
|
if anon > len(fs) // 2:
|
|
return "unnamed", ("offsets and types recovered; the writer passes a "
|
|
"NULL name so the fields have no on-disk names and "
|
|
"cannot be matched against Read by name")
|
|
if L["read_comparable"] and L["read_agree"] == L["read_comparable"]:
|
|
return "verified", (f"fields exact; Read agrees on all "
|
|
f"{L['read_comparable']} comparable offsets"
|
|
+ ("; sizeof corroborated" if L["sizeof"] else ""))
|
|
if L["sizeof"]:
|
|
return "clean", "fields exact; sizeof corroborated"
|
|
return "clean", "fields exact; sizeof is a lower bound only"
|
|
|
|
|
|
def fmt_field(f):
|
|
off = f.get("off")
|
|
o = f"+0x{off:x}" if isinstance(off, int) and off >= 0 else \
|
|
(f"-0x{-off:x}" if isinstance(off, int) else "?")
|
|
sz = f.get("size")
|
|
s = f"0x{sz:x}" if isinstance(sz, int) else "?"
|
|
extra = ""
|
|
if f.get("inner"):
|
|
extra = f" <{f['inner']}>"
|
|
if f.get("ptr"):
|
|
extra += " (ptr)"
|
|
if not f.get("this", True):
|
|
extra += f" [base={f.get('base')}]"
|
|
if f.get("unresolved"):
|
|
extra += " UNRESOLVED"
|
|
return f" {o:>8} {s:>5} {f['kind']:<10} {f['tag']!r}{extra}"
|
|
|
|
|
|
def validate(lab, verbose=False):
|
|
"""Reproduce, exactly, the layouts the campaign already had.
|
|
|
|
Nothing new is claimed until this passes: a recovered layout that disagrees
|
|
with `save_reader.py` or with `struct-recovery.md` is a finding to
|
|
investigate, not a number to publish.
|
|
"""
|
|
from serializers_golden import GOLDEN, GOLDEN_SIZEOF, DISK_ORDER
|
|
sizes = lab.sizeof_from_vectorhelper()
|
|
hw = lab.helper_writers()
|
|
ok = miss = wrong = 0
|
|
print("=" * 72)
|
|
print("A. field offsets and kinds vs struct-recovery.md / observedtech-append.md")
|
|
print("=" * 72)
|
|
for cls, g in sorted(GOLDEN.items()):
|
|
info = {"class": cls, "write": g["write"], "read": 0, "vftable": 0,
|
|
"col_offset": None}
|
|
vf = [v for v in lab.vfts.values()
|
|
if len(v["slots"]) == 3 and v["slots"][2] == g["write"]]
|
|
if vf:
|
|
info["col_offset"] = vf[0]["offset"]
|
|
elif cls in hw:
|
|
info["col_offset"] = 0 # POD helper thunk: no sub-object
|
|
else:
|
|
print(f" {cls}: no 3-slot vftable for 0x{g['write']:08x}")
|
|
continue
|
|
lay = lab.build(info, sizes)
|
|
got = {}
|
|
for f in lay["fields"]:
|
|
got.setdefault(f["tag"], []).append(f)
|
|
c_ok = c_miss = c_wrong = 0
|
|
bad = []
|
|
for tag, off, kind in g["fields"]:
|
|
cands = got.get(tag, [])
|
|
hit = [f for f in cands if f["off_abs"] == off]
|
|
if hit:
|
|
k = hit[0]["kind"]
|
|
if k == kind or (kind == "object" and k in ("object", "vector")) \
|
|
or (kind == "handle" and k in ("handle", "int")) \
|
|
or (kind == "int" and k in ("int", "handle", "enum")):
|
|
c_ok += 1
|
|
else:
|
|
c_wrong += 1
|
|
bad.append(f" {tag} +0x{off:x}: kind {k} != {kind}")
|
|
elif cands:
|
|
c_wrong += 1
|
|
bad.append(f" {tag}: at +0x{cands[0]['off_abs']:x}, "
|
|
f"expected +0x{off:x}")
|
|
else:
|
|
c_miss += 1
|
|
bad.append(f" {tag} +0x{off:x}: not recovered")
|
|
ok, miss, wrong = ok + c_ok, miss + c_miss, wrong + c_wrong
|
|
n = len(g["fields"])
|
|
flag = "OK " if c_wrong == 0 and c_miss == 0 else \
|
|
("MISS" if c_wrong == 0 else "FAIL")
|
|
print(f" [{flag}] {cls:38s} {c_ok:3d}/{n:<3d} exact"
|
|
f"{'' if not c_miss else f' {c_miss} missing'}"
|
|
f"{'' if not c_wrong else f' {c_wrong} WRONG'}")
|
|
if bad and (verbose or c_wrong):
|
|
print("\n".join(bad))
|
|
tot = ok + miss + wrong
|
|
print(f"\n fields: {ok}/{tot} exact, {miss} not recovered, {wrong} WRONG")
|
|
|
|
print()
|
|
print("=" * 72)
|
|
print("B. sizeof, from the container-stride divides")
|
|
print("=" * 72)
|
|
for cls, want in sorted(GOLDEN_SIZEOF.items()):
|
|
got = sizes.get(cls)
|
|
print(f" [{'OK ' if got == want else 'FAIL'}] {cls:38s} "
|
|
f"got 0x{got:x}" if got else
|
|
f" [-- ] {cls:38s} no VectorHelper stride "
|
|
f"(expected 0x{want:x})")
|
|
|
|
print()
|
|
print("=" * 72)
|
|
print("C. on-disk tag order vs save_reader.py")
|
|
print("=" * 72)
|
|
exp = disk_expectations()
|
|
agree = disagree = 0
|
|
for va, name in sorted(DISK_ORDER.items()):
|
|
want = exp.get(name)
|
|
if not want:
|
|
continue
|
|
r = lab.raw(va)
|
|
if not r:
|
|
continue
|
|
got = [f["tag"] for f in lab.layout(va)]
|
|
i, missing = 0, []
|
|
for t in want:
|
|
j = got.index(t, i) if t in got[i:] else -1
|
|
if j < 0:
|
|
missing.append(t)
|
|
else:
|
|
i = j + 1
|
|
if missing:
|
|
disagree += 1
|
|
print(f" [FAIL] {name:22s} 0x{va:08x} {len(want)-len(missing)}"
|
|
f"/{len(want)} in order; not found in order: "
|
|
f"{missing[:8]}{'...' if len(missing) > 8 else ''}")
|
|
else:
|
|
agree += 1
|
|
print(f" [OK ] {name:22s} 0x{va:08x} {len(want)} tags, "
|
|
f"same relative order as the save oracle")
|
|
print(f"\n {agree} shapes agree, {disagree} disagree")
|
|
return 0 if (wrong == 0 and disagree == 0) else 1
|
|
|
|
|
|
def disk_expectations():
|
|
"""Top-level tag sequence of each save_reader.py Shape."""
|
|
sys.path.insert(0, os.path.join(REPO, "verify", "save-reader"))
|
|
import save_reader as SR
|
|
out = {}
|
|
|
|
def tags(shape):
|
|
seq = []
|
|
fields = shape.fields if isinstance(shape, (SR.Shape, SR.Seq)) else []
|
|
for f in fields:
|
|
nm = getattr(f, "name", None)
|
|
cls = type(f).__name__
|
|
if cls in ("Opt", "If", "Rest", "Repeat"):
|
|
continue # legacy / conditional / open-ended
|
|
if not getattr(f, "auth", False):
|
|
continue # R(): community field name, not a disk tag
|
|
if nm and nm != ".":
|
|
seq.append(nm)
|
|
return seq
|
|
|
|
for nm in dir(SR):
|
|
v = getattr(SR, nm)
|
|
if isinstance(v, SR.Shape):
|
|
out[nm] = tags(v)
|
|
return out
|
|
|
|
|
|
CNAME = {"bool": "bool", "int": "int", "int16": "short", "int8": "char",
|
|
"float": "float", "int64": "longlong", "enum": "int"}
|
|
|
|
|
|
C_KEYWORDS = {
|
|
"auto", "break", "case", "char", "const", "continue", "default", "do",
|
|
"double", "else", "enum", "extern", "float", "for", "goto", "if", "inline",
|
|
"int", "long", "register", "restrict", "return", "short", "signed",
|
|
"sizeof", "static", "struct", "switch", "typedef", "union", "unsigned",
|
|
"void", "volatile", "while", "bool", "class", "new", "delete", "this",
|
|
"operator", "template", "typename", "namespace", "public", "private"}
|
|
|
|
|
|
def cident(s, used):
|
|
out = "".join(c if (c.isalnum() or c == "_") else "_" for c in s)
|
|
if not out or out[0].isdigit():
|
|
out = "f_" + out
|
|
if out in C_KEYWORDS:
|
|
out += "_"
|
|
base, n = out, 1
|
|
while out in used:
|
|
out = f"{base}_{n}"
|
|
n += 1
|
|
used.add(out)
|
|
return out
|
|
|
|
|
|
VALID_ID = __import__("re").compile(r"^[A-Za-z_]\w*$")
|
|
|
|
|
|
def emit_c(cls, L, sizes, emitted):
|
|
"""One recovered layout as a C struct, gaps made explicit as padding."""
|
|
nm = cls.replace("::", "_")
|
|
if not VALID_ID.match(nm):
|
|
return None
|
|
lines, used, cur, k = [], set(), 0, 0
|
|
for f in L["fields"]:
|
|
off, kind = f["off_abs"], f["kind"]
|
|
sz = f.get("size")
|
|
if kind in ("object", "vector") and f.get("ptr"):
|
|
ctype, sz = "void *", 4
|
|
elif kind == "vector":
|
|
ctype, sz = "Mars_vector", 0x10
|
|
elif kind == "string":
|
|
ctype, sz = "Mars_string", 0x1C
|
|
elif kind == "handle":
|
|
ctype, sz = "void *", 4
|
|
elif kind == "object":
|
|
t = (f.get("inner") or "").replace("::", "_")
|
|
if t in emitted and VALID_ID.match(t) and sizes.get(f.get("inner")):
|
|
ctype, sz = t, sizes[f["inner"]]
|
|
elif isinstance(sz, int) and sz > 0:
|
|
ctype = f"char[{sz}]"
|
|
else:
|
|
return None # unsized nested member
|
|
elif kind in CNAME:
|
|
ctype = CNAME[kind]
|
|
else:
|
|
return None
|
|
if not isinstance(sz, int) or sz <= 0 or off < cur:
|
|
return None
|
|
if off > cur:
|
|
lines.append(f" char _pad{k}[{off - cur}];")
|
|
k += 1
|
|
fname = cident(f["tag"] if f["tag"] != "." else f"f{off:x}", used)
|
|
if ctype.endswith("]"):
|
|
base, n = ctype.split("[")
|
|
lines.append(f" {base} {fname}[{n};")
|
|
else:
|
|
lines.append(f" {ctype} {fname};")
|
|
cur = off + sz
|
|
if not lines:
|
|
return None
|
|
total = L["sizeof"] or L["sizeof_lower"]
|
|
if total and total > cur:
|
|
lines.append(f" char _pad{k}[{total - cur}];")
|
|
return f"struct {nm} {{\n" + "\n".join(lines) + "\n};"
|
|
|
|
|
|
def build_c_header(lay, sizes):
|
|
"""All emittable layouts, nested types first."""
|
|
order, emitted, out = [], set(), []
|
|
todo = [c for c, L in lay.items()
|
|
if L["grade"] in ("verified", "clean", "unnamed")]
|
|
for _ in range(8):
|
|
progress = False
|
|
for c in list(todo):
|
|
L = lay[c]
|
|
deps = {(f.get("inner") or "") for f in L["fields"]
|
|
if f["kind"] == "object" and not f.get("ptr")}
|
|
if any(d in lay and d.replace("::", "_") not in emitted and d != c
|
|
for d in deps):
|
|
continue
|
|
s = emit_c(c, L, sizes, emitted)
|
|
todo.remove(c)
|
|
if s:
|
|
emitted.add(c.replace("::", "_"))
|
|
order.append(c)
|
|
out.append(s)
|
|
progress = True
|
|
if not progress:
|
|
break
|
|
return order, out
|
|
|
|
|
|
def write_markdown(lay):
|
|
p = os.path.join(OUT, "layouts.md")
|
|
with open(p, "w") as fh:
|
|
fh.write("# Recovered class layouts (`tools/serializers.py all`)\n\n"
|
|
"Generated -- do not hand-edit. Offsets are absolute (object "
|
|
"base). `sizeof` is quoted only where a second, independent "
|
|
"line of evidence agrees with the field enumeration; "
|
|
"otherwise a lower bound is given.\n\n")
|
|
for c in sorted(lay):
|
|
L = lay[c]
|
|
if L["grade"] == "empty":
|
|
continue
|
|
sz = (f"0x{L['sizeof']:x} ({L['sizeof_by']})" if L["sizeof"]
|
|
else f">= 0x{L['sizeof_lower']:x} (lower bound)")
|
|
fh.write(f"## `{c}`\n\n"
|
|
f"Write `0x{L['write']:08x}`, Read "
|
|
f"`{('0x%08x' % L['read']) if L['read'] else '-'}`, "
|
|
f"IStreamable vftable `0x{L['vftable']:08x}` "
|
|
f"(COL offset +0x{L['col_offset']:x}). "
|
|
f"sizeof {sz}. **{L['grade']}** -- {L['why']}\n\n")
|
|
if L["fields"]:
|
|
fh.write("| offset | size | kind | tag | type |\n"
|
|
"|---|---|---|---|---|\n")
|
|
for f in L["fields"]:
|
|
s = f"0x{f['size']:x}" if isinstance(f.get("size"), int) \
|
|
else "?"
|
|
fh.write(f"| 0x{f['off_abs']:x} | {s} | {f['kind']}"
|
|
f"{' (ptr)' if f.get('ptr') else ''} | "
|
|
f"`{f['tag']}` | {f.get('inner', '')} |\n")
|
|
fh.write("\n")
|
|
if L["elements"]:
|
|
fh.write("Container-element fields (offsets are within the "
|
|
"element / map node, not the object): "
|
|
+ ", ".join(f"`{f['tag']}`@+0x{f['off']:x}"
|
|
for f in L["elements"]) + "\n\n")
|
|
print(f"wrote {p}")
|
|
|
|
|
|
def main():
|
|
cmd = sys.argv[1] if len(sys.argv) > 1 else "find"
|
|
lab = Lab()
|
|
if cmd == "validate":
|
|
return validate(lab, "-v" in sys.argv)
|
|
if cmd == "dump":
|
|
va = int(sys.argv[2], 0)
|
|
owners = lab.writers.get(va, [])
|
|
print(f"0x{va:08x} {lab.fnames.get(va,'')} "
|
|
f"{[o[0] for o in owners]}")
|
|
for f in lab.layout(va):
|
|
print(fmt_field(f))
|
|
r = lab.raw(va)
|
|
for n in r["notes"]:
|
|
print(" note: " + n)
|
|
elif cmd == "all":
|
|
lay = lab.run_all()
|
|
os.makedirs(OUT, exist_ok=True)
|
|
with open(os.path.join(OUT, "layouts.json"), "w") as fh:
|
|
json.dump(lay, fh, indent=1, sort_keys=True)
|
|
by = {}
|
|
for c, L in lay.items():
|
|
by.setdefault(L["grade"], []).append(c)
|
|
nf = sum(len(L["fields"]) for L in lay.values())
|
|
print(f"serializable classes with a Write : {len(lay)}")
|
|
for g in ("verified", "clean", "unnamed", "partial", "conflict",
|
|
"empty"):
|
|
print(f" {g:8s}: {len(by.get(g, []))}")
|
|
print(f"member fields recovered : {nf}")
|
|
print(f"classes with a corroborated sizeof: "
|
|
f"{sum(1 for L in lay.values() if L['sizeof'])}")
|
|
ra = sum(L["read_agree"] for L in lay.values())
|
|
rt = sum(L["read_comparable"] for L in lay.values())
|
|
bad = [c for c, L in lay.items()
|
|
if L["read_comparable"] and L["read_agree"] < L["read_comparable"]]
|
|
print(f"Read/Write cross-check : {ra}/{rt} fields agree "
|
|
f"({len(bad)} class(es) with any disagreement)")
|
|
for c in bad[:12]:
|
|
L = lay[c]
|
|
print(f" {c}: {L['read_agree']}/{L['read_comparable']}")
|
|
why = {}
|
|
for L in lay.values():
|
|
if L["grade"] not in ("verified", "clean"):
|
|
why[L["why"]] = why.get(L["why"], 0) + 1
|
|
print("\nwhy not clean:")
|
|
for w, n in sorted(why.items(), key=lambda x: -x[1]):
|
|
print(f" {n:4d} {w}")
|
|
write_markdown(lay)
|
|
sizes = {c: (L["sizeof"] or L["sizeof_lower"]) for c, L in lay.items()}
|
|
order, defs = build_c_header(lay, sizes)
|
|
hdr = ("// GENERATED by tools/serializers.py -- do not hand-edit.\n"
|
|
"// Layouts recovered from the Mars::IStreamable serializers.\n"
|
|
"// `Mars_string` is 0x1c and `Mars_vector` 0x10 -- allocator-LAST\n"
|
|
"// in this build's STL, which is why a touch-scan undercounts\n"
|
|
"// both by exactly 4 (findings/objects/struct-recovery.md 0).\n"
|
|
"struct Mars_string { char _bx[16]; unsigned int size; "
|
|
"unsigned int res; void* alval; };\n"
|
|
"struct Mars_vector { void* first; void* last; void* end; "
|
|
"void* alval; };\n\n" + "\n\n".join(defs) + "\n")
|
|
with open(os.path.join(OUT, "layouts.h"), "w") as fh:
|
|
fh.write(hdr)
|
|
print(f"wrote {OUT}/layouts.h ({len(defs)} structs)")
|
|
elif cmd == "find":
|
|
vs = lab.serializer_vftables()
|
|
print(f"3-slot vftables : {len(vs)}")
|
|
print(f"distinct Write slots : {len(lab.writers)}")
|
|
classes = {c for o in lab.writers.values() for c, _, _ in o}
|
|
print(f"distinct classes : {len(classes)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main() or 0)
|