sots-re/tools/strfootprint.py

#!/usr/bin/env python3
"""Whole-binary audit of the std::string member footprint in Sword of the Stars.exe.

Why this exists
---------------
`sizeof(std::string)` is load-bearing: it is embedded in ~65 recovered class
layouts, and a 4-byte error in it silently shifts every field that follows.
Lane X's ObservedTech work reported the embedded string as 0x18 bytes, against
the campaign-wide 0x1c.  Arguing about it from one class is how you get a
plausible answer instead of a true one, so this settles it from the whole
binary at once.

Method
------
Every serialiser in this exe hands a member pointer to one of the Mars::Stream
primitive helpers with the same idiom:

    push 0xff                      ; default arg
    lea  <r>,[<base>+<disp>]       ; &this->member
    push <r>
    push <tag>                     ; -> .rdata "otch", "pswd", ...
    push <stream>
    call WriteString / ReadString / WriteBool / ...

So for each helper call we recover (base register, displacement, tag).  Then,
per (function, base register), we take every OTHER displacement the same
function touches off that register.  If sizeof(std::string) were 0x18 for some
instantiation, that class would necessarily have a real member somewhere in
(N+4 .. N+0x1b).  The audit is: does one exist, anywhere?

Two things must be excluded or the answer is noise:
  * ebp/esp bases -- those are stack temporaries, not class members.  (A local
    string at [ebp-0x2c] shows _Mysize at -0x1c and _Myres at -0x18, which
    looks exactly like two "members" inside the span.)
  * N+0x10 and N+0x14 -- the string's OWN _Mysize/_Myres, read inline whenever
    the compiler inlines the SSO `_Myres >= 16 ? _Ptr : _Buf` test.

Result (2026-09-08, lane S): 65 string members off a non-stack base, ZERO with
a sibling inside the 0x1c span, and 51 of the 52 that have a next member have
it at exactly +0x1c.  One layout, 0x1c, binary-wide.

Usage:  uv run python3 tools/strfootprint.py [--all]
        --all also lists every recovered string member and its gap.
"""
import json
import os
import struct
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(HERE)
sys.path.insert(0, HERE)
from x86disp import decode, Desync, load_pe, EXE, FUNCS  # noqa: E402

# Mars::Stream primitive wrappers (struct-recovery.md S0).  All take
# (stream, tagname, &member, default) and are called from the class Read/Write.
HELPERS = {
    0x008b9d70: "WriteString", 0x008b9d90: "ReadString",
    0x008b9c20: "WriteBool",   0x008b9c00: "ReadBool",
    0x008b9be0: "WriteFloat",  0x008b9bc0: "ReadFloat",
    0x008b9d50: "WriteInt",    0x008b9d20: "ReadInt",
    0x008b9d00: "WriteInt16",  0x008b9cd0: "ReadInt16",
    0x008b9c60: "WriteInt64",  0x008b9c40: "ReadInt64",
}
STRING_HELPERS = ("WriteString", "ReadString")
STR_SIZE = 0x1c
SSO_OWN = (0x10, 0x14)          # the string's own _Mysize / _Myres
STACK_BASES = ("ebp", "esp")


# ---------------------------------------------------------------- .rdata reader
def _load_rdata():
    data = open(EXE, "rb").read()
    pe = struct.unpack_from("<I", data, 0x3C)[0]
    nsec = struct.unpack_from("<H", data, pe + 6)[0]
    optsz = struct.unpack_from("<H", data, pe + 20)[0]
    base = struct.unpack_from("<I", data, pe + 24 + 28)[0]
    secs = []
    for i in range(nsec):
        o = pe + 24 + optsz + i * 40
        vsz, va, rsz, ro = struct.unpack_from("<IIII", data, o + 8)
        secs.append((base + va, vsz, ro, rsz))
    return data, secs


_DATA, _SECS = _load_rdata()


def cstr(va, maxn=16):
    for sva, vsz, ro, rsz in _SECS:
        if sva <= va < sva + max(vsz, rsz):
            d = va - sva
            if d >= rsz:
                return None
            o = ro + d
            e = _DATA.find(b"\0", o, o + maxn)
            if e < 0:
                return None
            try:
                return _DATA[o:e].decode("ascii")
            except UnicodeDecodeError:
                return None
    return None


def main():
    show_all = "--all" in sys.argv
    _, secs = load_pe(EXE)
    sva, eva, buf, _ = secs[0]
    funcs = json.load(open(FUNCS))
    starts = sorted((int(k, 16), v[0]) for k, v in funcs.items())
    svas = [s[0] for s in starts]
    rd_lo, rd_hi = 0x9DD000, 0xAD9000

    records, touch = [], {}
    for idx, (fva, nm) in enumerate(starts):
        if not (sva <= fva < eva):
            continue
        fend = svas[idx + 1] if idx + 1 < len(starts) else eva
        ins, i, end = [], fva - sva, fend - sva
        while i < end:
            try:
                ln, info = decode(buf, i, len(buf))
            except Desync:
                break
            ins.append((sva + i, ln, info, buf[i:i + ln]))
            i += ln
        if not any(r[0] == 0xE8 and l == 5 and
                   (v + 5 + struct.unpack_from("<i", r, 1)[0]) in HELPERS
                   for v, l, _, r in ins):
            continue
        for v, l, info, r in ins:
            if info and info["base"] is not None and info["disp"] is not None:
                touch.setdefault((fva, info["base"]), set()).add(info["disp"])
        for k, (v, l, _info, r) in enumerate(ins):
            if not (r[0] == 0xE8 and l == 5):
                continue
            tgt = v + 5 + struct.unpack_from("<i", r, 1)[0]
            if tgt not in HELPERS:
                continue
            tag, ptr_reg = None, None
            for j in range(k - 1, max(-1, k - 15), -1):
                rr, ll = ins[j][3], ins[j][1]
                if rr[0] == 0x68 and ll == 5:
                    imm = struct.unpack_from("<I", rr, 1)[0]
                    if rd_lo <= imm < rd_hi:
                        s = cstr(imm)
                        if s and 1 <= len(s) <= 8 and s.isprintable():
                            tag = s
                            for m in range(j - 1, max(-1, j - 6), -1):
                                r2, l2 = ins[m][3], ins[m][1]
                                if 0x50 <= r2[0] <= 0x57 and l2 == 1:
                                    ptr_reg = ["eax", "ecx", "edx", "ebx",
                                               "esp", "ebp", "esi",
                                               "edi"][r2[0] - 0x50]
                                    break
                            break
            base = disp = None
            if ptr_reg:
                for m in range(k - 1, max(-1, k - 25), -1):
                    i2 = ins[m][2]
                    if i2 and i2["lea"] and i2["reg"] == ptr_reg \
                            and i2["base"] is not None:
                        base, disp = i2["base"], i2["disp"]
                        break
            records.append((fva, nm, base, disp, tag, HELPERS[tgt]))

    strs = [r for r in records if r[5] in STRING_HELPERS]
    members, stack, unresolved = [], 0, 0
    for fva, nm, base, disp, tag, helper in strs:
        if base is None or disp is None:
            unresolved += 1
        elif base in STACK_BASES:
            stack += 1
        else:
            members.append((fva, nm, base, disp, tag, helper))

    violations, gaps, seen = [], {}, set()
    for fva, nm, base, disp, tag, helper in members:
        others = sorted(d for d in touch.get((fva, base), set())
                        if d > disp and (d - disp) not in SSO_OWN)
        inside = [d for d in others if d < disp + STR_SIZE]
        if inside:
            violations.append((fva, nm, base, disp, tag, helper, inside))
        if others:
            g = others[0] - disp
            gaps[g] = gaps.get(g, 0) + 1
        if show_all and (fva, base, disp, helper) not in seen:
            seen.add((fva, base, disp, helper))
            nxt = f"+0x{others[0]:x}" if others else "-"
            print(f"0x{fva:08x} {nm:26s} {base} +0x{disp:<6x} {tag!r:12s} "
                  f"{helper:12s} next={nxt}")

    print(f"\nstring helper call sites            : {len(strs)}")
    print(f"  resolved to a class member offset : {len(members)}")
    print(f"  stack temporaries (ebp/esp base)  : {stack}")
    print(f"  offset not reached by a plain lea : {unresolved}")
    print(f"\nmembers with a sibling inside (N, N+0x{STR_SIZE:x}) "
          f": {len(violations)}   <-- must be 0 for sizeof(std::string)==0x1c")
    for v in violations:
        print(f"  0x{v[0]:08x} {v[1]} {v[2]}+0x{v[3]:x} {v[4]!r} "
              f"inside={[hex(x) for x in v[6]]}")
    print("\ngap from a string member to the next member on the same base:")
    for g in sorted(gaps):
        print(f"  +0x{g:<4x} : {gaps[g]}")
    return 1 if violations else 0


if __name__ == "__main__":
    sys.exit(main())