symbols: the four WRAM names the whole-map grid reads

resolve_wram.py is a second reading of the disassembly beside gen_symbols.py:
it walks ram/wram.asm with a cursor that is only ever live while anchored on an
address symbols.rs already pins, re-derives 40 of the 63 it carries with no
disagreement, and emits an address only when a pinned one after it agrees too.
wOverworldMap, wCurMapTileset, wTilesetBank and wTilesetBlocksPtr come out of
that pass; no address is hand-written and nothing else in the table moves.

services/flysim/crates/flybrain-gb/src/pokemon_red/symbols.rs

@@ -16,6 +16,7 @@ pub mod ram {
     pub const wSpriteStateData1: u16 = 0xc100;            // 49408
     pub const wSpriteStateData2: u16 = 0xc200;            // 49664
     pub const wTileMap: u16 = 0xc3a0;                     // 50080
+    pub const wOverworldMap: u16 = 0xc6e8;                // 50920
     pub const wTopMenuItemY: u16 = 0xcc24;                // 52260
     pub const wTopMenuItemX: u16 = 0xcc25;                // 52261
     pub const wCurrentMenuItem: u16 = 0xcc26;             // 52262
@@ -59,6 +60,7 @@ pub mod ram {
     pub const wCurMap: u16 = 0xd35e;                      // 54110
     pub const wYCoord: u16 = 0xd361;                      // 54113
     pub const wXCoord: u16 = 0xd362;                      // 54114
+    pub const wCurMapTileset: u16 = 0xd367;               // 54119
     pub const wCurMapHeight: u16 = 0xd368;                // 54120
     pub const wCurMapWidth: u16 = 0xd369;                 // 54121
     pub const wCurMapConnections: u16 = 0xd370;           // 54128
@@ -68,6 +70,8 @@ pub mod ram {
     pub const wSignCoords: u16 = 0xd4b1;                  // 54449
     pub const wSignTextIDs: u16 = 0xd4d1;                 // 54481
     pub const wNumSprites: u16 = 0xd4e1;                  // 54497
+    pub const wTilesetBank: u16 = 0xd52b;                 // 54571
+    pub const wTilesetBlocksPtr: u16 = 0xd52c;            // 54572
     pub const wTilesetCollisionPtr: u16 = 0xd530;         // 54576
     pub const wTilesetTalkingOverTiles: u16 = 0xd532;     // 54578
     pub const wNumHoFTeams: u16 = 0xd5a2;                 // 54690

services/flysim/tools/gen_symbols.py

@@ -182,6 +182,24 @@ EXTRA_RAM = (
     # standing behind a desk can be talked to at all: none of the four tiles around either of
     # them is walkable.
     'wTilesetTalkingOverTiles',
+    # The whole-map walkability grid (docs/design/macros.md section 15).
+    #
+    # LoadTileBlockMap copies the loaded map out of its ROM bank into wOverworldMap
+    # as one byte per 4x4-tile block, in rows of wCurMapWidth + MAP_BORDER * 2 with
+    # the map itself three rows and three columns in, so the blocks of the current
+    # map are a WRAM read rather than a ROM one. wCurMapTileset keys the tile-pair
+    # collision lists (CheckForTilePairCollisions). wTilesetBank and
+    # wTilesetBlocksPtr are the tileset header's blockset: 16 bytes per block id,
+    # four rows of four tile ids, which DrawTileBlock indexes exactly that way --
+    # and it is not in bank 0, which is why the memory seam grew a bank-aware ROM
+    # read for it. All four are resolved the same way every other name here is;
+    # services/flysim/tools/resolve_wram.py is the second reading of them, from
+    # ram/wram.asm at this commit, and it re-derives 40 of the addresses this table
+    # already carries before it emits one of these four.
+    'wOverworldMap',
+    'wCurMapTileset',
+    'wTilesetBank',
+    'wTilesetBlocksPtr',
 )
 
 

services/flysim/tools/resolve_wram.py

@@ -0,0 +1,402 @@
+"""Resolve WRAM symbol addresses out of a pret/pokered checkout, and verify the pinned ones.
+
+`gen_symbols.py` owns `symbols.rs` and takes its addresses from the prototype's
+already-generated table; it refuses a hand-written address. Section 15 of
+`docs/design/macros.md` needs four symbols that table does not carry
+(`wOverworldMap`, `wCurMapTileset`, `wTilesetBank`, `wTilesetBlocksPtr`), and a
+hand-computed address is exactly what neither script will take. So this tool
+does the same job the disassembly's own `.sym` would, from `ram/wram.asm`:
+
+* it walks the file in order, keeping a byte cursor;
+* the cursor is **only ever live while it is anchored**: it is set from a symbol
+  `symbols.rs` already pins, and any declaration form this tool cannot evaluate
+  exactly kills it until the next pinned symbol revives it. An unanchored region
+  can therefore not produce a number at all, rather than producing a wrong one;
+* every pinned symbol it reaches while live is checked against `symbols.rs`, and
+  a single disagreement is a failure with no output. A resolved symbol is only
+  reported when the run also re-derived the *next* pinned symbol after it, so
+  each answer is bracketed by two addresses the table already carries.
+
+Usage (read-only; `--emit` rewrites the `ram` block of symbols.rs in place):
+
+    python3 services/flysim/tools/resolve_wram.py --pokered <checkout> [--emit]
+"""
+
+from __future__ import annotations
+
+import argparse
+import math
+import re
+from pathlib import Path
+
+SYMBOLS = (
+    Path(__file__).resolve().parents[1] / 'crates/flybrain-gb/src/pokemon_red/symbols.rs'
+)
+
+#: The symbols this run is for, with why the macro layer needs each one. Every
+#: one is checked to be bracketed by two pinned addresses before it is emitted.
+WANTED = {
+    # The current map's block ids, as LoadTileBlockMap copies them out of the
+    # map's ROM bank: rows of (width + MAP_BORDER * 2) bytes, the map itself
+    # offset by three rows and three columns. This is what makes a whole-map
+    # walkability grid a WRAM read rather than a ROM one.
+    'wOverworldMap': 'the loaded map, one byte per 4x4-tile block',
+    # Which tileset the loaded map uses: the tile-pair collision lists are keyed
+    # by it (CheckForTilePairCollisions).
+    'wCurMapTileset': 'the loaded map tileset id',
+    # The tileset header's blockset: a bank byte and a little-endian pointer at
+    # 16 bytes per block, four rows of four tile ids (DrawTileBlock). The bank is
+    # not bank 0, so this is the read the memory seam grew a bank for.
+    'wTilesetBank': 'the ROM bank the blockset lives in',
+    'wTilesetBlocksPtr': 'blocks to tiles, 16 bytes per block',
+}
+
+
+def pinned(text: str) -> dict[str, int]:
+    """Every address `symbols.rs` carries today, by symbol name."""
+    return {
+        name: int(value, 16)
+        for name, value in re.findall(r'pub const (w\w+): u16 = 0x([0-9a-f]{4});', text)
+    }
+
+
+def constants(root: Path) -> dict[str, int]:
+    """Every `DEF NAME EQU <expression>` the declarations below need.
+
+    Resolved by repeated passes rather than in one, because the decomp defines
+    constants in terms of each other (`SURROUNDING_WIDTH EQU SCREEN_BLOCK_WIDTH *
+    BLOCK_WIDTH`). A name whose expression never becomes evaluable is simply left
+    out, which kills the cursor at any declaration that uses it.
+    """
+    pending: dict[str, str] = {}
+    sources = sorted((root / 'constants').glob('*.asm')) + sorted(
+        (root / 'constants').glob('*.inc')
+    )
+    for path in sources:
+        for name, value in re.findall(
+            r'^\s*(?:DEF|def)\s+(\w+)\s+(?:EQU|equ)\s+([^;\n]+)', path.read_text(), re.M
+        ):
+            pending.setdefault(name, value.strip())
+    out: dict[str, int] = {}
+    while pending:
+        progressed = False
+        for name in list(pending):
+            try:
+                out[name] = size_of(pending[name], out)
+            except Unevaluable:
+                continue
+            del pending[name]
+            progressed = True
+        if not progressed:
+            break
+    return out
+
+
+def number(token: str) -> int:
+    token = token.strip()
+    if token.startswith('$'):
+        return int(token[1:], 16)
+    if token.startswith('%'):
+        return int(token[1:], 2)
+    return int(token, 10)
+
+
+class Unevaluable(Exception):
+    """A declaration this tool will not guess the size of."""
+
+
+def size_of(expression: str, known: dict[str, int]) -> int:
+    """Bytes in a `ds`/`EQU` expression: the decomp's own arithmetic, nothing else.
+
+    `$`/`%` literals and the constants resolved so far are substituted, the
+    `tiles` unit is a factor of sixteen, and what is left must be plain
+    arithmetic over integers -- so a name this tool has not resolved, a function
+    call or anything else raises [`Unevaluable`] instead of becoming a guess.
+    """
+    expression = expression.split(';')[0].strip()
+    if not expression:
+        raise Unevaluable('empty')
+    scale = 1
+    if expression.endswith('tiles'):
+        expression = expression[: -len('tiles')].strip()
+        scale = 16  # one 8x8 2bpp tile is 16 bytes
+    def substitute(match: re.Match[str]) -> str:
+        token = match.group(0)
+        if token[0] in '$%':
+            return str(number(token))
+        if token in known:
+            return str(known[token])
+        raise Unevaluable(token)
+    substituted = re.sub(r'\$[0-9A-Fa-f_]+|%[01_]+|[A-Za-z_]\w*', substitute, expression)
+    if not re.fullmatch(r'[\d\s()+\-*/]+', substituted):
+        raise Unevaluable(expression)
+    try:
+        value = eval(substituted, {'__builtins__': {}}, {})  # arithmetic only, checked above
+    except (SyntaxError, ZeroDivisionError, TypeError) as error:
+        raise Unevaluable(expression) from error
+    if not isinstance(value, int):
+        raise Unevaluable(expression)
+    return value * scale
+
+
+def macro_sizes(root: Path, known: dict[str, int]) -> dict[str, int]:
+    """Sizes of the RAM struct macros, counted from their own declarations."""
+    out: dict[str, int] = {}
+    for path in sorted((root / 'macros').glob('*.asm')):
+        text = path.read_text()
+        for match in re.finditer(r'^MACRO\??\s+(\w+)\n(.*?)^ENDM', text, re.M | re.S):
+            name, body = match.group(1), match.group(2)
+            total = 0
+            for line in body.splitlines():
+                line = line.split(';')[0].strip()
+                # A struct macro labels each field with its argument
+                # (`\\1YCoord:: db`), so the label is stripped and the
+                # declaration after it is what reserves the bytes.
+                line = re.sub(r'^[\\\w{}:.\d]+::\s*', '', line)
+                if not line or line.startswith(('IF', 'ELSE', 'ENDC', 'ASSERT')):
+                    continue
+                if re.match(r'^\w+::?$', line):
+                    continue
+                if line.startswith('db'):
+                    total += max(1, len([part for part in line[2:].split(',') if part.strip()]))
+                elif line.startswith('dw'):
+                    total += 2 * max(1, len([p for p in line[2:].split(',') if p.strip()]))
+                elif line.startswith('ds '):
+                    try:
+                        total += size_of(line[3:], known)
+                    except Unevaluable:
+                        total = None
+                        break
+                else:
+                    total = None
+                    break
+            if total is not None:
+                out[name] = total
+    return out
+
+
+def walk(
+    root: Path, table: dict[str, int], known: dict[str, int], verbose: bool = False
+) -> tuple[dict[str, int], list[str], int]:
+    """Resolve every symbol of wram.asm the anchored cursor can reach exactly."""
+    macros = macro_sizes(root, known)
+    lines = (root / 'ram/wram.asm').read_text().splitlines()
+    cursor: int | None = None
+    resolved: dict[str, int] = {}
+    # Symbols counted since the last pinned anchor, held back until a pinned
+    # address after them agrees.
+    pending_run: dict[str, int] = {}
+    order: list[str] = []
+    checked = 0
+    problems: list[str] = []
+    lost: list[str] = []
+    # UNION frames: (start address, widest branch so far, whether a branch was
+    # unevaluable). A frame whose start is unknown, or any one of whose branches this
+    # tool could not size, poisons the whole union: what the section advances by is
+    # the widest branch, so one branch it cannot measure means it cannot measure any.
+    unions: list[tuple[int | None, int, bool]] = []
+    index = 0
+    while index < len(lines):
+        raw = lines[index]
+        index += 1
+        line = raw.split(';')[0].strip()
+        if not line:
+            continue
+        if line.startswith('SECTION'):
+            # A section's address comes from the linker, not the source. WRAM0
+            # sections are packed in declaration order, so the cursor carries on
+            # -- and the next address `symbols.rs` pins is what tests that: any
+            # padding the linker inserted would land as a MISMATCH and this tool
+            # would emit nothing.
+            continue
+        if line == 'UNION':
+            unions.append((cursor, 0, False))
+            continue
+        if line == 'NEXTU':
+            if not unions:
+                cursor = None
+                continue
+            start, widest, poisoned = unions.pop()
+            poisoned = poisoned or start is None or cursor is None
+            if not poisoned:
+                widest = max(widest, cursor - start)
+            unions.append((start, widest, poisoned))
+            cursor = start
+            continue
+        if line == 'ENDU':
+            if not unions:
+                cursor = None
+                continue
+            start, widest, poisoned = unions.pop()
+            if poisoned or start is None or cursor is None:
+                cursor = None
+                continue
+            cursor = start + max(widest, cursor - start)
+            continue
+        if line.startswith(('FOR ', 'REPT ')):
+            # Evaluate the body only when every line of it has a known size.
+            head = line.split(None, 1)[1]
+            # `REPT n`, `FOR v, stop` and `FOR v, start, stop`: rgbasm's own
+            # three forms, and the third iterates stop - start times.
+            arguments = [part.strip() for part in head.split(',')]
+            count_token = arguments[-1]
+            start_token = arguments[-2] if line.startswith('FOR ') and len(arguments) == 3 else None
+            body: list[str] = []
+            depth = 1
+            while index < len(lines):
+                inner = lines[index].split(';')[0].strip()
+                index += 1
+                if inner.startswith(('FOR ', 'REPT ')):
+                    depth += 1
+                if inner == 'ENDR':
+                    depth -= 1
+                    if depth == 0:
+                        break
+                body.append(inner)
+            try:
+                count = size_of(count_token, known)
+                if start_token is not None:
+                    count -= size_of(start_token, known)
+                per = 0
+                for inner in body:
+                    per += declaration_size(inner, known, macros)
+                if cursor is not None:
+                    cursor += count * per
+            except Unevaluable:
+                if cursor is not None:
+                    lost.append(f'line {index}: {line}')
+                cursor = None
+                pending_run.clear()
+            continue
+        label = re.match(r'^(w\w+)::', line)
+        if label is not None:
+            name = label.group(1)
+            if name in table:
+                if cursor is not None and cursor != table[name]:
+                    problems.append(
+                        f'{name}: wram.asm gives ${cursor:04x}, symbols.rs pins ${table[name]:04x}'
+                    )
+                    pending_run.clear()
+                elif cursor is not None:
+                    checked += 1
+                    # Everything counted since the last anchor is now bracketed
+                    # by two addresses the table already carries.
+                    resolved.update(pending_run)
+                    order.extend(pending_run)
+                    pending_run.clear()
+                else:
+                    if verbose:
+                        lost.append(f'cold anchor at {name} (line {index})')
+                    pending_run.clear()
+                cursor = table[name]
+            elif cursor is not None:
+                pending_run[name] = cursor
+            line = line[label.end() :].strip()
+            if not line:
+                continue
+        if line.endswith('::') or re.fullmatch(r'\.\w+', line):
+            continue
+        try:
+            if cursor is not None:
+                cursor += declaration_size(line, known, macros)
+        except Unevaluable:
+            if cursor is not None:
+                lost.append(f'line {index}: {line}')
+            cursor = None
+            pending_run.clear()
+    if verbose:
+        for entry in lost:
+            print(f'UNEVALUABLE {entry}')
+    return resolved, problems, checked
+
+
+def declaration_size(line: str, known: dict[str, int], macros: dict[str, int]) -> int:
+    """Bytes one declaration line reserves, or [`Unevaluable`]."""
+    line = line.split(';')[0].strip()
+    if not line or line.endswith('::') or line.startswith(('ENDSECTION', 'ASSERT', 'ENDR')):
+        return 0
+    # Any label, including the `{02d:n}` interpolations a FOR body labels its
+    # iterations with; what reserves the bytes is the declaration after it.
+    line = re.sub(r'^[\\\w{}:.\d]+::\s*', '', line)
+    if not line:
+        return 0
+    if line == 'db':
+        return 1
+    if line == 'dw':
+        return 2
+    if line.startswith('db '):
+        return max(1, len([part for part in line[3:].split(',') if part.strip()]))
+    if line.startswith('dw '):
+        return 2 * max(1, len([part for part in line[3:].split(',') if part.strip()]))
+    if line.startswith('ds '):
+        return size_of(line[3:], known)
+    if line.startswith('flag_array '):
+        return math.ceil(size_of(line[len('flag_array ') :], known) / 8)
+    head = line.split(None, 1)[0]
+    if head in macros:
+        return macros[head]
+    raise Unevaluable(line)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--pokered', type=Path, required=True)
+    parser.add_argument('--verbose', action='store_true', help='name every declaration it will not size')
+    parser.add_argument('--emit', action='store_true', help='write the new addresses into symbols.rs')
+    args = parser.parse_args()
+
+    text = SYMBOLS.read_text()
+    commit = re.search(r'POKERED_COMMIT: &str = "([0-9a-f]{40})"', text)
+    if commit is None:
+        raise SystemExit('symbols.rs carries no POKERED_COMMIT')
+    head = (args.pokered / '.git/HEAD').read_text().strip()
+    if head.startswith('ref:'):
+        head = (args.pokered / '.git' / head.split()[1]).read_text().strip()
+    if head != commit.group(1):
+        raise SystemExit(
+            f'checkout is at {head}, symbols.rs pins {commit.group(1)}: two revisions of the '
+            'disassembly renumber RAM relative to each other'
+        )
+
+    table = pinned(text)
+    known = constants(args.pokered)
+    resolved, problems, checked = walk(args.pokered, table, known, args.verbose)
+    if problems:
+        for problem in problems:
+            print(f'MISMATCH {problem}')
+        raise SystemExit('the walk disagrees with symbols.rs; nothing emitted')
+    print(f'{checked} of {len(table)} pinned addresses re-derived from wram.asm, no disagreement')
+
+    missing = [name for name in WANTED if name not in resolved]
+    if missing:
+        raise SystemExit(f'unanchored, so not resolved: {", ".join(missing)}')
+    for name in WANTED:
+        print(f'{name} = ${resolved[name]:04x}  ({WANTED[name]})')
+
+    if not args.emit:
+        return
+    block = re.search(r'(pub mod ram \{\n)(.*?)(\n\}\n)', text, re.S)
+    if block is None:
+        raise SystemExit('no ram block in symbols.rs')
+    rows = []
+    for row in block.group(2).splitlines():
+        name = re.match(r'\s*pub const (w\w+): u16 = 0x([0-9a-f]{4});', row)
+        if name is None:
+            continue
+        rows.append((int(name.group(2), 16), name.group(1)))
+    for name in WANTED:
+        if name not in table:
+            rows.append((resolved[name], name))
+    rows = sorted(set(rows))
+    width = max(len(name) for _, name in rows)
+    body = '\n'.join(
+        f'    pub const {name}: u16 = 0x{address:04x};'.ljust(width + 31)
+        + f'// {address}'
+        for address, name in rows
+    )
+    SYMBOLS.write_text(text[: block.start(2)] + body + text[block.end(2) :])
+    print(f'symbols.rs: {len(rows)} addresses written')
+
+
+if __name__ == '__main__':
+    main()