304 lines
11 KiB
Python
304 lines
11 KiB
Python
"""mars_data.py -- reader for the Mars engine's brace-block key/value format,
|
|
built on a tokenizer with the engine's own `Mars::Script` semantics.
|
|
|
|
Covers: *.weapon, *.shipsection, *.tech, *.combat, *.def, *.script and the
|
|
block-form *.txt files (scenarios, tutorial, credits, systemnames, skydefs,
|
|
ctechvars, shipai). flat_kv.py reuses `Script` for the Data/**/*.txt tables.
|
|
|
|
The engine (findings/subsystems/loader-prototypes.md, M3) has no tree: every
|
|
loader pulls `Script::Next()` steps and interprets them itself.
|
|
|
|
step := KEY value (one token each; the key may itself be quoted)
|
|
| NAME { (a block, until the matching `}`)
|
|
| }
|
|
|
|
`parse()` runs that step loop and records it as nested dicts, so the result
|
|
is the step stream and nothing more.
|
|
|
|
Tokenizer (exact engine rules):
|
|
* whitespace is ' ', '\\t', '\\r', '\\n' only
|
|
* a bareword runs until whitespace -- braces are NOT delimiters, `{`/`}`
|
|
count only as whole tokens (`name{` is one word)
|
|
* a token starting with ", ' or ` is quoted; it ends at the same character;
|
|
no escapes; the quotes are stripped; an unterminated quote runs to EOF
|
|
* a token whose text starts with // is a comment (rest of line skipped);
|
|
checked on the extracted text, so `3//x` is one word and a quoted token
|
|
whose content starts with // is a comment too
|
|
* token text is truncated to 1023 bytes
|
|
* read_token status: 0 ok, 1 nothing read, 2 token touches end of input;
|
|
next() returns the status of its VALUE read
|
|
|
|
End-of-input consequences (each recorded in `warnings`; strict=True raises):
|
|
* input ends inside a block -> block simply ends
|
|
* `}` at top level -> ignored
|
|
* final `KEY value` whose value touches EOF (no trailing newline) -> DROPPED
|
|
* final KEY without a value / final `NAME {` -> dropped
|
|
* unterminated quote -> runs to EOF
|
|
* `{` in key position -> ordinary key (kept)
|
|
A `}` that closes a block and touches EOF is a plain close (identical
|
|
outcome to the engine, which stops there either way).
|
|
|
|
Result shape: plain dicts. A key seen once maps to its value; a key seen more
|
|
than once maps to a list (use get_list() when you want a list always).
|
|
Keys are lower-cased unless keep_case=True (the engine matches them with
|
|
_stricmp). Bareword values are typed unless typed=False.
|
|
|
|
Stdlib only.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
__all__ = ["Script", "ScriptToken", "MAX_TOKEN", "OK", "NO_INPUT", "AT_END",
|
|
"parse", "parse_file", "coerce", "get_list", "MarsSyntaxError"]
|
|
|
|
|
|
class MarsSyntaxError(ValueError):
|
|
pass
|
|
|
|
|
|
# --- tokenizer (Mars::Script) -----------------------------------------------
|
|
|
|
_WS = " \t\r\n"
|
|
_QUOTES = "\"'`"
|
|
MAX_TOKEN = 1023
|
|
OK, NO_INPUT, AT_END = 0, 1, 2
|
|
|
|
|
|
class _Raw:
|
|
__slots__ = ("text", "quoted", "unterminated", "line")
|
|
|
|
def __init__(self, text: str, quoted: bool, unterminated: bool, line: int):
|
|
self.text = text
|
|
self.quoted = quoted
|
|
self.unterminated = unterminated
|
|
self.line = line
|
|
|
|
|
|
class ScriptToken:
|
|
NONE, PAIR, OPEN, CLOSE = 0, 1, 2, 3
|
|
__slots__ = ("type", "key", "value", "key_quoted", "value_quoted",
|
|
"key_unterminated", "value_unterminated", "key_line", "value_line",
|
|
"key_status", "value_status")
|
|
|
|
def __init__(self):
|
|
self.type = self.NONE
|
|
self.key = ""
|
|
self.value = ""
|
|
self.key_quoted = self.value_quoted = False
|
|
self.key_unterminated = self.value_unterminated = False
|
|
self.key_line = self.value_line = 0
|
|
self.key_status = self.value_status = NO_INPUT
|
|
|
|
|
|
class Script:
|
|
"""Pull tokenizer: read_token / next / skip_block, as in the engine."""
|
|
|
|
def __init__(self, text: str):
|
|
self.s = text
|
|
self.pos = 0
|
|
self.line = 1
|
|
|
|
def _advance(self, p: int) -> None:
|
|
self.line += self.s.count("\n", self.pos, p)
|
|
self.pos = p
|
|
|
|
def read_token(self):
|
|
"""-> (status, _Raw | None)."""
|
|
s = self.s
|
|
n = len(s)
|
|
while True:
|
|
p = self.pos
|
|
while p < n and s[p] in _WS:
|
|
p += 1
|
|
self._advance(p)
|
|
if p >= n:
|
|
return NO_INPUT, None
|
|
line = self.line
|
|
c = s[p]
|
|
if c in _QUOTES:
|
|
close = s.find(c, p + 1)
|
|
if close < 0:
|
|
text, unterminated = s[p + 1:], True
|
|
self._advance(n)
|
|
else:
|
|
text, unterminated = s[p + 1:close], False
|
|
self._advance(close + 1)
|
|
quoted = True
|
|
else:
|
|
e = p
|
|
while e < n and s[e] not in _WS:
|
|
e += 1
|
|
text, quoted, unterminated = s[p:e], False, False
|
|
self.pos = e
|
|
if text.startswith("//"):
|
|
p = self.pos
|
|
while p < n and s[p] not in "\r\n":
|
|
p += 1
|
|
while p < n and s[p] in "\r\n":
|
|
p += 1
|
|
self._advance(p)
|
|
continue
|
|
raw = _Raw(text[:MAX_TOKEN], quoted, unterminated, line)
|
|
return (AT_END if self.pos >= n else OK), raw
|
|
|
|
def next(self):
|
|
"""-> (status, ScriptToken). Complete only when status == OK."""
|
|
t = ScriptToken()
|
|
rc, k = self.read_token()
|
|
t.key_status = rc
|
|
if k is not None:
|
|
t.key, t.key_quoted, t.key_unterminated, t.key_line = k.text, k.quoted, k.unterminated, k.line
|
|
if rc != OK:
|
|
return rc, t
|
|
if k.text == "}":
|
|
t.type = ScriptToken.CLOSE
|
|
return OK, t
|
|
rc, v = self.read_token()
|
|
t.value_status = rc
|
|
if v is not None:
|
|
t.value, t.value_quoted, t.value_unterminated, t.value_line = v.text, v.quoted, v.unterminated, v.line
|
|
t.type = ScriptToken.OPEN if t.value == "{" else ScriptToken.PAIR
|
|
return rc, t
|
|
|
|
def skip_block(self, depth: int) -> int:
|
|
while depth > 0:
|
|
rc, t = self.read_token()
|
|
if rc == NO_INPUT:
|
|
return rc
|
|
if t.text == "{":
|
|
depth += 1
|
|
elif t.text == "}":
|
|
depth -= 1
|
|
if rc != OK:
|
|
return OK if depth == 0 else rc
|
|
return OK
|
|
|
|
|
|
# --- values ---------------------------------------------------------------
|
|
|
|
_INT_RE = re.compile(r"[+-]?\d+$")
|
|
_FLOAT_RE = re.compile(r"[+-]?(\d+\.\d*|\.\d+|\d+)([eE][+-]?\d+)?$")
|
|
|
|
|
|
def coerce(tok: str) -> Any:
|
|
"""Bareword -> int / float / bool when it looks like one, else str."""
|
|
if _INT_RE.match(tok):
|
|
return int(tok)
|
|
if _FLOAT_RE.match(tok):
|
|
return float(tok)
|
|
low = tok.lower()
|
|
if low == "true":
|
|
return True
|
|
if low == "false":
|
|
return False
|
|
return tok
|
|
|
|
|
|
def _add(d: dict, key: str, value: Any) -> None:
|
|
if key in d:
|
|
cur = d[key]
|
|
if isinstance(cur, list):
|
|
cur.append(value)
|
|
else:
|
|
d[key] = [cur, value]
|
|
else:
|
|
d[key] = value
|
|
|
|
|
|
def get_list(d: dict, key: str) -> list:
|
|
"""Always return a list for a key (missing -> [], single -> [x])."""
|
|
v = d.get(key)
|
|
if v is None:
|
|
return []
|
|
return v if isinstance(v, list) else [v]
|
|
|
|
|
|
# --- parser (the loaders' step loop) --------------------------------------
|
|
|
|
_CLOSED, _EOF = 0, 1
|
|
|
|
|
|
class _Parser:
|
|
def __init__(self, text: str, typed: bool, keep_case: bool, strict: bool, warnings: list | None):
|
|
self.script = Script(text)
|
|
self.typed = typed
|
|
self.keep_case = keep_case
|
|
self.strict = strict
|
|
self.warnings = warnings if warnings is not None else []
|
|
|
|
def _warn(self, line: int, msg: str) -> None:
|
|
msg = f"line {line}: {msg}"
|
|
if self.strict:
|
|
raise MarsSyntaxError(msg)
|
|
self.warnings.append(msg)
|
|
|
|
def _key(self, name: str) -> str:
|
|
return name if self.keep_case else name.lower()
|
|
|
|
def body(self, depth: int):
|
|
d: dict = {}
|
|
sc = self.script
|
|
while True:
|
|
rc, t = sc.next()
|
|
if t.key_unterminated:
|
|
self._warn(t.key_line, "unterminated quote runs to end of input")
|
|
if t.value_unterminated:
|
|
self._warn(t.value_line, "unterminated quote runs to end of input")
|
|
if rc != OK:
|
|
if t.key_status == AT_END and t.key == "}":
|
|
if depth == 0:
|
|
self._warn(t.key_line, "stray '}' at top level")
|
|
return d, _EOF
|
|
return d, _CLOSED
|
|
if t.key_status == AT_END:
|
|
self._warn(t.key_line, f"trailing key '{t.key}' touches end of input: dropped")
|
|
elif t.key_status == OK:
|
|
if t.value_status == NO_INPUT:
|
|
self._warn(t.key_line, f"trailing key '{t.key}' has no value: dropped")
|
|
elif t.type == ScriptToken.OPEN:
|
|
self._warn(t.key_line, f"block header '{t.key}' {{ touches end of input: dropped")
|
|
else:
|
|
self._warn(t.key_line, f"final pair '{t.key}' '{t.value}' touches end of input "
|
|
"(no trailing newline): dropped")
|
|
return d, _EOF
|
|
if t.type == ScriptToken.CLOSE:
|
|
if depth == 0:
|
|
self._warn(t.key_line, "stray '}' at top level")
|
|
continue
|
|
return d, _CLOSED
|
|
if t.type == ScriptToken.OPEN:
|
|
if t.key == "{":
|
|
self._warn(t.key_line, "'{' used as a block name")
|
|
sub, how = self.body(depth + 1)
|
|
_add(d, self._key(t.key), sub)
|
|
if how == _EOF:
|
|
self._warn(sc.line, f"end of input inside block '{t.key}' (depth {depth + 1})")
|
|
return d, _EOF
|
|
continue
|
|
# PAIR
|
|
if t.key == "{":
|
|
self._warn(t.key_line, "'{' used as a key")
|
|
val: Any = t.value
|
|
if not t.value_quoted and self.typed:
|
|
val = coerce(val)
|
|
_add(d, self._key(t.key), val)
|
|
|
|
|
|
def parse(text: str, *, typed: bool = True, keep_case: bool = False,
|
|
strict: bool = False, warnings: list | None = None) -> dict:
|
|
"""Parse brace-block text into nested dicts.
|
|
|
|
typed -- convert bareword numbers/bools (quoted strings stay str)
|
|
keep_case -- keep key case instead of lower-casing
|
|
strict -- raise on anything the engine merely tolerates (see module
|
|
doc); pass warnings=[] to collect those cases instead
|
|
"""
|
|
return _Parser(text, typed, keep_case, strict, warnings).body(0)[0]
|
|
|
|
|
|
def parse_file(path, **kw) -> dict:
|
|
with open(path, "rb") as f:
|
|
raw = f.read()
|
|
return parse(raw.decode("cp1252"), **kw)
|