From 31153cd2786db6c258d948213aebf8dd6d3f986f Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 7 Sep 2026 18:10:17 -0400 Subject: [PATCH] parsers: engine parity (first-wins, Mars::Script tokenizer rules, trailing-pair drop); untrack pyc --- verify/parsers/PARITY_NOTES.md | 88 +++++ .../__pycache__/effect_txt.cpython-310.pyc | Bin 3381 -> 0 bytes .../__pycache__/flat_kv.cpython-310.pyc | Bin 5218 -> 0 bytes .../__pycache__/manifest.cpython-310.pyc | Bin 5082 -> 0 bytes .../__pycache__/mars_data.cpython-310.pyc | Bin 6793 -> 0 bytes verify/parsers/flat_kv.py | 94 +++-- verify/parsers/mars_data.py | 321 +++++++++++------- verify/parsers/verify.py | 10 +- 8 files changed, 365 insertions(+), 148 deletions(-) create mode 100644 verify/parsers/PARITY_NOTES.md delete mode 100644 verify/parsers/__pycache__/effect_txt.cpython-310.pyc delete mode 100644 verify/parsers/__pycache__/flat_kv.cpython-310.pyc delete mode 100644 verify/parsers/__pycache__/manifest.cpython-310.pyc delete mode 100644 verify/parsers/__pycache__/mars_data.cpython-310.pyc diff --git a/verify/parsers/PARITY_NOTES.md b/verify/parsers/PARITY_NOTES.md new file mode 100644 index 0000000..b4845e9 --- /dev/null +++ b/verify/parsers/PARITY_NOTES.md @@ -0,0 +1,88 @@ +# Reader parity with the original engine — 2026-09-07 + +Both reader stacks — the C++ modules in `sots-engine` (`src/mars/parse`, +`src/mars/text`) and these Python oracles — now implement the loader +semantics Ghidra confirmed in `findings/subsystems/loader-prototypes.md` +(§M1 `GlobalConsts::LoadFile`, §M3 `Mars::Script`). This file lists each rule, +where the evidence is, and what it does to the shipped data. "Before" is the +previous tree-grammar / line-based reading; all runs are over the owner's +`gob-extract` (1,595 data files). + +## Rules + +| # | rule | evidence | before | after | +|---|---|---|---|---| +| 1 | Flat kv: keys compared with `_stricmp`, **first occurrence wins**; later duplicates are "multiply defined" and ignored | §M1 `LoadFile` pseudo-code (`consts->erase(node)` after the parser runs; the "not recognized or is multiply defined" log) | `parse_kv(on_dup='last')`, exact-key dict | `on_dup='first'` default, keys folded, first spelling kept; `warnings` records duplicates | +| 2 | Flat kv is read with the `Mars::Script` stepper, not by lines: `KEY value` = two tokens, `NAME {` skipped via `SkipBlock(1)`, `}` ignored | §M1 loop (`Script::Next`, `tok.type` 1/2/3), leniency bullets ("the value is one token", "`{}` blocks … follow the `Mars::Script` tokenizer rules (M3)") | line split, first token = key, rest = value list | `flat_kv._steps()` over `mars_data.Script` | +| 3 | Final `KEY value` whose value touches EOF is **dropped** (`Next` returns the status of the value read; loops stop on non-zero) | §M3 `Next()` pseudo-code and consequence bullet; §M1 comment "EOF (or last pair without trailing newline)" | kept | dropped, warning `final pair … touches end of input` | +| 4 | Whitespace = space, tab, CR, LF only | §M3 tokenizer rules, bullet 1 | Python `\s` (also `\v \f \x1c-\x1f \xa0`), C++ `isspace` set | exact four characters | +| 5 | Braces are **not** delimiters; `{`/`}` only as whole tokens (`strcmp` on the buffer) | §M3 bullet 3 ("barewords end only at whitespace… `weapon{` is one token"), `Next()` `strcmp` | `{ } "` cut a bareword | bareword runs to whitespace; quoted `"{"`/`"}"` act as braces | +| 6 | Quotes `"`, `'`, backtick; closes at the same character; no escapes; unterminated runs to EOF | §M3 bullet 2 | `"` only; unterminated = hard error | all three; unterminated tolerated (warning) | +| 7 | A token whose extracted text starts with `//` is a comment (bare or quoted); glued `3//x` is one word | §M3 comment bullet | `//` recognised only at a token boundary in the raw text (same result for bare; quoted-`//` was a string) | test on the extracted token | +| 8 | EOF ends parsing anywhere (unclosed blocks fine); stray top-level `}` ignored | §M3 consequence bullets 1–2 | same (already lenient) | same; warnings kept | +| 9 | A quoted string in key position is a key; its value is the next token | §M3 consequence bullet 4 (systemnames) | `_items` list | pair with `key_quoted` | +| 10 | Token text capped at 1023 bytes | §M3 bullet "output is truncated silently at outMax-1" | none | cap in both readers (longest shipped token is 95 bytes) | + +Where the doc is silent we kept prior behaviour and say so: text glued after +a closing quote starts a new token (`"0 0 0"// x` in `globals.txt:230` +yields `0 0 0` then a comment — the only such spot in the data); `SkipBlock` +is assumed to count raw brace tokens; a `}` that closes a block and is the +last byte of the file is treated as a plain close (the original stops there +either way — identical outcome, no warning; 27 catalog files end that way); +the row tables (`_turrets.txt` etc.) and `.effect` files are read by other +engine classes and were not changed. + +## Effect on the shipped data + +### Flat kv (20 files, `Data/**/*.txt`) + +- **Duplicates: none.** No shipped kv file repeats a key, exactly or + case-folded, so first-wins flips no value. +- **Dropped trailing pairs: 2 files, 1 key each** (file has no trailing newline): + - `Data/Strategy/StrategyVars.txt` line 118: `CIVILIAN_BURDEN_RATIO 0.5` — never read; the game runs on the compiled-in default. + - `Data/encounters.txt` line 225: `HERALD_SPEECH_MAX_INVERVAL 45` — same. +- Multi-token values, `{}` blocks, `'`/backtick quotes, unterminated quotes: + none in shipped kv files (single quotes appear only inside `//` comments). +- `globals.txt`: 364 keys, unchanged; line 230 (`"0 0 0"// …`) reads the same. + +### Brace-block (1,116 files) + +- Canonical output changed for **one file**: `Data/Strategy/systemnames.txt`. + Its lists are bare quoted names; under `Next()` they pair up + (`"Procyon" "Deneb"` → key `procyon` = `Deneb`). `hiver` (87 names) and + `liir` (81) are odd, so their last name takes `}` as its value and the + block stays open: `tarkas` and `liir` nest inside `hiver`, `morrigi` inside + `liir`; two blocks are left open at EOF (2 warnings). The original's + system-name loader interprets that stream itself; a dedicated consumer is + needed for name-generation parity (open question in `docs/mars-parse.md`). +- Every `.weapon`, `.shipsection`, `.tech`, `.combat`, `.def`, `.script`, + scenario and other block-form `.txt` renders **byte-identically** to before; + `verify.py`'s artifacts (`tech_tree.json`, `weapons.json`, + `shipsections.json`, `strings.json`, `schema_stats.json`, `crosslink.json`, + `tech_tree.dot`) are unchanged. +- No brace file drops a pair (all end in `}` or a newline); 27 end with `}` + as the last byte (plain close); no unterminated quotes, no quoted-`//` + tokens, no `'`/backtick-quoted tokens, longest token 95 bytes + (`skydefs.txt`). +- `.effect` files (415): untouched reader, unchanged. + +## Oracle results + +| suite | before | after | +|---|---|---| +| `verify.py` | 1,595 parsed / 0 failed; 12 tolerances (12 shipsections) | 1,595 / 0; 15 tolerances (12 shipsections + `systemnames.txt` + 2 kv dropped pairs); artifacts identical | +| engine `tests/mars_parse/build_and_run.sh` (vs. updated oracle) | 1,531 / 1,531 | **1,531 / 1,531**, 51 unit tests, warning counts equal (13 files) | +| engine `tests/mars_text/build_and_run.sh` (vs. updated oracle) | 64 / 64 | **64 / 64**, 201 unit checks, 205 real-data checks | + +## Files + +- Python (this dir): `mars_data.py` (new `Script` tokenizer + step-loop + parser; `_items` gone), `flat_kv.py` (`parse_kv` over `Script`, first-wins, + `warnings=`), `verify.py` (kv warnings reported; counts unchanged). +- Engine worktree `~/sots-engine-wt/engine-parity` (branch `wip/engine-parity`): + `src/mars/parse/script.h` (new, header-only, shared), `blocks.{h,cpp}`, + `src/mars/text/flat_kv.{h,cpp}`, `result.h` (new Problem kinds), + `tests/mars_parse/{test_blocks,canon}.cpp`, + `tests/mars_text/{unit_tests,realdata_test,dump_json}.cpp`, + `docs/mars-parse.md`, `docs/mars-text.md`. No CMake or build-script edits; + no game data or decompiler text. diff --git a/verify/parsers/__pycache__/effect_txt.cpython-310.pyc b/verify/parsers/__pycache__/effect_txt.cpython-310.pyc deleted file mode 100644 index dc1e5ed0acc95a36c550f28cbbf12ef5918ba69c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3381 zcmZuz-EJI76|U;;`D>3I+u3BDz%HdEU=JI6cEgX5leJ>vNw79M4z>wo*l1{X)r=eW zbdRfQ?AYqL!9lPRTEq=EOGJ#|VkO>y2jCs*60Y(J+$`r*k0*|l9(7Hfs#B-V?{_+# zob(Ai7cTtu*Z-*!@?Rv*KOPWw38Cr_FyW*{IOR@Edrr%tICo>W=e0aL_u@jY*edo) zt&*h+ak*D%RVe8aUi=$v`C_6~eL#4Lm!A_}7LzA#Yl>I6{};-?$!GZN^J41)e~(|{ zm!EsBY2bX9&jIJ0m;lZh{yx9LufonO|Hc-neJkKDA)?a}p>FGEdZ~ZJ78jWm0T+^W zQpp;2Pu0J>lyk9#ejs%e#$s`_z(prYA|0iPsxA3`b7Pz7UBPx_5Q@c}I1TquP%qF+ zj6DpF(t%#~eFo3=FShaT?ay`A3%XIrI+0YG#Ze;cnNJ!|*g+5v#M@3?XDSThK(d8o z(AyF6CfnZpS)+b`_2E|ICcCyjNVVYCYGXlflVR7cv#v}B{bh#Nr3s6)V(DRG)wqe) zB~}LV{=EL>-4HgdM zfl$C2XtpEbH0d&(<|GevppW|ECJPeI=0Dw7Tfe{F*qryJ=!*aiO+vx?pcQgK%(hY{ zf^e6`;y}cxj1+nx6Hq2lAcLi70#I=Tfl536FZaaJ%{)j?SwDdIv4z|VAD_hs@@nKF zJXC{AblT=os26Ii8=>Zbk0d*Y#35Ta(`<3YUZQI35JHe&j0z>G26=)D4*B=fK_vGS z+Xd@l!I!}t0Z9}KSW)cW=EM89*?e>L(+0cN1lf&Vgs#6fpDV0bm?k=i5?}QpebsFi zC0xJ?rZ-*{rm80({n>&Jc2w=QjYgi{xQ%(bxDzO=$(H7Ev;zvnM@#-I1cVwj6F~y% zT6qdccQrYxIi{-maRe2Z?ukU1Nu_1fZ-;5G2X}u(O)*S`3`NZ|1*kqHjBo$iXqPFC zd}n6zTHQJVeV;X?Ol63do_{D1cM*vIridUd3TWhT3P|9{30P{@TvKkhlb|QsZR5Aw zy_64PoLAfJ{Xr1tJ6{ob0l4k(zj}Q)?TLC2i)VF}YPBdu{Xod5b5yrlN>$I1VgxYu zqU&&6Aug7QNA1cs*jd1oX@lbok_p z0@|U=S?PohD_I5Y#CPa}+{#Gq=b4W;{*H7gpW+vumw|6&>l%S`6Ju@h)Wqo2Ke7pw zl+bJ*xv~xzy5wLywg@c-m^!zvIIb$`PJ+lO5L6Ti;N(`5A0s4<;dXZ8Q zAQSch3-+&Y^Ob`f1&gc41uQ6bd}%qyS#}udU1%UIU;r|&zJ|xRfTPD3ml^zxxAVir zW#nSVmn}e?hfupe&LFrCGlyV0Uc?N#0h2D|(9W}Cu&}F-G z8UCvyzN>TrX1Id8Rw@D`pn~zRZM+d#vL*8G#z#fap3y7ozHFdg90r zGFN_p6lQC_qMe+As&Jr2$Geup#WR2qP{Bh$kc=J>xq&&70gl-7E4q+V+~wYhs~$kr z*xfGV5zg+6xL=ks7tvuKq~+@{Ls%~46(g^L;>K%3C%j)P8$W`=Q96K5jqZmye+TyE zJ2>E{s3~;?^cXGS$foGaZ^K&6lech-a`_?tyAYt)C@yV$@N5%ag_1n*@e262Oy7ry zOjF+hq(+Vq-ZvgB+q>vq6!N;Ov|J8FQgh{xV0pU<3Ls9)NgSqd_&yHfM)JC3?19f< zbJQUq#wg;AuwubtI(>roVmg820sKy!KVBL-8eIF*IVCSagHzg}5xDv9^b7iP;s5}9 z@H=rS0MbW9y>|$1$KMlpIf9hiG=Rp?bNe6N`oS&exn2snpJ$neU&f}$MVN8{Mep?k zy=$DkLqj{~EV0h4fKtDLS0O-D=^VZBwZye%t(aGt4Hq^zc?pNhIJ^tPC{lKwuYM0# zopT%)O{v}HX_zCa8{{w*nLudd1 diff --git a/verify/parsers/__pycache__/flat_kv.cpython-310.pyc b/verify/parsers/__pycache__/flat_kv.cpython-310.pyc deleted file mode 100644 index bbe3d5c82174cfed459b7b16e58bff6a5ae2e375..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5218 zcma)A+jARN8Q-%PNs(`9>bhydgj1)jwW%y6>5V#RptnFNm{O-Hb!s$gog-U#wJV=p z*|AwMvh-&x7B9T$*hkG?&ZZ_oMtzS}CE zm?&HLOilgyXAe$W)<5Xx@W;i?cPxt~e}f>bWlJz&hpb~S+l*uR0(gzil^(6(=X%iD2hHAw51Xd16{;w1H} zxi%R!<6e})HjD~htAVH?D=OYjhPHE`5p-I24Y~tZhuqY=VXz}rZZ%)8Yv42H=N7qOPR7uK^54uYv+{>*@dEicAGYG%hMA>Ee$%G zo*Jn#9lKMrl{-J3UAg4XW|dEuUi7s~AxJyWsSB{Ro{E}`wmkTF_@$)tHYJ^P2!i#I za{6{^W!62`cQQM3HY{~DvoiY`tM6v6a5ijpHe=&fpG9YI@8bSa#*#~!wa0{)+52>l zWR%&z!kQ(l6vfx9zMbnz#xwN!rQL6TC_C{Mv5$b##y;10y*}i;%G+yEOd*gu2X-Rj z9!xVgmZWx5(McY$*+;CtYuD?GU(&sAYbTVEE+Y!7G}&&a>%ONZF{0ge)Y#Olty!d5 z^P#2A(fVqAJ?_YQ5X!stBu@(%E?^3^yFS5}VT??6idm%;x72x~w!1&*1u&)%_CE)Z&awqW~U zrN`9yjO~GwL66h2_ZhP`iwC_f=6R!D;9S_PE|CQRW-hASCIUrVL(vFQ;_>Jr@^S#Xm-yvatAqyl7c=Zqh3c+0iiC_5(*RQ z3N_>aY8s7y>JTXfVh{1{B^ssRq+Uj&9b_c!BFQD%L3@lPUC8(FLrhI((AdA6IUs5r zRB?Eh`!BUXs@yO@wzG2^DCFr&T{k15-AoUVAzF#wgD6BtZd;%Zgu<+eO#Z3Ucl%zy zka?N2WvPpZ=tRYZ6h4%>Us%G)yf3Ujf!NO(#wa6PL{utd=)X?!iCzo5-M^12>tPYb zQNhuEP%*tZD3m*RQ{Zt6B^kmns{LLxs1EWRXlf7S>S0!=xR%iR{EVn9=;U*}f;Wso z3C_2XKu~bbwGiUcIS5jzSk2Xpu8&Y6AJha%#YdHi$^kDRc{zb{fQpNn_ROMP&xFz4 z(+i_TG=Xc+Xb0p5|MG2muX&fA;}=KF=M*rZUIomGBBOep8cIcZWBifW_ykvS294`Z znh36luzhr%Rg7#}4+HRc^Q$P_tLCkaK*Bp4>Mu$Rf#RJQ?q=#>OeMb}%U+BcDZR9npL!v>r_L-2rupOb11{38&ga-s>bNBPgnBH<2PdTiU|DP%f(*3Zn^4M>Mi_&zd6k5W2%@Q1(71Ay-lI0*T*$T&tqI(#2&6 zo3YLPV`m^x4M-HthQ|E+sXo##B_PyBFyN0Z8-Jj<_}+6E&UD2}p5BI8)AtixCRI3# z+E%M5(z{o#zH-%fv>T(O(JuX2p$h`9i$bEFbh|;iu376i23~_E_GrK4kCZf@`?tCc z@v%Jpv_*HS;g05Bp&M#V#0Q6$#(#`dx2Dk`ORahZx? znwfhl^ifQZ5e+}Y_wc#uW84g%LE+v!L;ZxLqs*!a^!fo=AhM>Qd zD6G=bAPljzNM#(}$kYXN=h=UZBm?yVNrqT5U1QWa{6wTd_z(w%4axz!biEUcUMO!6 ksU}9c*7&)S4Lh?-cGhxH7?96z1s$432WpC2`4-jD4O#lD@ diff --git a/verify/parsers/__pycache__/manifest.cpython-310.pyc b/verify/parsers/__pycache__/manifest.cpython-310.pyc deleted file mode 100644 index 643da22874a9ab614aa410011084a44f3465b758..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5082 zcmZ`-TW=f372erhUUjv6H{x_^D-jusq#Yzck+4DwUlO&cl8R2;Fjj~aXGE>N+@)q# zl0{HJ3K{5AS`>Xuf=WeSjQ)W>_IK=SpYj*bIzPOm+zcWG&NPU z@SBO`B2Q37n>5-fq)1?@Tj;w;N=; zxn{1LZ|0fRu|(z{tXYtm!=lK_%(ICjyE$pTa!0H=C37EGA}KCa!8fMy**y3@OV@5aJ#!Ml@#0;d^D}O7YZ}GaC@QBj=EhL z##nN{@tLj`i#<yCL@?$M1>sdQqrrt=+U!jbFruZKEgKzSOn) zs@IlFH7(n*Zx-rJzc>1#)$bWT#H;Ryam5!)yuRTFGW5D~iH6f(6iVKciU)ouPbg|N z{@~$94DW)n*dLy_q+>q_xRP3`9VyC&!Lon?lr#zeXYK14=jWk}cO&(j z*SPj}WCv0}XS&MyqoGhKRo9E#n^G7~ZBNUJA8HwDKlXQIu*XZJbUJDe3LWWC^G&ei z3lAPX@l>_(;qoVqhxZ>Zf4s7=X!tDLU;Xv+{c7V>LknES?JD;=9@xh`h<2r_&~}_u zq{+HZ{7^)@nm=lkPmI>qJE!&vaIh~-qhqzUW_DU7id{e6zAWc6^VgfdJku(pYQL$WtP1H)0X^>o`uZS1o=|a8hQ;gcpWU>=}1oDceawc(KK|#tE#ccmtt@c#E zHH;JVDtJmy#hr5h@|UCTQ+=oOv~j15@A)>JWBdn1Nulu%=t!%{0FU;kmUJY5@M{Jb zv}IN}_0y4@a50h@8_5F>vm#fw#H=Wa2>@3CAeTa4QOp3k=2Qmkp}!d+H#|5;Yr|{D zk=pBDHqIMw_X7C^AYu{bgdZn4M5_YC^b2MlGZ+&xkZ_;lbc_}z`JRf_1KHI{HnsBc zwD8;UDXTUcJ1zBXJohOVP|Ap{f%S|Xu!i-DkvUx=R#TXH#S&M`VB?jguHy5!QrnEW zvgQTyMNLPst|(dCk;>oLs~ICHT^k_*kU7b$@3nkU0;Q%6i>yCA@`|dN8p?@7EW(N7 zwf`^YEa;Jo?3W~~^}r%&%TE8sDP9Nz7=)<=;l4P-iwsQpuR&+NfsPM8YlWxwKHIkk z_MvrT4cOo8ExVDP6h_Jz)@3KLt9KLTCoD8NNSkY5kreBsD1>ouse*5F#v4|De*BCi zPqiLIZ7ryVnsxD=s@gKdnsXRJcbN00HuOYqCsIm=n&x`Nk6NAoGp=otx5tOK!x)^gv_V`}zICAV08o zEY&$fn;qoAAvdrG`Rf*X`R`B*F>Bz^NFAQ_Z2B+<%EAWZFfu__oN9JIKjinPAwTd3 zeepZTtlJ%smrtyGPa%73*J}NY$tL}o_xN_GWIO7Fz6PX!PvJ$Q0ubW20q#@<^=El0 z&2~sBhVYn``wq?AZ*!iW-b#VgihZG5uaL z(HhdWRv#u=Ej`uVG+vWrJ><(!Bo1Oa$#hh--2(&4X`1p&QfNI~dA#~L(Pd1Pk@^`S zPPwSw#4BN4LLM0=S=uv<5=VKvY4jcx!xo7FCyRbR^eU3+9tn92LwpvfJ?cTk&-+dbp<3`X|;!;LSvI2G$l5n`^ zL0N3${P`xZ|3`QniB&N`eEsCiFgY)Vl1^^nwz>ort9#H=@ zz)fuDF!p;q2HGZ>qHkCt+EutO=a6r9m(?XSY=u?0#}HO`!f7X8#D%wSw6I8wu;V{TWog zWfODg(^`4OR2A!9*@~7YtbOjlNul16p?uL(OJ87tCXlqh0;a#-@^x!iAdbQLXfaHf z4NO!h^0~zEf&l7$M1535C^*k=G%?2TQ`(A5n(Oa1NNd2yC>0piI3k9N|Lm z4(UB)gA{!*3$)IBgOy9Q5;3K3wa zu#HvDiJ&O9zoflWGh71{*z9!}!pt0(($efi`Fy}vP~k5ERnOc!PK^RyXN}UsfMEM< z%qX^ShDLdViV%H)3?f{FiHk5nS~|=KBp!7q&JJ*u7TM>nx+8KT4^!D>s(k=Un2V8k z4w$}&J_84a3+t&^wz>lvf8QQX2t7tFdUVrxZ zLVan03S4t@eBUUN?{3PFB*A4Bi7AY@-fnM_4As>A6#UCFNTA5ORSC_H{E(_^aL#<#W`@M%Bh1#6I8qIHF&EH z__)HR(K89}he=z}svl{JA{K`P0;4lZ3>zrC2JD~g&p^vjf*^s?1Yw&5k=W07&&yH3 zniqJ|xMhmIM`5=>^cScxSD#_YN~2Kt>A0Jol9Ea~`T?!TrlD(i7~{}rI=@Qu!_YSK zh8JOW`AU*&wM5iTtKW@n+E0c+nYpc|XvQ}CYgBTa%l**5t*!ork?fb?@r#8O{tpmZDZ`OI|7EkpObIesJdSG{`Q^%#whS1kOlT=~Ppv7cDhf6~MGUjYxxmL=kU2cWD?O9^EMqGfN|f?y|b zHXYpEz-<*a3$5a2QCNGHD*RDwj(ljTq8j}-~`b5c?#$P!Vn-dI80=l9W)br{p-cG95)l2FpfSyw~)ywJ?KvS6cZ5Tx`j(>0CXPNWwqTDQV$%+t&uNuDHf*AH|oXX2&~ zJ>y65oJ=}NWPC4}lTD+Qp8yqi<3zVYuchhXpo_WVxk{53eTl^oP^4hhuYsC3Y3gOEn##mFYHRvpz0HovhL})6C}`z*VcTp>%8vDP7` zXlhK*CdP?f$r5-MEDuO3tq77!_PtBQo(%jrku#lG%RQYm=xL@V5BD)^H|oek4`S*Q zdBLIAjltIKb6Oy+#gY)sCO$2Fz&SR7>W-)OH1N0iUX$}nXY&Tpp*HhP%ubtu|HXKx+YQ3JzhOF>+31@b z1mBfWo7KRp$50}~3bn$;JlQt72{(nQl3noPGu5PHj80Gbs+1f)f&fw z!Q^BSyaILzJzrdaC&MD7at=}0avp(b9%gdKYaYab7w;p8^j+nG|Kx%Updtg=&wSyB zDmsk8YO){K-lgTBT0iEM!bs}42~$M82@~%LFTf5obixaBlkw02+>l1%oF+zlcR4#NB_ z%4+?s>YS|3*WaPr+xo^k)k@`lq5^*h!V9|f%D0r$Y9br;La^phc#hXv-K_Kx zi2>2IR>%r$eOZNX4T{29u^DO8)LCiZDcKcD6Zd)XRbwL4q}k8^1Rj>vP~JNwVIQa8V%BNqmfk_jaH;O0l{O9#-olG6_@T|VDAGP#?7w9Jo zag>mMEFgIMyWNGp$1I#rZoJMszK&fCCCe3zIr=T@bmrEl?;XZVTk#E&ZYt@yudStp zg$KXf3b$v^-jJCuZ9P3hG@%{6wADMqx_~r#35&!} zd(~~`K+4Qb^phlW5Z|*RoTOvIXRizt^C0Q81N{K+5F%*s!(@BANn#%SMgWv>K^G$a zF983H@}!4irDrErY8{K7liI0sV3|jWII&YJwNLGyo4U&BSY|qLl$(mLQMk~X)K`P>q?pPwx(63y@(?;#$il7`EgiKFFT z_!vcCi}L1tBTVKP4T)*GO@5{aiZ7iM3b%mgHz?SbJ7r-#nUHM9v~t zL2NzHAqA{&uYw@6593T!vXQ>AWaXiPk6u_r)PyIs{Lnkg)LAh%jc>ohuU5*OK!;hm zv9bR1)wRaG)vSP8s=3d$nUxW-l06f3+F2pLIfxQjp-C~aHe#+22QG1#Iqe9Hgqdh% zF6}XM!04H`mZ-mT~Wf{7%nM_441@&umRCEiR&uHF?;gAO%0zM zl^@)g-@dh_ZoGC>V{iuWR((s&rZnD2an)uX=+*53VeQsyW(J=c2g3z3wSgey9Wv9u z2as4NBDGH8oe7Nc7%?Jo`k1N2%`vIPW+zgi1dQgovhTx4lmob`T)-A0OW|mlGW20& z6lFDdh+OE$$pzFfQIMy1AtNlzJZbBF=R$sB3sLqGbQvS5ntu-#>*he#)gG#{n~m ziL*k|#knPO$-cAluv*KCFlREP(xGQUDl?vqa>biUwx%M!9sCF;ZtoDcw*e$1ZgGsi z)M`Kp7%6pF4Hx!fHIM`_A}A}XBVcn#7>yCKu01F(hidj3htpb#rOHtRz6Gz#gm0Fy zhM|QFmb45B2%hHBX(Qj+GNCULT+##;4}Rz;jm9x(+1Mr!%JRSK;%KrjsrrB{b6WA9 zd5$LjTMQR$&<0{C0P^$HdT5!ETnNZ~)6w=H; z&~V8$3Jp@lzu-+w5~sL3DUQa6jMCc+k&^P!kS&P^Vhc=xonwJCK0y%SUWDvP{yAfg z`4F_`It_O*)I`qH_td+n;b5$EJJ~l>P?|R}T;OFC2nzAc%j-~_ zkJ_jVj!rC8Nlxm(RV?%4RG4ePp(f)>%lXD3o8pM(Y&kvDS5#Rk7isNH(AlS43&l72 z8`MCk&Wiarj}RTPPnWB&Td1w_x21ks4_ z`l7|FHZM`;Z>Yjc2mBBP;6hqBh2~FCm#MKIA$~fP}aA?b0E;tlH`0|;kp%$VF&QKmb`tc%VZ5%_< zWsx`z4cix)OV!@dON-oyIrNkLs6*$md^R#A^AaY=iuoCxV-;I00x3#}TG@Pz*XHLm ztkJ-3&Kc}e`VQe%b@MnzYOjsvF7_Fba+dsrplKp?ki^Za427`cLdeCM8v>n)#w8;( zUx4syTpWj8F2^XzOgV19VgPL0z~p5bia0z`xkh&?=&s|N3H+7Bl(7Hf%!Y~KwOZvY zbL8ApS(v*AL$F`VFhoJwtYSc*<7hTXh++H$mVM|9OAxRu0(KInB~O7gsl{r@NerC_ z%||q_a9PYO;ZF%i+dn4~4p~&lnp+rtg)63~l2s9vQmIgKDuoG$-)-E*+RJ!%Ic+fR zY?r*j;ogQZ&c&AnToivmBI0WR=+uP53g?Z~Jubj&kpU>SdA|{s#fkVrf0RydRCGo% z)aXhgd2_=m?;>)RMzaGy^jw=!h=Xgwrw=Yy`)B(F%Hn7NT<9jmuyL3;44n>}IM56_ zGCUPMQ3D6371Xd)x}l#-PBiT0Ao=rmc|kxkaNF1OL662tC*1J@D)-QTqi!#^O9qpm zg+nb~s-UP(g+3Th#ghyNdeiSbsfaoltLu03a~3+SaKnCIj}NZc$gp4d15=S7`+FgJ zb_z|Hwuc7PpniG(!8WzI`rS5WXTY7#+cbH_VY5B3CDjW5-*Uo`^8bR1WB;`ZKM$<~ zc%-=ebPcN=99vSHuaE729ivK>@=eeA0%_RAAH}q=bQ9J7p%94S!uka+9B-TxN1+(s z6+SHMPvR@GCuc3Q%?_F@+y#OeZ(ym+rM^6~krnffP|eL;YENbM!C@wLGsk$}J7$pm z{uLM3r(*~i*T1`F8^U5j70xA{o8^YRrxWS4)1O1Y>j-rdhho7{Z;ca@j+)@Ge<+628!(YsfDWj z+^on>Se`!=rLxj{c^hj90m6#uG~-%hua#_^sO?F|Eh2E2-T$dfRIVd{Ik=Bbl-w0p JOnf&n{(nEDe%t^6 diff --git a/verify/parsers/flat_kv.py b/verify/parsers/flat_kv.py index 1ce67cd..d18e608 100644 --- a/verify/parsers/flat_kv.py +++ b/verify/parsers/flat_kv.py @@ -4,11 +4,22 @@ whitespace-positional tables under Data/, Weapons/, Badges/, Avatars/, GUI/. Two shapes exist: parse_kv(text) KEY value -> {KEY: value} - one constant per line; value is a bareword or a - "quoted string"; '//' comments; colors are quoted - "r g b" (use color()). Files: Data/globals.txt, - Data/species.txt, Data/Strategy/StrategyVars.txt, - Data/Combat/*.txt (most), Data/encounters.txt, ... + the engine's GlobalConsts loader (loader-prototypes.md + M1): the file is stepped through with the Mars::Script + tokenizer (mars_data.Script), NOT read by lines. + KEY value one token each: `COLOR "48 29 2"` must be + quoted; `LIST 1 2 3` is LIST=1 and 2=3 + NAME { a block, skipped to its matching `}` + } ignored + Keys match case-insensitively (_stricmp) and the FIRST + occurrence wins: a key is consumed on first sight, later + duplicates are "multiply defined" and ignored. A final + pair whose value touches EOF (no trailing newline) is + DROPPED -- StrategyVars.txt and encounters.txt each lose + their last key this way in the shipped data. + Files: Data/globals.txt, Data/species.txt, + Data/Strategy/StrategyVars.txt, Data/Combat/*.txt, + Data/encounters.txt, ... parse_rows(text) tok tok tok ... -> [[tok, ...], ...] one record per line, whitespace separated, quoted @@ -17,10 +28,7 @@ Two shapes exist: Data/Combat/damfx*.txt, Data/Strategy/playercolors.txt, Badges/BadgeTable.txt, Avatars/AvatarTable.txt, GUI/WeaponIconPlacements.txt - -Quirks handled: '//' inside a quoted value is not a comment; a quoted -value may be empty (""); keys repeat in a few files (kept as list); -duplicate-key detection is exposed via parse_kv(..., on_dup=). + (read by other engine code; line-based, unchanged) Stdlib only. """ @@ -29,7 +37,7 @@ from __future__ import annotations import re from typing import Any -from mars_data import coerce +from mars_data import AT_END, NO_INPUT, OK, Script, ScriptToken, coerce __all__ = ["parse_kv", "parse_rows", "duplicates", "color", "strip_comment", "split_tokens", "parse_kv_file", "parse_rows_file"] @@ -37,6 +45,8 @@ __all__ = ["parse_kv", "parse_rows", "duplicates", "color", "strip_comment", _TOK_RE = re.compile(r'"([^"]*)"|(\S+)') +# --- row tables (line-based) ------------------------------------------------ + def strip_comment(line: str) -> str: """Remove a trailing // comment, ignoring // inside double quotes.""" in_q = False @@ -74,38 +84,64 @@ def parse_rows(text: str, *, typed: bool = True) -> list[list[Any]]: return rows -def _pairs(text: str, typed: bool): - for lineno, raw in enumerate(text.splitlines(), 1): - line = strip_comment(raw).strip() - if not line: - continue - toks = split_tokens(line) - key = toks[0][0] - vals = [coerce(t) if (typed and not q) else t for t, q in toks[1:]] - val: Any = None if not vals else (vals[0] if len(vals) == 1 else vals) - yield lineno, key, val +# --- KEY value tables (the GlobalConsts loader loop) ------------------------- + +def _steps(text: str, warnings: list): + """Yield (line, key, value_text, value_quoted) for every complete pair + the loader would consume; record every tolerance in `warnings`.""" + sc = Script(text) + while True: + rc, t = sc.next() + if t.key_unterminated: + warnings.append(f"line {t.key_line}: unterminated quote runs to end of input") + if t.value_unterminated: + warnings.append(f"line {t.value_line}: unterminated quote runs to end of input") + if rc != OK: + if t.key_status != NO_INPUT and t.key != "}": + if t.value_status == NO_INPUT or t.key_status == AT_END: + warnings.append(f"line {t.key_line}: trailing key '{t.key}' without a value: dropped") + else: + warnings.append(f"line {t.key_line}: final pair '{t.key}' '{t.value}' touches end of " + "input (no trailing newline): dropped") + return + if t.type == ScriptToken.PAIR: + yield t.key_line, t.key, t.value, t.value_quoted + elif t.type == ScriptToken.OPEN: + warnings.append(f"line {t.key_line}: block '{t.key}' skipped") + if sc.skip_block(1) != OK: + return + # CLOSE at top level: ignored -def parse_kv(text: str, *, typed: bool = True, on_dup: str = "last") -> dict: - """KEY value per line -> dict. A value made of several unquoted tokens - is kept as a list. on_dup: 'last' (later line wins), 'first', 'error'. - Use duplicates() to find repeated keys.""" +def parse_kv(text: str, *, typed: bool = True, on_dup: str = "first", + warnings: list | None = None) -> dict: + """KEY value per line -> dict keyed by the first-seen spelling. + on_dup: 'first' (engine: first occurrence wins), 'last', 'error'. + Pass warnings=[] to collect dropped pairs, duplicates and skipped blocks.""" + w = warnings if warnings is not None else [] d: dict = {} - for lineno, key, val in _pairs(text, typed): - if key in d: + spelling: dict[str, str] = {} # folded key -> first spelling + for lineno, key, txt, quoted in _steps(text, w): + val = coerce(txt) if (typed and not quoted) else txt + lk = key.lower() + if lk in spelling: if on_dup == "error": raise ValueError(f"line {lineno}: duplicate key {key}") + w.append(f"line {lineno}: '{key}' multiply defined (first occurrence kept)") if on_dup == "first": continue + d[spelling[lk]] = val + continue + spelling[lk] = key d[key] = val return d def duplicates(text: str) -> dict[str, list[int]]: - """key -> line numbers, for keys that appear more than once.""" + """folded key -> line numbers, for keys that appear more than once.""" seen: dict[str, list[int]] = {} - for lineno, key, _ in _pairs(text, False): - seen.setdefault(key, []).append(lineno) + for lineno, key, _, _ in _steps(text, []): + seen.setdefault(key.lower(), []).append(lineno) return {k: v for k, v in seen.items() if len(v) > 1} diff --git a/verify/parsers/mars_data.py b/verify/parsers/mars_data.py index fe5dcd9..06359d4 100644 --- a/verify/parsers/mars_data.py +++ b/verify/parsers/mars_data.py @@ -1,94 +1,182 @@ -"""mars_data.py -- reader for the Mars engine's brace-block key/value format. +"""mars_data.py -- reader for the Mars engine's brace-block key/value format, +built on a tokenizer with the engine's own `Mars::Script` semantics. Covers: *.weapon, *.shipsection, *.tech, *.combat, *.def, *.script and the block-form *.txt files (scenarios, tutorial, credits, systemnames, skydefs, -ctechvars, shipai). +ctechvars, shipai). flat_kv.py reuses `Script` for the Data/**/*.txt tables. -Grammar (as observed in every shipped SOTS1 file): +The engine (findings/subsystems/loader-prototypes.md, M3) has no tree: every +loader pulls `Script::Next()` steps and interprets them itself. - body := (block | pair | item)* - block := NAME '{' body '}' - pair := NAME value - item := QUOTED # bare quoted string inside a block - value := QUOTED | BAREWORD - comment := '//' .* EOL + step := KEY value (one token each; the key may itself be quoted) + | NAME { (a block, until the matching `}`) + | } -A file's top level is itself a body (scenario .txt files mix top-level pairs -and player{} blocks; catalog files hold one or many named blocks). +`parse()` runs that step loop and records it as nested dicts, so the result +is the step stream and nothing more. -Result shape: plain dicts. A key seen once maps to its value; a key seen -more than once maps to a list (use get_list() when you want a list always). -Bare quoted items are collected under the key "_items". +Tokenizer (exact engine rules): + * whitespace is ' ', '\\t', '\\r', '\\n' only + * a bareword runs until whitespace -- braces are NOT delimiters, `{`/`}` + count only as whole tokens (`name{` is one word) + * a token starting with ", ' or ` is quoted; it ends at the same character; + no escapes; the quotes are stripped; an unterminated quote runs to EOF + * a token whose text starts with // is a comment (rest of line skipped); + checked on the extracted text, so `3//x` is one word and a quoted token + whose content starts with // is a comment too + * token text is truncated to 1023 bytes + * read_token status: 0 ok, 1 nothing read, 2 token touches end of input; + next() returns the status of its VALUE read -Quirks handled (all seen in the real data, see parsers-report.md): - * keys are case-insensitive to the engine ("Requires"/"requires", - "badge"/"Badge") -> keys are lower-cased unless keep_case=True - * a block may open on the same line as a preceding pair - ("turretsize small mount {") and a block name may sit on the same line - as its brace ("weapon {") - * backslashes inside quoted strings are literal (Windows paths); there is - no escape syntax - * '//' inside a quoted string is not a comment - * CRLF and LF line endings, cp1252 bytes (decoded losslessly) - * numbers use C float syntax: ".5", "-.8", "7e+8" +End-of-input consequences (each recorded in `warnings`; strict=True raises): + * input ends inside a block -> block simply ends + * `}` at top level -> ignored + * final `KEY value` whose value touches EOF (no trailing newline) -> DROPPED + * final KEY without a value / final `NAME {` -> dropped + * unterminated quote -> runs to EOF + * `{` in key position -> ordinary key (kept) + A `}` that closes a block and touches EOF is a plain close (identical + outcome to the engine, which stops there either way). + +Result shape: plain dicts. A key seen once maps to its value; a key seen more +than once maps to a list (use get_list() when you want a list always). +Keys are lower-cased unless keep_case=True (the engine matches them with +_stricmp). Bareword values are typed unless typed=False. Stdlib only. """ from __future__ import annotations import re -from typing import Any, Iterator +from typing import Any -__all__ = ["parse", "parse_file", "coerce", "get_list", "MarsSyntaxError"] +__all__ = ["Script", "ScriptToken", "MAX_TOKEN", "OK", "NO_INPUT", "AT_END", + "parse", "parse_file", "coerce", "get_list", "MarsSyntaxError"] class MarsSyntaxError(ValueError): pass -# --- tokenizer ------------------------------------------------------------- +# --- tokenizer (Mars::Script) ----------------------------------------------- -_TOKEN_RE = re.compile( - r""" - (?P\s+) - | (?P//[^\n]*) - | (?P\{) - | (?P\}) - | (?P"[^"]*") - | (?P") - | (?P[^\s{}"]+) - """, - re.VERBOSE, -) +_WS = " \t\r\n" +_QUOTES = "\"'`" +MAX_TOKEN = 1023 +OK, NO_INPUT, AT_END = 0, 1, 2 -def _tokenize(text: str) -> Iterator[tuple[str, str, int]]: - """Yield (kind, value, line). kind in {open, close, quoted, bare}.""" - line = 1 - pos = 0 - n = len(text) - while pos < n: - m = _TOKEN_RE.match(text, pos) - if m is None: # pragma: no cover - regex is exhaustive - raise MarsSyntaxError(f"line {line}: cannot tokenize {text[pos:pos+20]!r}") - kind = m.lastgroup - tok = m.group() - pos = m.end() - if kind == "ws": - line += tok.count("\n") - continue - if kind == "comment": - continue - if kind == "bad_quote": - raise MarsSyntaxError(f"line {line}: unterminated string") - if kind == "quoted": - yield kind, tok[1:-1], line - line += tok.count("\n") - else: - yield kind, tok, line +class _Raw: + __slots__ = ("text", "quoted", "unterminated", "line") + + def __init__(self, text: str, quoted: bool, unterminated: bool, line: int): + self.text = text + self.quoted = quoted + self.unterminated = unterminated + self.line = line -# --- parser ---------------------------------------------------------------- +class ScriptToken: + NONE, PAIR, OPEN, CLOSE = 0, 1, 2, 3 + __slots__ = ("type", "key", "value", "key_quoted", "value_quoted", + "key_unterminated", "value_unterminated", "key_line", "value_line", + "key_status", "value_status") + + def __init__(self): + self.type = self.NONE + self.key = "" + self.value = "" + self.key_quoted = self.value_quoted = False + self.key_unterminated = self.value_unterminated = False + self.key_line = self.value_line = 0 + self.key_status = self.value_status = NO_INPUT + + +class Script: + """Pull tokenizer: read_token / next / skip_block, as in the engine.""" + + def __init__(self, text: str): + self.s = text + self.pos = 0 + self.line = 1 + + def _advance(self, p: int) -> None: + self.line += self.s.count("\n", self.pos, p) + self.pos = p + + def read_token(self): + """-> (status, _Raw | None).""" + s = self.s + n = len(s) + while True: + p = self.pos + while p < n and s[p] in _WS: + p += 1 + self._advance(p) + if p >= n: + return NO_INPUT, None + line = self.line + c = s[p] + if c in _QUOTES: + close = s.find(c, p + 1) + if close < 0: + text, unterminated = s[p + 1:], True + self._advance(n) + else: + text, unterminated = s[p + 1:close], False + self._advance(close + 1) + quoted = True + else: + e = p + while e < n and s[e] not in _WS: + e += 1 + text, quoted, unterminated = s[p:e], False, False + self.pos = e + if text.startswith("//"): + p = self.pos + while p < n and s[p] not in "\r\n": + p += 1 + while p < n and s[p] in "\r\n": + p += 1 + self._advance(p) + continue + raw = _Raw(text[:MAX_TOKEN], quoted, unterminated, line) + return (AT_END if self.pos >= n else OK), raw + + def next(self): + """-> (status, ScriptToken). Complete only when status == OK.""" + t = ScriptToken() + rc, k = self.read_token() + t.key_status = rc + if k is not None: + t.key, t.key_quoted, t.key_unterminated, t.key_line = k.text, k.quoted, k.unterminated, k.line + if rc != OK: + return rc, t + if k.text == "}": + t.type = ScriptToken.CLOSE + return OK, t + rc, v = self.read_token() + t.value_status = rc + if v is not None: + t.value, t.value_quoted, t.value_unterminated, t.value_line = v.text, v.quoted, v.unterminated, v.line + t.type = ScriptToken.OPEN if t.value == "{" else ScriptToken.PAIR + return rc, t + + def skip_block(self, depth: int) -> int: + while depth > 0: + rc, t = self.read_token() + if rc == NO_INPUT: + return rc + if t.text == "{": + depth += 1 + elif t.text == "}": + depth -= 1 + if rc != OK: + return OK if depth == 0 else rc + return OK + + +# --- values --------------------------------------------------------------- _INT_RE = re.compile(r"[+-]?\d+$") _FLOAT_RE = re.compile(r"[+-]?(\d+\.\d*|\.\d+|\d+)([eE][+-]?\d+)?$") @@ -127,69 +215,75 @@ def get_list(d: dict, key: str) -> list: return v if isinstance(v, list) else [v] +# --- parser (the loaders' step loop) -------------------------------------- + +_CLOSED, _EOF = 0, 1 + + class _Parser: def __init__(self, text: str, typed: bool, keep_case: bool, strict: bool, warnings: list | None): - self.toks = list(_tokenize(text)) - self.i = 0 + self.script = Script(text) self.typed = typed self.keep_case = keep_case self.strict = strict self.warnings = warnings if warnings is not None else [] - def _warn(self, msg: str) -> None: + def _warn(self, line: int, msg: str) -> None: + msg = f"line {line}: {msg}" if self.strict: raise MarsSyntaxError(msg) self.warnings.append(msg) - def _peek(self): - return self.toks[self.i] if self.i < len(self.toks) else None - - def _next(self): - t = self.toks[self.i] - self.i += 1 - return t - def _key(self, name: str) -> str: return name if self.keep_case else name.lower() - def body(self, depth: int) -> dict: + def body(self, depth: int): d: dict = {} + sc = self.script while True: - t = self._peek() - if t is None: - if depth: - # 11 shipped shipsections never close their outer block; - # the engine treats EOF as closing every open block. - self._warn(f"end of file inside block (depth {depth})") - return d - kind, val, line = t - if kind == "close": - self._next() - if not depth: - # CrPropaganda.shipsection has one '}' too many. - self._warn(f"line {line}: stray '}}' at top level") + rc, t = sc.next() + if t.key_unterminated: + self._warn(t.key_line, "unterminated quote runs to end of input") + if t.value_unterminated: + self._warn(t.value_line, "unterminated quote runs to end of input") + if rc != OK: + if t.key_status == AT_END and t.key == "}": + if depth == 0: + self._warn(t.key_line, "stray '}' at top level") + return d, _EOF + return d, _CLOSED + if t.key_status == AT_END: + self._warn(t.key_line, f"trailing key '{t.key}' touches end of input: dropped") + elif t.key_status == OK: + if t.value_status == NO_INPUT: + self._warn(t.key_line, f"trailing key '{t.key}' has no value: dropped") + elif t.type == ScriptToken.OPEN: + self._warn(t.key_line, f"block header '{t.key}' {{ touches end of input: dropped") + else: + self._warn(t.key_line, f"final pair '{t.key}' '{t.value}' touches end of input " + "(no trailing newline): dropped") + return d, _EOF + if t.type == ScriptToken.CLOSE: + if depth == 0: + self._warn(t.key_line, "stray '}' at top level") continue - return d - if kind == "open": - raise MarsSyntaxError(f"line {line}: '{{' without a block name") - self._next() - if kind == "quoted": - # bare string item (systemnames.txt lists) -- never a key - _add(d, "_items", val) + return d, _CLOSED + if t.type == ScriptToken.OPEN: + if t.key == "{": + self._warn(t.key_line, "'{' used as a block name") + sub, how = self.body(depth + 1) + _add(d, self._key(t.key), sub) + if how == _EOF: + self._warn(sc.line, f"end of input inside block '{t.key}' (depth {depth + 1})") + return d, _EOF continue - nxt = self._peek() - if nxt is None or nxt[0] == "close": - # lone bareword at end of block: treat as flag item - _add(d, "_items", val) - continue - if nxt[0] == "open": - self._next() - _add(d, self._key(val), self.body(depth + 1)) - continue - nkind, nval, _ = self._next() - if nkind == "bare" and self.typed: - nval = coerce(nval) - _add(d, self._key(val), nval) + # PAIR + if t.key == "{": + self._warn(t.key_line, "'{' used as a key") + val: Any = t.value + if not t.value_quoted and self.typed: + val = coerce(val) + _add(d, self._key(t.key), val) def parse(text: str, *, typed: bool = True, keep_case: bool = False, @@ -198,11 +292,10 @@ def parse(text: str, *, typed: bool = True, keep_case: bool = False, typed -- convert bareword numbers/bools (quoted strings stay str) keep_case -- keep key case instead of lower-casing - strict -- raise on unbalanced braces instead of recovering the way - the engine does (EOF closes open blocks, stray top-level - '}' ignored); pass warnings=[] to collect the recoveries + strict -- raise on anything the engine merely tolerates (see module + doc); pass warnings=[] to collect those cases instead """ - return _Parser(text, typed, keep_case, strict, warnings).body(0) + return _Parser(text, typed, keep_case, strict, warnings).body(0)[0] def parse_file(path, **kw) -> dict: diff --git a/verify/parsers/verify.py b/verify/parsers/verify.py index cb4b6d3..1f6d618 100644 --- a/verify/parsers/verify.py +++ b/verify/parsers/verify.py @@ -139,10 +139,10 @@ def main(root: str, out: str) -> int: obj = flat_kv.parse_rows_file(p) elif k == "kv": txt = manifest.read_text(p) - obj = flat_kv.parse_kv(txt) - d = flat_kv.duplicates(txt) - if d: - warns.append((rel, [f"duplicate keys {d}"])) + w = [] + obj = flat_kv.parse_kv(txt, warnings=w) # engine rules: first occurrence wins + if w: + warns.append((rel, w)) else: continue parsed[rel] = obj @@ -170,7 +170,7 @@ def main(root: str, out: str) -> int: P(f"- `{rel}`: {e}") if warns: P("") - P("Lenient recoveries (engine-compatible; strict=True would reject these):") + P("Engine tolerances taken (what the original loader does with these files; strict=True would reject):") for rel, w in warns: P(f"- `{rel}`: {'; '.join(w)}") P("")