324 lines
17 KiB
Python
324 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Build and test a fresh local engine snapshot, recording reproducible evidence."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
INVENTORY = ROOT / "verify/tooling/host-tests.json"
|
|
BUILD_NAMES = {"build", "build-host", "build-shim"}
|
|
RE_TOOL_INPUTS = (
|
|
"tools/gate.py", "tools/gate.sh", "tools/evidence.py", "tools/standalone_report.py",
|
|
"verify/tooling/host-tests.json", "verify/state-checksum/state_checksum.py",
|
|
"verify/save-reader/save_reader.py",
|
|
"verify/harness/compare/mkfixture.py", "verify/harness/compare/oracle_parsers.py",
|
|
"verify/harness/compare/tracecmp.py",
|
|
)
|
|
CORPUS_TESTS = ("mars_stream_save", "mars_stream_domains", "app_turn", "app_turn_record")
|
|
REQUIRED_CHECKS = (
|
|
"sourceCopy", "cleanRoom", "shimConfigs", "configure", "build", "inventory",
|
|
"ctest", "junitComplete", "outputComplete", "testStatuses", "skipClassification", "corpusExecution",
|
|
"corpusNonzero", "sourceUnchanged", "toolSourceUnchanged", "corpusUnchanged", "inputsUnchanged",
|
|
"binaryExists",
|
|
)
|
|
|
|
|
|
def sha256(path):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for block in iter(lambda: f.read(1024 * 1024), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def file_row(root, rel):
|
|
path = root / rel
|
|
info = path.lstat()
|
|
if stat.S_ISLNK(info.st_mode):
|
|
raise ValueError("source/input symlinks are not accepted: " + str(path))
|
|
if not stat.S_ISREG(info.st_mode):
|
|
raise ValueError("source/input is not a regular file: " + str(path))
|
|
return {"path": str(rel), "sha256": sha256(path), "mode": stat.S_IMODE(info.st_mode)}
|
|
|
|
|
|
def files_for_git_tree(root):
|
|
"""Tracked plus nonignored untracked files, excluding exact top-level build outputs."""
|
|
result = subprocess.run(
|
|
["git", "-C", str(root), "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
|
|
capture_output=True, check=True)
|
|
paths = []
|
|
for raw in result.stdout.split(b"\0"):
|
|
if not raw:
|
|
continue
|
|
rel = Path(os.fsdecode(raw))
|
|
if rel.parts and rel.parts[0] in BUILD_NAMES:
|
|
continue
|
|
path = root / rel
|
|
if path.is_symlink():
|
|
raise ValueError("source symlinks are not accepted: " + str(path))
|
|
if path.is_file():
|
|
paths.append(rel)
|
|
elif path.is_dir():
|
|
raise ValueError("gitlink/submodule source requires an explicit snapshot: " + str(path))
|
|
return sorted(paths)
|
|
|
|
|
|
def source_manifest(root):
|
|
paths = files_for_git_tree(root)
|
|
head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], text=True,
|
|
capture_output=True, check=True).stdout.strip()
|
|
dirty = bool(subprocess.run(["git", "-C", str(root), "status", "--porcelain"], text=True,
|
|
capture_output=True, check=True).stdout.strip())
|
|
return {"path": str(root), "head": head, "dirty": dirty,
|
|
"files": [file_row(root, path) for path in paths]}
|
|
|
|
|
|
def explicit_manifest(root, rels):
|
|
head = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
|
|
dirty = bool(subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], text=True).strip())
|
|
return {"path": str(root), "head": head, "dirty": dirty,
|
|
"scope": "execution tooling only; campaign state and result projections excluded",
|
|
"files": [file_row(root, Path(rel)) for rel in rels]}
|
|
|
|
|
|
def input_manifest(path, top_level_saves=False):
|
|
path = path.resolve()
|
|
if path.is_symlink():
|
|
raise ValueError("input symlinks are not accepted: " + str(path))
|
|
if path.is_file():
|
|
return {"root": str(path), "kind": "file", "files": [file_row(path.parent, path.name)]}
|
|
if not path.is_dir():
|
|
raise ValueError("input must be a file or directory: " + str(path))
|
|
paths = sorted(path.glob("*.sav")) if top_level_saves else sorted(path.rglob("*"))
|
|
files = []
|
|
for child in paths:
|
|
if child.is_symlink():
|
|
raise ValueError("input symlinks are not accepted: " + str(child))
|
|
if child.is_file():
|
|
files.append(file_row(path, child.relative_to(path)))
|
|
return {"root": str(path), "kind": "directory", "files": files}
|
|
|
|
|
|
def copy_snapshot(root, destination, manifest):
|
|
for row in manifest["files"]:
|
|
rel = Path(row["path"])
|
|
# Recheck before copying so the snapshot never claims bytes or mode it did not receive.
|
|
if file_row(root, rel) != row:
|
|
raise ValueError("source changed before snapshot copy: " + str(rel))
|
|
target = destination / rel
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(root / rel, target)
|
|
if file_row(destination, rel) != row:
|
|
raise ValueError("source copy integrity failed: " + str(rel))
|
|
|
|
|
|
def run(command, cwd, logs, env=None):
|
|
try:
|
|
proc = subprocess.run(command, cwd=cwd, env=env, text=True, capture_output=True)
|
|
code, stdout, stderr = proc.returncode, proc.stdout, proc.stderr
|
|
except OSError as exc:
|
|
code, stdout, stderr = None, "", repr(exc)
|
|
logs.append({"command": command, "cwd": str(cwd), "returncode": code,
|
|
"stdout": stdout, "stderr": stderr})
|
|
return code
|
|
|
|
|
|
def ctest_names(build, logs):
|
|
try:
|
|
proc = subprocess.run(["ctest", "--show-only=json-v1"], cwd=build, text=True, capture_output=True)
|
|
logs.append({"command": ["ctest", "--show-only=json-v1"], "cwd": str(build),
|
|
"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr})
|
|
return sorted(t["name"] for t in json.loads(proc.stdout).get("tests", [])) if not proc.returncode else None
|
|
except (OSError, json.JSONDecodeError, KeyError) as exc:
|
|
logs.append({"command": ["ctest", "--show-only=json-v1"], "cwd": str(build),
|
|
"returncode": None, "stdout": "", "stderr": repr(exc)})
|
|
return None
|
|
|
|
|
|
def junit_statuses(path):
|
|
rows = []
|
|
for case in ET.parse(path).getroot().iter("testcase"):
|
|
status = "failed" if case.find("failure") is not None or case.find("error") is not None else (
|
|
"skipped" if case.find("skipped") is not None else "passed")
|
|
output = case.findtext("system-out")
|
|
rows.append({"name": case.attrib.get("name", ""), "status": status, "systemOut": output})
|
|
return rows
|
|
|
|
|
|
def corpus_counts(rows):
|
|
patterns = {
|
|
"mars_stream_save": r"^test_save: ok \((\d+) save\(s\),",
|
|
"mars_stream_domains": r"^test_domains: ok \((\d+) save\(s\),",
|
|
"app_turn": r"^app_test_turn: (\d+) save\(s\) driven,",
|
|
"app_turn_record": r"^app_test_turn_record: (\d+) save\(s\),",
|
|
}
|
|
counts = {}
|
|
for row in rows:
|
|
if row["name"] not in CORPUS_TESTS:
|
|
continue
|
|
text = row["systemOut"]
|
|
values = [int(value) for value in re.findall(patterns[row["name"]], text or "", re.M)]
|
|
counts.setdefault(row["name"], []).extend(values)
|
|
return counts
|
|
|
|
|
|
def tool_versions():
|
|
versions = {}
|
|
for name, command in (("python", [sys.executable, "--version"]), ("git", ["git", "--version"]),
|
|
("cmake", ["cmake", "--version"]), ("ctest", ["ctest", "--version"]),
|
|
("cc", ["cc", "--version"]), ("cxx", ["c++", "--version"])):
|
|
try:
|
|
proc = subprocess.run(command, text=True, capture_output=True)
|
|
versions[name] = {"command": command, "returncode": proc.returncode,
|
|
"stdout": proc.stdout, "stderr": proc.stderr}
|
|
except OSError as exc:
|
|
versions[name] = {"command": command, "returncode": None, "stdout": "", "stderr": repr(exc)}
|
|
return versions
|
|
|
|
|
|
def parse_sots_input(value):
|
|
name, sep, raw_path = value.partition("=")
|
|
if not sep or not re.fullmatch(r"SOTS_[A-Z0-9_]+", name) or not raw_path:
|
|
raise argparse.ArgumentTypeError("--sots-input must be SOTS_NAME=PATH")
|
|
return name, Path(raw_path)
|
|
|
|
|
|
def overlaps(first, second):
|
|
return first == second or first in second.parents or second in first.parents
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(epilog="--shim is a boolean: it builds binkw32.dll with cmake/toolchain-mingw-i686.cmake. "
|
|
"Supply every extra SOTS_* dependency as --sots-input SOTS_NAME=PATH.")
|
|
ap.add_argument("--engine", required=True, type=Path)
|
|
ap.add_argument("--corpus", required=True, type=Path)
|
|
ap.add_argument("--out", required=True, type=Path)
|
|
ap.add_argument("--profile", choices=("host", "full"), default="host")
|
|
ap.add_argument("--data", type=Path)
|
|
ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1)
|
|
ap.add_argument("--shim", action="store_true")
|
|
ap.add_argument("--sots-input", action="append", type=parse_sots_input, default=[], metavar="SOTS_NAME=PATH")
|
|
args = ap.parse_args(argv)
|
|
out, engine, corpus = args.out.resolve(), args.engine.resolve(), args.corpus.resolve()
|
|
if out.exists(): ap.error("--out must name a new directory")
|
|
if args.jobs <= 0: ap.error("--jobs must be positive")
|
|
if not engine.is_dir() or not (engine / ".git").exists(): ap.error("--engine must be a git checkout")
|
|
if not corpus.is_dir() or not list(corpus.glob("*.sav")): ap.error("--corpus must contain a top-level .sav")
|
|
if args.data and not args.data.is_dir(): ap.error("--data must be a directory")
|
|
if args.profile == "full" and (not args.data or not args.shim): ap.error("full requires --data and --shim")
|
|
if any(parent in (engine, ROOT.resolve()) for parent in out.parents): ap.error("--out cannot be inside a source tree")
|
|
supplied = dict(args.sots_input)
|
|
if len(supplied) != len(args.sots_input) or "SOTS_SAVES_DIR" in supplied or (args.data and "SOTS_DATA_DIR" in supplied):
|
|
ap.error("duplicate or reserved SOTS_* input")
|
|
input_roots = [corpus] + ([args.data.resolve()] if args.data else []) + [path.resolve() for path in supplied.values()]
|
|
if any(overlaps(out, root) for root in input_roots):
|
|
ap.error("--out must not overlap an input root")
|
|
|
|
out.mkdir(parents=True)
|
|
logs, checks = [], {}
|
|
manifest = {"schema": "sots-gate/1", "status": "failed", "profile": args.profile,
|
|
"generated": dt.datetime.now(dt.timezone.utc).isoformat(), "source": {}, "inputs": {},
|
|
"tools": tool_versions(), "binary": {}, "shimBinary": {},
|
|
"tests": {"expected": [], "passed": [], "skipped": [], "failed": [], "junit": []},
|
|
"checks": checks, "limitations": []}
|
|
try:
|
|
before, re_before = source_manifest(engine), explicit_manifest(ROOT, RE_TOOL_INPUTS)
|
|
manifest["source"] = {"engine": before, "re": re_before}
|
|
corpus_before = input_manifest(corpus, top_level_saves=True)
|
|
manifest["inputs"]["SOTS_SAVES_DIR"] = corpus_before
|
|
input_before = {"SOTS_SAVES_DIR": corpus_before}
|
|
if args.data:
|
|
input_before["SOTS_DATA_DIR"] = input_manifest(args.data)
|
|
for name, path in supplied.items():
|
|
input_before[name] = input_manifest(path)
|
|
manifest["inputs"].update(input_before)
|
|
snapshot = out / "source"
|
|
copy_snapshot(engine, snapshot, before)
|
|
copy_snapshot(ROOT, out / "re-tooling", re_before)
|
|
checks["sourceCopy"] = True
|
|
env = {key: value for key, value in os.environ.items() if not key.startswith("SOTS_")}
|
|
env.update({name: row["root"] for name, row in input_before.items()})
|
|
manifest["environment"] = {name: env[name] for name in sorted(input_before)}
|
|
manifest["buildSettings"] = {"type": "Release", "jobs": args.jobs,
|
|
"environment": {name: env[name] for name in
|
|
("CC", "CXX", "CFLAGS", "CXXFLAGS", "LDFLAGS", "CMAKE_GENERATOR") if name in env}}
|
|
checks["cleanRoom"] = run(["bash", "tools/clean_room_check.sh"], snapshot, logs, env) == 0
|
|
checks["shimConfigs"] = run([sys.executable, "tools/check_shim_configs.py"], snapshot, logs, env) == 0
|
|
build = out / "build-host"
|
|
checks["configure"] = run(["cmake", "-S", str(snapshot), "-B", str(build), "-DCMAKE_BUILD_TYPE=Release",
|
|
"-DPython3_EXECUTABLE=" + sys.executable,
|
|
"-DSOTS_TRACECMP_DIR=" + str(out / "re-tooling/verify/harness/compare")], out, logs, env) == 0
|
|
checks["build"] = checks["configure"] is True and run(["cmake", "--build", str(build), "--parallel", str(args.jobs)], out, logs, env) == 0
|
|
inventory = json.loads(INVENTORY.read_text())
|
|
expected = sorted(inventory["tests"])
|
|
manifest["tests"]["expected"] = expected
|
|
discovered = ctest_names(build, logs) if checks["build"] is True else None
|
|
checks["inventory"] = discovered == expected
|
|
junit = out / "ctest.xml"
|
|
checks["ctest"] = checks["build"] is True and run([
|
|
"ctest", "--verbose", "--output-junit", str(junit),
|
|
"--test-output-size-passed", "67108864", "--test-output-size-failed", "67108864",
|
|
"--no-tests=error"], build, logs, env) == 0
|
|
rows = junit_statuses(junit) if junit.exists() else []
|
|
manifest["tests"]["junit"] = rows
|
|
for row in rows:
|
|
if row["status"] in manifest["tests"]:
|
|
manifest["tests"][row["status"]].append(row["name"])
|
|
actual_names = [row["name"] for row in rows]
|
|
checks["junitComplete"] = (len(actual_names) == len(set(actual_names)) and sorted(actual_names) == expected)
|
|
checks["outputComplete"] = bool(rows) and all(row["systemOut"] is not None and
|
|
"This part of the test output was removed" not in row["systemOut"] for row in rows)
|
|
allowed = set(inventory["hostSkip"] if args.profile == "host" else [])
|
|
skips, passed, failed = set(manifest["tests"]["skipped"]), set(manifest["tests"]["passed"]), set(manifest["tests"]["failed"])
|
|
checks["skipClassification"] = skips <= allowed
|
|
manifest["limitations"] = (["host profile allowed skips: " + ", ".join(sorted(skips))] if skips else [])
|
|
checks["testStatuses"] = not failed and passed | skips == set(expected) and not (passed & skips)
|
|
checks["corpusExecution"] = set(CORPUS_TESTS) <= passed
|
|
counts = corpus_counts(rows)
|
|
manifest["tests"]["corpusSummaryCounts"] = counts
|
|
count = len(corpus_before["files"])
|
|
checks["corpusNonzero"] = all(counts.get(name) == [count] and count > 0 for name in CORPUS_TESTS)
|
|
binary = build / "src/app/sots_turn"
|
|
checks["binaryExists"] = binary.is_file()
|
|
if checks["binaryExists"]:
|
|
manifest["binary"] = {"path": str(binary), "sha256": sha256(binary)}
|
|
if args.shim:
|
|
shim_build = out / "build-shim"
|
|
toolchain = snapshot / "cmake/toolchain-mingw-i686.cmake"
|
|
checks["shimBuild"] = run(["cmake", "-S", str(snapshot), "-B", str(shim_build),
|
|
"-DCMAKE_TOOLCHAIN_FILE=" + str(toolchain), "-DCMAKE_BUILD_TYPE=Release"], out, logs, env) == 0
|
|
checks["shimBuild"] = checks["shimBuild"] is True and run(["cmake", "--build", str(shim_build), "--parallel", str(args.jobs)], out, logs, env) == 0
|
|
dlls = list(shim_build.rglob("binkw32.dll"))
|
|
checks["shimBinaryExists"] = checks["shimBuild"] is True and len(dlls) == 1 and dlls[0].is_file()
|
|
if checks["shimBinaryExists"]:
|
|
manifest["shimBinary"] = {"path": str(dlls[0]), "sha256": sha256(dlls[0])}
|
|
after = source_manifest(engine)
|
|
checks["sourceUnchanged"] = before == after
|
|
checks["toolSourceUnchanged"] = re_before == explicit_manifest(ROOT, RE_TOOL_INPUTS)
|
|
checks["corpusUnchanged"] = corpus_before == input_manifest(corpus, top_level_saves=True)
|
|
for name, old in input_before.items():
|
|
checks[name + "Unchanged"] = old == input_manifest(Path(old["root"]), name == "SOTS_SAVES_DIR")
|
|
checks["inputsUnchanged"] = all(checks[name + "Unchanged"] is True for name in input_before)
|
|
required = list(REQUIRED_CHECKS) + (["shimBuild", "shimBinaryExists"] if args.shim else [])
|
|
manifest["requiredChecks"] = required
|
|
manifest["status"] = "passed" if all(checks.get(name) is True for name in required) else "failed"
|
|
except Exception as exc:
|
|
checks["exception"] = repr(exc)
|
|
manifest["commands"] = logs
|
|
(out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
|
return 0 if manifest["status"] == "passed" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|