#!/usr/bin/env python3
"""Build and test a fresh local engine snapshot, recording reproducible evidence."""
from __future__ import annotations

import argparse
import datetime as dt
import hashlib
import json
import os
import re
import shutil
import stat
import subprocess
import sys
import xml.etree.ElementTree as ET
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
INVENTORY = ROOT / "verify/tooling/host-tests.json"
BUILD_NAMES = {"build", "build-host", "build-shim"}
RE_TOOL_INPUTS = (
    "tools/gate.py", "tools/gate.sh", "tools/evidence.py", "tools/standalone_report.py",
    "verify/tooling/host-tests.json", "verify/state-checksum/state_checksum.py",
    "verify/save-reader/save_reader.py",
    "verify/harness/compare/mkfixture.py", "verify/harness/compare/oracle_parsers.py",
    "verify/harness/compare/tracecmp.py",
)
CORPUS_TESTS = ("mars_stream_save", "mars_stream_domains", "app_turn", "app_turn_record")
REQUIRED_CHECKS = (
    "sourceCopy", "cleanRoom", "shimConfigs", "configure", "build", "inventory",
    "ctest", "junitComplete", "outputComplete", "testStatuses", "skipClassification", "corpusExecution",
    "corpusNonzero", "sourceUnchanged", "toolSourceUnchanged", "corpusUnchanged", "inputsUnchanged",
    "binaryExists",
)


def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for block in iter(lambda: f.read(1024 * 1024), b""):
            h.update(block)
    return h.hexdigest()


def file_row(root, rel):
    path = root / rel
    info = path.lstat()
    if stat.S_ISLNK(info.st_mode):
        raise ValueError("source/input symlinks are not accepted: " + str(path))
    if not stat.S_ISREG(info.st_mode):
        raise ValueError("source/input is not a regular file: " + str(path))
    return {"path": str(rel), "sha256": sha256(path), "mode": stat.S_IMODE(info.st_mode)}


def files_for_git_tree(root):
    """Tracked plus nonignored untracked files, excluding exact top-level build outputs."""
    result = subprocess.run(
        ["git", "-C", str(root), "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
        capture_output=True, check=True)
    paths = []
    for raw in result.stdout.split(b"\0"):
        if not raw:
            continue
        rel = Path(os.fsdecode(raw))
        if rel.parts and rel.parts[0] in BUILD_NAMES:
            continue
        path = root / rel
        if path.is_symlink():
            raise ValueError("source symlinks are not accepted: " + str(path))
        if path.is_file():
            paths.append(rel)
        elif path.is_dir():
            raise ValueError("gitlink/submodule source requires an explicit snapshot: " + str(path))
    return sorted(paths)


def source_manifest(root):
    paths = files_for_git_tree(root)
    head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], text=True,
                          capture_output=True, check=True).stdout.strip()
    dirty = bool(subprocess.run(["git", "-C", str(root), "status", "--porcelain"], text=True,
                                capture_output=True, check=True).stdout.strip())
    return {"path": str(root), "head": head, "dirty": dirty,
            "files": [file_row(root, path) for path in paths]}


def explicit_manifest(root, rels):
    head = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
    dirty = bool(subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], text=True).strip())
    return {"path": str(root), "head": head, "dirty": dirty,
            "scope": "execution tooling only; campaign state and result projections excluded",
            "files": [file_row(root, Path(rel)) for rel in rels]}


def input_manifest(path, top_level_saves=False):
    path = path.resolve()
    if path.is_symlink():
        raise ValueError("input symlinks are not accepted: " + str(path))
    if path.is_file():
        return {"root": str(path), "kind": "file", "files": [file_row(path.parent, path.name)]}
    if not path.is_dir():
        raise ValueError("input must be a file or directory: " + str(path))
    paths = sorted(path.glob("*.sav")) if top_level_saves else sorted(path.rglob("*"))
    files = []
    for child in paths:
        if child.is_symlink():
            raise ValueError("input symlinks are not accepted: " + str(child))
        if child.is_file():
            files.append(file_row(path, child.relative_to(path)))
    return {"root": str(path), "kind": "directory", "files": files}


def copy_snapshot(root, destination, manifest):
    for row in manifest["files"]:
        rel = Path(row["path"])
        # Recheck before copying so the snapshot never claims bytes or mode it did not receive.
        if file_row(root, rel) != row:
            raise ValueError("source changed before snapshot copy: " + str(rel))
        target = destination / rel
        target.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(root / rel, target)
        if file_row(destination, rel) != row:
            raise ValueError("source copy integrity failed: " + str(rel))


def run(command, cwd, logs, env=None):
    try:
        proc = subprocess.run(command, cwd=cwd, env=env, text=True, capture_output=True)
        code, stdout, stderr = proc.returncode, proc.stdout, proc.stderr
    except OSError as exc:
        code, stdout, stderr = None, "", repr(exc)
    logs.append({"command": command, "cwd": str(cwd), "returncode": code,
                 "stdout": stdout, "stderr": stderr})
    return code


def ctest_names(build, logs):
    try:
        proc = subprocess.run(["ctest", "--show-only=json-v1"], cwd=build, text=True, capture_output=True)
        logs.append({"command": ["ctest", "--show-only=json-v1"], "cwd": str(build),
                     "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr})
        return sorted(t["name"] for t in json.loads(proc.stdout).get("tests", [])) if not proc.returncode else None
    except (OSError, json.JSONDecodeError, KeyError) as exc:
        logs.append({"command": ["ctest", "--show-only=json-v1"], "cwd": str(build),
                     "returncode": None, "stdout": "", "stderr": repr(exc)})
        return None


def junit_statuses(path):
    rows = []
    for case in ET.parse(path).getroot().iter("testcase"):
        status = "failed" if case.find("failure") is not None or case.find("error") is not None else (
            "skipped" if case.find("skipped") is not None else "passed")
        output = case.findtext("system-out")
        rows.append({"name": case.attrib.get("name", ""), "status": status, "systemOut": output})
    return rows


def corpus_counts(rows):
    patterns = {
        "mars_stream_save": r"^test_save: ok \((\d+) save\(s\),",
        "mars_stream_domains": r"^test_domains: ok \((\d+) save\(s\),",
        "app_turn": r"^app_test_turn: (\d+) save\(s\) driven,",
        "app_turn_record": r"^app_test_turn_record: (\d+) save\(s\),",
    }
    counts = {}
    for row in rows:
        if row["name"] not in CORPUS_TESTS:
            continue
        text = row["systemOut"]
        values = [int(value) for value in re.findall(patterns[row["name"]], text or "", re.M)]
        counts.setdefault(row["name"], []).extend(values)
    return counts


def tool_versions():
    versions = {}
    for name, command in (("python", [sys.executable, "--version"]), ("git", ["git", "--version"]),
                          ("cmake", ["cmake", "--version"]), ("ctest", ["ctest", "--version"]),
                          ("cc", ["cc", "--version"]), ("cxx", ["c++", "--version"])):
        try:
            proc = subprocess.run(command, text=True, capture_output=True)
            versions[name] = {"command": command, "returncode": proc.returncode,
                              "stdout": proc.stdout, "stderr": proc.stderr}
        except OSError as exc:
            versions[name] = {"command": command, "returncode": None, "stdout": "", "stderr": repr(exc)}
    return versions


def parse_sots_input(value):
    name, sep, raw_path = value.partition("=")
    if not sep or not re.fullmatch(r"SOTS_[A-Z0-9_]+", name) or not raw_path:
        raise argparse.ArgumentTypeError("--sots-input must be SOTS_NAME=PATH")
    return name, Path(raw_path)


def overlaps(first, second):
    return first == second or first in second.parents or second in first.parents


def main(argv=None):
    ap = argparse.ArgumentParser(epilog="--shim is a boolean: it builds binkw32.dll with cmake/toolchain-mingw-i686.cmake. "
                                               "Supply every extra SOTS_* dependency as --sots-input SOTS_NAME=PATH.")
    ap.add_argument("--engine", required=True, type=Path)
    ap.add_argument("--corpus", required=True, type=Path)
    ap.add_argument("--out", required=True, type=Path)
    ap.add_argument("--profile", choices=("host", "full"), default="host")
    ap.add_argument("--data", type=Path)
    ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1)
    ap.add_argument("--shim", action="store_true")
    ap.add_argument("--sots-input", action="append", type=parse_sots_input, default=[], metavar="SOTS_NAME=PATH")
    args = ap.parse_args(argv)
    out, engine, corpus = args.out.resolve(), args.engine.resolve(), args.corpus.resolve()
    if out.exists(): ap.error("--out must name a new directory")
    if args.jobs <= 0: ap.error("--jobs must be positive")
    if not engine.is_dir() or not (engine / ".git").exists(): ap.error("--engine must be a git checkout")
    if not corpus.is_dir() or not list(corpus.glob("*.sav")): ap.error("--corpus must contain a top-level .sav")
    if args.data and not args.data.is_dir(): ap.error("--data must be a directory")
    if args.profile == "full" and (not args.data or not args.shim): ap.error("full requires --data and --shim")
    if any(parent in (engine, ROOT.resolve()) for parent in out.parents): ap.error("--out cannot be inside a source tree")
    supplied = dict(args.sots_input)
    if len(supplied) != len(args.sots_input) or "SOTS_SAVES_DIR" in supplied or (args.data and "SOTS_DATA_DIR" in supplied):
        ap.error("duplicate or reserved SOTS_* input")
    input_roots = [corpus] + ([args.data.resolve()] if args.data else []) + [path.resolve() for path in supplied.values()]
    if any(overlaps(out, root) for root in input_roots):
        ap.error("--out must not overlap an input root")

    out.mkdir(parents=True)
    logs, checks = [], {}
    manifest = {"schema": "sots-gate/1", "status": "failed", "profile": args.profile,
                "generated": dt.datetime.now(dt.timezone.utc).isoformat(), "source": {}, "inputs": {},
                "tools": tool_versions(), "binary": {}, "shimBinary": {},
                "tests": {"expected": [], "passed": [], "skipped": [], "failed": [], "junit": []},
                "checks": checks, "limitations": []}
    try:
        before, re_before = source_manifest(engine), explicit_manifest(ROOT, RE_TOOL_INPUTS)
        manifest["source"] = {"engine": before, "re": re_before}
        corpus_before = input_manifest(corpus, top_level_saves=True)
        manifest["inputs"]["SOTS_SAVES_DIR"] = corpus_before
        input_before = {"SOTS_SAVES_DIR": corpus_before}
        if args.data:
            input_before["SOTS_DATA_DIR"] = input_manifest(args.data)
        for name, path in supplied.items():
            input_before[name] = input_manifest(path)
        manifest["inputs"].update(input_before)
        snapshot = out / "source"
        copy_snapshot(engine, snapshot, before)
        copy_snapshot(ROOT, out / "re-tooling", re_before)
        checks["sourceCopy"] = True
        env = {key: value for key, value in os.environ.items() if not key.startswith("SOTS_")}
        env.update({name: row["root"] for name, row in input_before.items()})
        manifest["environment"] = {name: env[name] for name in sorted(input_before)}
        manifest["buildSettings"] = {"type": "Release", "jobs": args.jobs,
            "environment": {name: env[name] for name in
                ("CC", "CXX", "CFLAGS", "CXXFLAGS", "LDFLAGS", "CMAKE_GENERATOR") if name in env}}
        checks["cleanRoom"] = run(["bash", "tools/clean_room_check.sh"], snapshot, logs, env) == 0
        checks["shimConfigs"] = run([sys.executable, "tools/check_shim_configs.py"], snapshot, logs, env) == 0
        build = out / "build-host"
        checks["configure"] = run(["cmake", "-S", str(snapshot), "-B", str(build), "-DCMAKE_BUILD_TYPE=Release",
                                    "-DPython3_EXECUTABLE=" + sys.executable,
                                    "-DSOTS_TRACECMP_DIR=" + str(out / "re-tooling/verify/harness/compare")], out, logs, env) == 0
        checks["build"] = checks["configure"] is True and run(["cmake", "--build", str(build), "--parallel", str(args.jobs)], out, logs, env) == 0
        inventory = json.loads(INVENTORY.read_text())
        expected = sorted(inventory["tests"])
        manifest["tests"]["expected"] = expected
        discovered = ctest_names(build, logs) if checks["build"] is True else None
        checks["inventory"] = discovered == expected
        junit = out / "ctest.xml"
        checks["ctest"] = checks["build"] is True and run([
            "ctest", "--verbose", "--output-junit", str(junit),
            "--test-output-size-passed", "67108864", "--test-output-size-failed", "67108864",
            "--no-tests=error"], build, logs, env) == 0
        rows = junit_statuses(junit) if junit.exists() else []
        manifest["tests"]["junit"] = rows
        for row in rows:
            if row["status"] in manifest["tests"]:
                manifest["tests"][row["status"]].append(row["name"])
        actual_names = [row["name"] for row in rows]
        checks["junitComplete"] = (len(actual_names) == len(set(actual_names)) and sorted(actual_names) == expected)
        checks["outputComplete"] = bool(rows) and all(row["systemOut"] is not None and
            "This part of the test output was removed" not in row["systemOut"] for row in rows)
        allowed = set(inventory["hostSkip"] if args.profile == "host" else [])
        skips, passed, failed = set(manifest["tests"]["skipped"]), set(manifest["tests"]["passed"]), set(manifest["tests"]["failed"])
        checks["skipClassification"] = skips <= allowed
        manifest["limitations"] = (["host profile allowed skips: " + ", ".join(sorted(skips))] if skips else [])
        checks["testStatuses"] = not failed and passed | skips == set(expected) and not (passed & skips)
        checks["corpusExecution"] = set(CORPUS_TESTS) <= passed
        counts = corpus_counts(rows)
        manifest["tests"]["corpusSummaryCounts"] = counts
        count = len(corpus_before["files"])
        checks["corpusNonzero"] = all(counts.get(name) == [count] and count > 0 for name in CORPUS_TESTS)
        binary = build / "src/app/sots_turn"
        checks["binaryExists"] = binary.is_file()
        if checks["binaryExists"]:
            manifest["binary"] = {"path": str(binary), "sha256": sha256(binary)}
        if args.shim:
            shim_build = out / "build-shim"
            toolchain = snapshot / "cmake/toolchain-mingw-i686.cmake"
            checks["shimBuild"] = run(["cmake", "-S", str(snapshot), "-B", str(shim_build),
                                        "-DCMAKE_TOOLCHAIN_FILE=" + str(toolchain), "-DCMAKE_BUILD_TYPE=Release"], out, logs, env) == 0
            checks["shimBuild"] = checks["shimBuild"] is True and run(["cmake", "--build", str(shim_build), "--parallel", str(args.jobs)], out, logs, env) == 0
            dlls = list(shim_build.rglob("binkw32.dll"))
            checks["shimBinaryExists"] = checks["shimBuild"] is True and len(dlls) == 1 and dlls[0].is_file()
            if checks["shimBinaryExists"]:
                manifest["shimBinary"] = {"path": str(dlls[0]), "sha256": sha256(dlls[0])}
        after = source_manifest(engine)
        checks["sourceUnchanged"] = before == after
        checks["toolSourceUnchanged"] = re_before == explicit_manifest(ROOT, RE_TOOL_INPUTS)
        checks["corpusUnchanged"] = corpus_before == input_manifest(corpus, top_level_saves=True)
        for name, old in input_before.items():
            checks[name + "Unchanged"] = old == input_manifest(Path(old["root"]), name == "SOTS_SAVES_DIR")
        checks["inputsUnchanged"] = all(checks[name + "Unchanged"] is True for name in input_before)
        required = list(REQUIRED_CHECKS) + (["shimBuild", "shimBinaryExists"] if args.shim else [])
        manifest["requiredChecks"] = required
        manifest["status"] = "passed" if all(checks.get(name) is True for name in required) else "failed"
    except Exception as exc:
        checks["exception"] = repr(exc)
    manifest["commands"] = logs
    (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    return 0 if manifest["status"] == "passed" else 1


if __name__ == "__main__":
    sys.exit(main())