131 lines
5.6 KiB
Python
131 lines
5.6 KiB
Python
"""Publishing must not turn missing/stale/failed evidence into current success."""
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
import sys
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "verify/tooling"))
|
|
from fixtures import passed_gate
|
|
spec = importlib.util.spec_from_file_location("dashboard", ROOT / "tools/dashboard.py")
|
|
d = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(d)
|
|
|
|
|
|
class PublishingTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
(self.root / "campaign").mkdir()
|
|
self.current = {"schema": "sots-current/1", "source": None, "gate": None,
|
|
"replay": None, "next_action": "Establish baseline."}
|
|
self.gate = passed_gate("/binary", "d" * 64)
|
|
|
|
def put(self, name, value):
|
|
data = json.dumps(value).encode()
|
|
(self.root / name).write_bytes(data)
|
|
return {"path": name, "sha256": d.digest(data)}
|
|
|
|
def select_gate(self):
|
|
self.current["gate"] = self.put("gate.json", self.gate)
|
|
self.current["source"] = copy.deepcopy(self.gate["source"])
|
|
|
|
def render(self):
|
|
self.put("campaign/current.json", self.current)
|
|
return d.project(self.root)
|
|
|
|
def test_missing_baseline_ignores_historical_success(self):
|
|
self.put("gate.json", self.gate)
|
|
docs, errors = self.render()
|
|
self.assertFalse(errors)
|
|
self.assertIn("Current baseline: not established", docs["board.md"])
|
|
self.assertNotIn("Gate: **passed", docs["board.md"])
|
|
|
|
def test_selected_dirty_snapshot_and_skips_visible(self):
|
|
self.select_gate()
|
|
docs, errors = self.render()
|
|
self.assertFalse(errors)
|
|
self.assertIn("dirty snapshot", docs["board.md"])
|
|
self.assertIn('Tests skipped: ["assets"]', docs["board.md"])
|
|
|
|
def test_bad_hash_missing_file_and_schema(self):
|
|
for mutation in ("hash", "missing", "schema"):
|
|
with self.subTest(mutation=mutation):
|
|
self.select_gate()
|
|
if mutation == "hash":
|
|
self.current["gate"]["sha256"] = "0" * 64
|
|
elif mutation == "missing":
|
|
self.current["gate"]["path"] = "absent.json"
|
|
else:
|
|
bad = dict(self.gate, schema="old/1")
|
|
self.current["gate"] = self.put("gate.json", bad)
|
|
docs, errors = self.render()
|
|
self.assertTrue(errors)
|
|
self.assertIn("INVALID / STALE", docs["board.md"])
|
|
|
|
def test_stale_source(self):
|
|
self.select_gate()
|
|
self.current["source"]["engine"]["files"][0]["sha256"] = "e" * 64
|
|
self.assertIn("source mismatch", self.render()[1][0])
|
|
|
|
def test_failed_zero_and_missing_execution(self):
|
|
for kind in ("failed", "zero", "required-skipped", "duplicate"):
|
|
with self.subTest(kind=kind):
|
|
gate = copy.deepcopy(self.gate)
|
|
if kind == "failed":
|
|
gate["status"] = "failed"
|
|
elif kind == "duplicate":
|
|
gate["tests"]["passed"] *= 2
|
|
else:
|
|
gate["tests"]["passed"] = [] if kind == "zero" else ["unrelated"]
|
|
self.current["gate"] = self.put("gate.json", gate)
|
|
self.current["source"] = gate["source"]
|
|
self.assertTrue(self.render()[1])
|
|
|
|
def test_replay_measurement_acceptance_and_provenance(self):
|
|
self.select_gate()
|
|
replay = {"schema": "sots-standalone/2", "status": "measured",
|
|
"postRunIntegrity": True,
|
|
"binary": self.gate["binary"],
|
|
"provenance": dict(self.current["gate"], gate=self.gate),
|
|
"pairs": [{"input": "a", "oracle": "b", "exit": 0,
|
|
"inputState": {"stateCoverage": True}, "oracleState": {"stateCoverage": True},
|
|
"outputState": {"stateCoverage": True},
|
|
"fileByteMatch": False, "inflatedByteMatch": True, "stateDigestMatch": True}]}
|
|
self.current["replay"] = self.put("replay.json", replay)
|
|
self.assertFalse(self.render()[1])
|
|
for kind in ("accepted-mismatch", "zero-pairs", "failed-pair", "stale-provenance", "bad-binary"):
|
|
with self.subTest(kind=kind):
|
|
bad = copy.deepcopy(replay)
|
|
if kind == "accepted-mismatch":
|
|
bad["status"] = "accepted"
|
|
elif kind == "zero-pairs":
|
|
bad["pairs"] = []
|
|
elif kind == "failed-pair":
|
|
bad["pairs"][0]["exit"] = 1
|
|
elif kind == "stale-provenance":
|
|
bad["provenance"]["sha256"] = "f" * 64
|
|
else:
|
|
bad["binary"]["sha256"] = "f" * 64
|
|
self.current["replay"] = self.put("replay.json", bad)
|
|
self.assertTrue(self.render()[1])
|
|
|
|
def test_error_overwrites_previously_successful_projection(self):
|
|
self.select_gate()
|
|
self.render()
|
|
self.assertEqual(d.main(["--root", str(self.root)]), 0)
|
|
(self.root / "gate.json").write_text("{}")
|
|
self.assertEqual(d.main(["--root", str(self.root)]), 1)
|
|
self.assertIn("INVALID / STALE", (self.root / "campaign/board.md").read_text())
|
|
|
|
def test_malformed_current(self):
|
|
(self.root / "campaign/current.json").write_text("[]")
|
|
self.assertTrue(d.project(self.root)[1])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|