loop recovery: reset only to an archive the running build can restore

fly-reset-to-milestone does not check compatibility and a flysim that
refuses every checkpoint does not start. fly-loop-reset --check applies
05-deploy's rule (identical, or an adapter-only difference named in
FLY_ACCEPT_ADAPTERS); the ladder picks the highest restorable rung and
falls back to a restart when there is none.

infra/bin/fly-loop-recover

@@ -148,12 +148,21 @@ def plan(level, rank, best, resets_left):
     if level == 0 or not resets_left or rank is None:
         return "restart", None
     ceiling = rank if level == 1 else min(rank, max(best, rank) - 1)
-    below = [rung for rung in rungs() if rung <= ceiling]
-    if below:
-        return "reset", below[-1]
+    for rung in reversed([rung for rung in rungs() if rung <= ceiling]):
+        if restorable(rung):
+            return "reset", rung
     return "restart", None
 
 
+def restorable(rung):
+    """Whether the running build can restore this archive (fly-loop-reset --check)."""
+    try:
+        return subprocess.run(["sudo", "-n", RESET_BIN, "--check", str(rung)], check=False, timeout=60,
+                              stdout=subprocess.DEVNULL).returncode == 0
+    except (OSError, subprocess.SubprocessError):
+        return False
+
+
 def status():
     try:
         with urlopen(STATUS_URL, timeout=5) as response:

infra/bin/fly-loop-reset

@@ -8,25 +8,79 @@
 # again whether or not the reset worked, so the stream never stays down on a failure;
 # fly-reset-to-milestone copies both stores aside before it rewrites anything.
 #
-# Usage: fly-loop-reset <rung>
+# fly-reset-to-milestone does not check that the running build can restore the archive, and a
+# flysim that refuses every checkpoint refuses to start: a black stream. So an archive is only
+# used when its compatibility string equals this build's, or differs only in an adapter id that
+# FLY_ACCEPT_ADAPTERS in /etc/fly/fly.env names (05-deploy.sh's adapter_migration_accepted).
+#
+# Usage: fly-loop-reset [--check] <rung>    (--check: exit 0 if restorable, 3 if not; touch nothing)
 set -euo pipefail
 
 : "${FLY_STATE_DIR:=/srv/fly/state}"
 : "${FLY_SERVICE:=flysim.service}"
 : "${FLY_RESET_BIN:=/opt/fly/bin/fly-reset-to-milestone}"
+: "${FLY_ENV_FILE:=/etc/fly/fly.env}"
+: "${FLY_BIN:=/opt/fly/current/flysim}"
 
 log() { echo "fly-loop-reset: $*" >&2; }
 
+CHECK=0
+if [ "${1:-}" = "--check" ]; then
+    CHECK=1
+    shift
+fi
 RANK="${1:-}"
 if [ "$#" -ne 1 ] || ! [[ "$RANK" =~ ^[0-9]{1,2}$ ]]; then
-    log "usage: fly-loop-reset <rung>"
+    log "usage: fly-loop-reset [--check] <rung>"
     exit 2
 fi
-if [ ! -f "${FLY_STATE_DIR}/milestone-${RANK}.checkpoint" ]; then
+ARCHIVE="${FLY_STATE_DIR}/milestone-${RANK}.checkpoint"
+if [ ! -f "$ARCHIVE" ]; then
     log "no milestone archive for rung ${RANK}; nothing touched"
     exit 1
 fi
 
+# The same rule as 05-deploy.sh: exactly one '/'-separated field differs, it is the adapter
+# (index 1), and the archive's adapter id is listed.
+migration_accepted() {
+    local old="$1" new="$2" accepted="$3" i differing=0 index=-1 entry
+    local -a old_parts new_parts
+    IFS='/' read -r -a old_parts <<< "$old"
+    IFS='/' read -r -a new_parts <<< "$new"
+    [ "${#old_parts[@]}" -eq "${#new_parts[@]}" ] || return 1
+    for ((i = 0; i < ${#old_parts[@]}; i++)); do
+        if [ "${old_parts[$i]}" != "${new_parts[$i]}" ]; then
+            differing=$((differing + 1))
+            index=$i
+        fi
+    done
+    [ "$differing" -eq 1 ] && [ "$index" -eq 1 ] || return 1
+    for entry in ${accepted//,/ }; do
+        [ "$entry" = "${old_parts[1]}" ] && return 0
+    done
+    return 1
+}
+
+accepted=""
+if [ -r "$FLY_ENV_FILE" ]; then
+    set -a
+    # shellcheck disable=SC1090
+    . "$FLY_ENV_FILE"
+    set +a
+    accepted="${FLY_ACCEPT_ADAPTERS:-}"
+fi
+archive_compat="$(head -c 262144 "$ARCHIVE" 2>/dev/null | grep -a -o -m1 '"compatibility":"[^"]*"' | head -n1 | cut -d'"' -f4 || true)"
+build_compat="$("$FLY_BIN" --print-compatibility 2>/dev/null | tail -n1 || true)"
+if [ -z "$archive_compat" ] || [ -z "$build_compat" ]; then
+    log "cannot read the compatibility of rung ${RANK}'s archive or of this build; nothing touched"
+    exit 3
+fi
+if [ "$archive_compat" != "$build_compat" ] && ! migration_accepted "$archive_compat" "$build_compat" "$accepted"; then
+    log "rung ${RANK}'s archive ($(echo "$archive_compat" | cut -d/ -f2)) is not restorable by this build ($(echo "$build_compat" | cut -d/ -f2)), FLY_ACCEPT_ADAPTERS='${accepted}'; nothing touched"
+    exit 3
+fi
+[ "$CHECK" -eq 0 ] || exit 0
+
 log "stopping ${FLY_SERVICE} to reset to rung ${RANK}"
 systemctl stop "$FLY_SERVICE"
 status=0

infra/docs/loop-recovery.md

@@ -19,6 +19,14 @@ a ladder one step per trap that outlives the previous step:
 - **Starting over:** the ladder returns to level 0 when the fly reaches a new best rung, or after
   six hours with no suspected report.
 
+A milestone step only uses an archive the running build can restore: `fly-loop-reset --check`
+compares the archive's compatibility string with `flysim --print-compatibility` and accepts an
+adapter-only difference that `FLY_ACCEPT_ADAPTERS` in `/etc/fly/fly.env` names (the rule
+`05-deploy.sh` applies). A flysim that refuses every checkpoint refuses to start, so an archive
+from an older adapter is skipped for the next one down unless the deploy named its adapter; with
+none restorable the step is a restart. Keep `FLY_ACCEPT_ADAPTERS` in the release env file so a
+deploy does not drop it.
+
 A milestone step runs `/opt/fly/bin/fly-loop-reset <rung>` through sudo (the one line in
 `config/fly-sudoers`): stop flysim, `fly-reset-to-milestone`, start flysim. flysim is started again
 even when the reset fails. The reset copies both stores to `/srv/fly/state.reset-<UTC>` first, as

infra/tests/test_loop_recover.py

@@ -40,6 +40,10 @@ class RecoveryTests(unittest.TestCase):
         for key in ("FLY_LOOP_ROUTER_URL", "FLY_LOOP_MODELS", "FLY_LOOP_MODEL", "FLY_LOOP_ROUTER_KEY"):
             recover.os.environ.pop(key, None)
         self.acts = []
+        self.unrestorable = set()
+        restorable = patch.object(recover, "restorable", side_effect=lambda rung: rung not in self.unrestorable)
+        restorable.start()
+        self.addCleanup(restorable.stop)
         self.act = patch.object(recover, "act", side_effect=lambda action, target: self.acts.append((action, target)) or True)
         self.act.start()
         self.addCleanup(self.act.stop)
@@ -89,6 +93,18 @@ class RecoveryTests(unittest.TestCase):
         self.confirm(T0, rank=11)
         self.assertEqual(self.acts, [("reset", 11)])
 
+    def test_an_archive_this_build_cannot_restore_is_skipped(self):
+        self.unrestorable = {11}
+        recover.write_json(recover.STATE, {"level": 2, "bestRank": 12, "resets": [], "actedAt": 0})
+        self.confirm(T0)
+        self.assertEqual(self.acts, [("reset", 10)])
+
+    def test_no_restorable_archive_means_a_restart(self):
+        self.unrestorable = {1, 9, 10, 11, 12}
+        recover.write_json(recover.STATE, {"level": 1, "bestRank": 12, "resets": [], "actedAt": 0})
+        self.confirm(T0)
+        self.assertEqual(self.acts, [("restart", None)])
+
     def test_new_best_rung_starts_the_ladder_over(self):
         recover.write_json(recover.STATE, {"level": 2, "bestRank": 12, "resets": [], "actedAt": 0})
         self.confirm(T0, rank=13)