From 9c9cec49a92b3808da4df1ae7a3fdcf9071cc4fa Mon Sep 17 00:00:00 2001 From: acamilo Date: Mon, 28 Sep 2026 21:25:58 +0000 Subject: [PATCH] loop recovery: reset only to an archive the running build can restore fly-reset-to-milestone does not check compatibility and a flysim that refuses every checkpoint does not start. fly-loop-reset --check applies 05-deploy's rule (identical, or an adapter-only difference named in FLY_ACCEPT_ADAPTERS); the ladder picks the highest restorable rung and falls back to a restart when there is none. --- infra/bin/fly-loop-recover | 15 ++++++-- infra/bin/fly-loop-reset | 60 ++++++++++++++++++++++++++++++-- infra/docs/loop-recovery.md | 8 +++++ infra/tests/test_loop_recover.py | 16 +++++++++ 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/infra/bin/fly-loop-recover b/infra/bin/fly-loop-recover index 3ea369f..1fdc355 100755 --- a/infra/bin/fly-loop-recover +++ b/infra/bin/fly-loop-recover @@ -148,12 +148,21 @@ def plan(level, rank, best, resets_left): if level == 0 or not resets_left or rank is None: return "restart", None ceiling = rank if level == 1 else min(rank, max(best, rank) - 1) - below = [rung for rung in rungs() if rung <= ceiling] - if below: - return "reset", below[-1] + for rung in reversed([rung for rung in rungs() if rung <= ceiling]): + if restorable(rung): + return "reset", rung return "restart", None +def restorable(rung): + """Whether the running build can restore this archive (fly-loop-reset --check).""" + try: + return subprocess.run(["sudo", "-n", RESET_BIN, "--check", str(rung)], check=False, timeout=60, + stdout=subprocess.DEVNULL).returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + def status(): try: with urlopen(STATUS_URL, timeout=5) as response: diff --git a/infra/bin/fly-loop-reset b/infra/bin/fly-loop-reset index de16570..99a5e4f 100755 --- a/infra/bin/fly-loop-reset +++ b/infra/bin/fly-loop-reset @@ -8,25 +8,79 @@ # again whether or not the reset worked, so the stream never stays down on a failure; # fly-reset-to-milestone copies both stores aside before it rewrites anything. # -# Usage: fly-loop-reset +# fly-reset-to-milestone does not check that the running build can restore the archive, and a +# flysim that refuses every checkpoint refuses to start: a black stream. So an archive is only +# used when its compatibility string equals this build's, or differs only in an adapter id that +# FLY_ACCEPT_ADAPTERS in /etc/fly/fly.env names (05-deploy.sh's adapter_migration_accepted). +# +# Usage: fly-loop-reset [--check] (--check: exit 0 if restorable, 3 if not; touch nothing) set -euo pipefail : "${FLY_STATE_DIR:=/srv/fly/state}" : "${FLY_SERVICE:=flysim.service}" : "${FLY_RESET_BIN:=/opt/fly/bin/fly-reset-to-milestone}" +: "${FLY_ENV_FILE:=/etc/fly/fly.env}" +: "${FLY_BIN:=/opt/fly/current/flysim}" log() { echo "fly-loop-reset: $*" >&2; } +CHECK=0 +if [ "${1:-}" = "--check" ]; then + CHECK=1 + shift +fi RANK="${1:-}" if [ "$#" -ne 1 ] || ! [[ "$RANK" =~ ^[0-9]{1,2}$ ]]; then - log "usage: fly-loop-reset " + log "usage: fly-loop-reset [--check] " exit 2 fi -if [ ! -f "${FLY_STATE_DIR}/milestone-${RANK}.checkpoint" ]; then +ARCHIVE="${FLY_STATE_DIR}/milestone-${RANK}.checkpoint" +if [ ! -f "$ARCHIVE" ]; then log "no milestone archive for rung ${RANK}; nothing touched" exit 1 fi +# The same rule as 05-deploy.sh: exactly one '/'-separated field differs, it is the adapter +# (index 1), and the archive's adapter id is listed. +migration_accepted() { + local old="$1" new="$2" accepted="$3" i differing=0 index=-1 entry + local -a old_parts new_parts + IFS='/' read -r -a old_parts <<< "$old" + IFS='/' read -r -a new_parts <<< "$new" + [ "${#old_parts[@]}" -eq "${#new_parts[@]}" ] || return 1 + for ((i = 0; i < ${#old_parts[@]}; i++)); do + if [ "${old_parts[$i]}" != "${new_parts[$i]}" ]; then + differing=$((differing + 1)) + index=$i + fi + done + [ "$differing" -eq 1 ] && [ "$index" -eq 1 ] || return 1 + for entry in ${accepted//,/ }; do + [ "$entry" = "${old_parts[1]}" ] && return 0 + done + return 1 +} + +accepted="" +if [ -r "$FLY_ENV_FILE" ]; then + set -a + # shellcheck disable=SC1090 + . "$FLY_ENV_FILE" + set +a + accepted="${FLY_ACCEPT_ADAPTERS:-}" +fi +archive_compat="$(head -c 262144 "$ARCHIVE" 2>/dev/null | grep -a -o -m1 '"compatibility":"[^"]*"' | head -n1 | cut -d'"' -f4 || true)" +build_compat="$("$FLY_BIN" --print-compatibility 2>/dev/null | tail -n1 || true)" +if [ -z "$archive_compat" ] || [ -z "$build_compat" ]; then + log "cannot read the compatibility of rung ${RANK}'s archive or of this build; nothing touched" + exit 3 +fi +if [ "$archive_compat" != "$build_compat" ] && ! migration_accepted "$archive_compat" "$build_compat" "$accepted"; then + log "rung ${RANK}'s archive ($(echo "$archive_compat" | cut -d/ -f2)) is not restorable by this build ($(echo "$build_compat" | cut -d/ -f2)), FLY_ACCEPT_ADAPTERS='${accepted}'; nothing touched" + exit 3 +fi +[ "$CHECK" -eq 0 ] || exit 0 + log "stopping ${FLY_SERVICE} to reset to rung ${RANK}" systemctl stop "$FLY_SERVICE" status=0 diff --git a/infra/docs/loop-recovery.md b/infra/docs/loop-recovery.md index 8f3df98..cff1c24 100644 --- a/infra/docs/loop-recovery.md +++ b/infra/docs/loop-recovery.md @@ -19,6 +19,14 @@ a ladder one step per trap that outlives the previous step: - **Starting over:** the ladder returns to level 0 when the fly reaches a new best rung, or after six hours with no suspected report. +A milestone step only uses an archive the running build can restore: `fly-loop-reset --check` +compares the archive's compatibility string with `flysim --print-compatibility` and accepts an +adapter-only difference that `FLY_ACCEPT_ADAPTERS` in `/etc/fly/fly.env` names (the rule +`05-deploy.sh` applies). A flysim that refuses every checkpoint refuses to start, so an archive +from an older adapter is skipped for the next one down unless the deploy named its adapter; with +none restorable the step is a restart. Keep `FLY_ACCEPT_ADAPTERS` in the release env file so a +deploy does not drop it. + A milestone step runs `/opt/fly/bin/fly-loop-reset ` through sudo (the one line in `config/fly-sudoers`): stop flysim, `fly-reset-to-milestone`, start flysim. flysim is started again even when the reset fails. The reset copies both stores to `/srv/fly/state.reset-` first, as diff --git a/infra/tests/test_loop_recover.py b/infra/tests/test_loop_recover.py index 881971c..264a3a7 100644 --- a/infra/tests/test_loop_recover.py +++ b/infra/tests/test_loop_recover.py @@ -40,6 +40,10 @@ class RecoveryTests(unittest.TestCase): for key in ("FLY_LOOP_ROUTER_URL", "FLY_LOOP_MODELS", "FLY_LOOP_MODEL", "FLY_LOOP_ROUTER_KEY"): recover.os.environ.pop(key, None) self.acts = [] + self.unrestorable = set() + restorable = patch.object(recover, "restorable", side_effect=lambda rung: rung not in self.unrestorable) + restorable.start() + self.addCleanup(restorable.stop) self.act = patch.object(recover, "act", side_effect=lambda action, target: self.acts.append((action, target)) or True) self.act.start() self.addCleanup(self.act.stop) @@ -89,6 +93,18 @@ class RecoveryTests(unittest.TestCase): self.confirm(T0, rank=11) self.assertEqual(self.acts, [("reset", 11)]) + def test_an_archive_this_build_cannot_restore_is_skipped(self): + self.unrestorable = {11} + recover.write_json(recover.STATE, {"level": 2, "bestRank": 12, "resets": [], "actedAt": 0}) + self.confirm(T0) + self.assertEqual(self.acts, [("reset", 10)]) + + def test_no_restorable_archive_means_a_restart(self): + self.unrestorable = {1, 9, 10, 11, 12} + recover.write_json(recover.STATE, {"level": 1, "bestRank": 12, "resets": [], "actedAt": 0}) + self.confirm(T0) + self.assertEqual(self.acts, [("restart", None)]) + def test_new_best_rung_starts_the_ladder_over(self): recover.write_json(recover.STATE, {"level": 2, "bestRank": 12, "resets": [], "actedAt": 0}) self.confirm(T0, rank=13)