9c9cec49a9
fly-reset-to-milestone does not check compatibility and a flysim that refuses every checkpoint does not start. fly-loop-reset --check applies 05-deploy's rule (identical, or an adapter-only difference named in FLY_ACCEPT_ADAPTERS); the ladder picks the highest restorable rung and falls back to a restart when there is none.
90 lines
3.7 KiB
Bash
Executable file
90 lines
3.7 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# infra/bin/fly-loop-reset — the milestone step of fly-loop-recover's ladder, as root.
|
|
#
|
|
# fly-loop-recover runs as User=fly and reaches this through one NOPASSWD sudoers line
|
|
# (config/fly-sudoers). It does what infra/docs/runbook.md "Restart the run from a rung"
|
|
# does by hand for a release whose adapter already wrote the archive: stop flysim, promote
|
|
# milestone-<N>.checkpoint with fly-reset-to-milestone, start flysim. flysim is started
|
|
# again whether or not the reset worked, so the stream never stays down on a failure;
|
|
# fly-reset-to-milestone copies both stores aside before it rewrites anything.
|
|
#
|
|
# fly-reset-to-milestone does not check that the running build can restore the archive, and a
|
|
# flysim that refuses every checkpoint refuses to start: a black stream. So an archive is only
|
|
# used when its compatibility string equals this build's, or differs only in an adapter id that
|
|
# FLY_ACCEPT_ADAPTERS in /etc/fly/fly.env names (05-deploy.sh's adapter_migration_accepted).
|
|
#
|
|
# Usage: fly-loop-reset [--check] <rung> (--check: exit 0 if restorable, 3 if not; touch nothing)
|
|
set -euo pipefail
|
|
|
|
: "${FLY_STATE_DIR:=/srv/fly/state}"
|
|
: "${FLY_SERVICE:=flysim.service}"
|
|
: "${FLY_RESET_BIN:=/opt/fly/bin/fly-reset-to-milestone}"
|
|
: "${FLY_ENV_FILE:=/etc/fly/fly.env}"
|
|
: "${FLY_BIN:=/opt/fly/current/flysim}"
|
|
|
|
log() { echo "fly-loop-reset: $*" >&2; }
|
|
|
|
CHECK=0
|
|
if [ "${1:-}" = "--check" ]; then
|
|
CHECK=1
|
|
shift
|
|
fi
|
|
RANK="${1:-}"
|
|
if [ "$#" -ne 1 ] || ! [[ "$RANK" =~ ^[0-9]{1,2}$ ]]; then
|
|
log "usage: fly-loop-reset [--check] <rung>"
|
|
exit 2
|
|
fi
|
|
ARCHIVE="${FLY_STATE_DIR}/milestone-${RANK}.checkpoint"
|
|
if [ ! -f "$ARCHIVE" ]; then
|
|
log "no milestone archive for rung ${RANK}; nothing touched"
|
|
exit 1
|
|
fi
|
|
|
|
# The same rule as 05-deploy.sh: exactly one '/'-separated field differs, it is the adapter
|
|
# (index 1), and the archive's adapter id is listed.
|
|
migration_accepted() {
|
|
local old="$1" new="$2" accepted="$3" i differing=0 index=-1 entry
|
|
local -a old_parts new_parts
|
|
IFS='/' read -r -a old_parts <<< "$old"
|
|
IFS='/' read -r -a new_parts <<< "$new"
|
|
[ "${#old_parts[@]}" -eq "${#new_parts[@]}" ] || return 1
|
|
for ((i = 0; i < ${#old_parts[@]}; i++)); do
|
|
if [ "${old_parts[$i]}" != "${new_parts[$i]}" ]; then
|
|
differing=$((differing + 1))
|
|
index=$i
|
|
fi
|
|
done
|
|
[ "$differing" -eq 1 ] && [ "$index" -eq 1 ] || return 1
|
|
for entry in ${accepted//,/ }; do
|
|
[ "$entry" = "${old_parts[1]}" ] && return 0
|
|
done
|
|
return 1
|
|
}
|
|
|
|
accepted=""
|
|
if [ -r "$FLY_ENV_FILE" ]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
. "$FLY_ENV_FILE"
|
|
set +a
|
|
accepted="${FLY_ACCEPT_ADAPTERS:-}"
|
|
fi
|
|
archive_compat="$(head -c 262144 "$ARCHIVE" 2>/dev/null | grep -a -o -m1 '"compatibility":"[^"]*"' | head -n1 | cut -d'"' -f4 || true)"
|
|
build_compat="$("$FLY_BIN" --print-compatibility 2>/dev/null | tail -n1 || true)"
|
|
if [ -z "$archive_compat" ] || [ -z "$build_compat" ]; then
|
|
log "cannot read the compatibility of rung ${RANK}'s archive or of this build; nothing touched"
|
|
exit 3
|
|
fi
|
|
if [ "$archive_compat" != "$build_compat" ] && ! migration_accepted "$archive_compat" "$build_compat" "$accepted"; then
|
|
log "rung ${RANK}'s archive ($(echo "$archive_compat" | cut -d/ -f2)) is not restorable by this build ($(echo "$build_compat" | cut -d/ -f2)), FLY_ACCEPT_ADAPTERS='${accepted}'; nothing touched"
|
|
exit 3
|
|
fi
|
|
[ "$CHECK" -eq 0 ] || exit 0
|
|
|
|
log "stopping ${FLY_SERVICE} to reset to rung ${RANK}"
|
|
systemctl stop "$FLY_SERVICE"
|
|
status=0
|
|
"$FLY_RESET_BIN" "$RANK" || status=$?
|
|
[ "$status" -eq 0 ] || log "fly-reset-to-milestone ${RANK} failed (exit ${status}); starting ${FLY_SERVICE} on the state it left"
|
|
systemctl start "$FLY_SERVICE"
|
|
exit "$status"
|