From 3d9a08d0bede8f1b0306fa285a8bdce8e8b1dda3 Mon Sep 17 00:00:00 2001 From: acamilo Date: Wed, 23 Sep 2026 08:19:51 +0000 Subject: [PATCH] infra: flyedge.service, off by default, and the feed bus through build, deploy and watchdog flyedge.service runs /opt/fly/current/fly-edge After= and Requires= flysim.service, with its metrics on loopback :9102 and a ConditionPathExists so a release without the binary leaves it inactive. It is in no target and 07-enable.sh does not enable it; the header has the switch and the way back. build-flysim.sh also builds fly-edge beside the flysim binary and package-release.sh ships it when present. 05-deploy.sh writes FLY_FEED_VIA (default direct) into fly.env, flysim.service names FLY_BUS_DIR=/run/fly/bus and tmpfiles creates it. Watchdog check 2 reads the feed counters from whoever serves the feed: flyedge when fly.env says bus. lint.sh holds all of that, and drives check 2's choice against a fixture. --- infra/05-deploy.sh | 9 ++++ infra/bin/fly-watchdog | 26 +++++++++++- infra/build/build-flysim.sh | 12 ++++++ infra/build/package-release.sh | 10 +++++ infra/config/fly-tmpfiles.conf | 3 ++ infra/env/example.env | 7 ++++ infra/tests/lint.sh | 77 ++++++++++++++++++++++++++++++++++ infra/units/flyedge.service | 56 +++++++++++++++++++++++++ infra/units/flysim.service | 4 ++ 9 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 infra/units/flyedge.service diff --git a/infra/05-deploy.sh b/infra/05-deploy.sh index 705fb13..5d9b922 100644 --- a/infra/05-deploy.sh +++ b/infra/05-deploy.sh @@ -499,6 +499,15 @@ trap 'rm -f "$tmp_fly_env" "$tmp_flypush_env"' EXIT # "palette"/"plan" as "macros" with a warning, and refuses an unrecognised # value outright. echo "FLY_MACRO_MODE=${FLY_MACRO_MODE:-raw}" + # Who serves the feed WebSocket (docs/design/flybus.md, "Feed over the + # bus"). "direct" is the default and is flysim binding :7400 itself, as + # every release before this knob. "bus" makes flysim publish on its + # embedded feed bus and leave :7400 to flyedge.service, which this script + # never enables: see that unit's header for the switch. Written + # unconditionally, like FLY_MACRO_MODE, so one grep says which a box runs. + # Watchdog check 2 reads this line to know whose /metrics carries the + # feed counters (flysim's :9101, or flyedge's loopback :9102). + echo "FLY_FEED_VIA=${FLY_FEED_VIA:-direct}" # How long a macro leaves a target alone after a walk to it aborted # (macros.md section 12.1, the Viridian stall). Only written when it is set, # because the default lives in the crate and a box that has not tuned it diff --git a/infra/bin/fly-watchdog b/infra/bin/fly-watchdog index 038ed88..8499058 100755 --- a/infra/bin/fly-watchdog +++ b/infra/bin/fly-watchdog @@ -32,6 +32,9 @@ log_info() { : "${FLY_CONTROL_URL:=http://127.0.0.1:7401}" : "${FLY_METRICS_URL:=http://127.0.0.1:9101}" +# flyedge's loopback /metrics (units/flyedge.service), read by check 2 when +# fly.env says FLY_FEED_VIA=bus. +: "${FLY_EDGE_METRICS_URL:=http://127.0.0.1:9102}" : "${FLY_STATE_HOT:=/run/fly/state}" : "${FLY_MEDIA_DIR:=/srv/fly/media}" : "${MEDIAMTX_API:=http://127.0.0.1:9997}" @@ -335,10 +338,31 @@ check_flysim() { # read-only metrics listener (infra.md section 5; not superseded by the # feed/control contracts). Flat frames counter across two passes, or zero # clients, means the page is dead/frozen even though Chromium is alive. +# +# The two counters belong to whoever serves the feed: flysim itself, or with +# FLY_FEED_VIA=bus in fly.env, flyedge (docs/design/flybus.md, "Feed over the +# bus"), which exports them under the same names. FLY_FEED_METRICS_URL in the +# watchdog's own environment overrides both. # ============================================================================ +feed_metrics_url() { + if [ -n "${FLY_FEED_METRICS_URL:-}" ]; then + echo "$FLY_FEED_METRICS_URL" + return + fi + local via="" + if [ -f "$FLY_ENV_FILE" ]; then + via="$(awk -F= '/^FLY_FEED_VIA=/ { print $2; exit }' "$FLY_ENV_FILE" 2>/dev/null | tr -d ' \r' || true)" + fi + if [ "$via" = "bus" ]; then + echo "$FLY_EDGE_METRICS_URL" + else + echo "$FLY_METRICS_URL" + fi +} + check_flystage() { local metrics frames clients ok=1 - metrics="$(curl -fsS "${FLY_METRICS_URL}/metrics" 2>/dev/null || true)" + metrics="$(curl -fsS "$(feed_metrics_url)/metrics" 2>/dev/null || true)" if [ -z "$metrics" ]; then ok=0 else diff --git a/infra/build/build-flysim.sh b/infra/build/build-flysim.sh index ac6385a..3a56ca7 100755 --- a/infra/build/build-flysim.sh +++ b/infra/build/build-flysim.sh @@ -79,6 +79,12 @@ log "building in $crate_dir for target-cpu=haswell (the host is E5-2660 v3, Hasw ( cd "$crate_dir" RUSTFLAGS="-C target-cpu=haswell" cargo build --release --target "$CARGO_TARGET" --bin flysim "${features_args[@]}" + # fly-edge (FLY_FEED_VIA=bus, docs/design/flybus.md): the feed WebSocket + # served from flysim's feed bus. Small, and no cargo features of its own; + # built every time so a release can switch a container onto the bus + # without a rebuild. It lands next to OUT_PATH, where package-release.sh + # looks for it. + RUSTFLAGS="-C target-cpu=haswell" cargo build --release --target "$CARGO_TARGET" --bin fly-edge ) built="${crate_dir}/target/${CARGO_TARGET}/release/flysim" @@ -106,4 +112,10 @@ fi cp "$built" "$OUT_PATH" chmod 0755 "$OUT_PATH" log "built $OUT_PATH ($(du -h "$OUT_PATH" | cut -f1))" +edge_built="${crate_dir}/target/${CARGO_TARGET}/release/fly-edge" +[ -x "$edge_built" ] || die "expected binary not found after build: $edge_built" +edge_out="$(dirname "$OUT_PATH")/fly-edge" +cp "$edge_built" "$edge_out" +chmod 0755 "$edge_out" +log "built $edge_out ($(du -h "$edge_out" | cut -f1))" log "next: infra/build/package-release.sh VERSION $OUT_PATH " diff --git a/infra/build/package-release.sh b/infra/build/package-release.sh index 718d678..0610db2 100755 --- a/infra/build/package-release.sh +++ b/infra/build/package-release.sh @@ -11,6 +11,9 @@ # # Output: OUT_DIR/flybrain-.tar.gz, laid out as # flysim (the binary, mode 0755) +# fly-edge (the feed-bus edge, mode 0755, when build-flysim.sh +# left one beside FLYSIM_BIN; flyedge.service stays +# inactive on a release without it) # stage/... (apps/stage's build output) # bridge/... (services/bridge + node_modules) # data/fafb-v783/... (the connectome, from the repo; FLY_DATASET points here) @@ -72,6 +75,13 @@ mkdir -p "$release_dir" cp "$FLYSIM_BIN" "${release_dir}/flysim" chmod 0755 "${release_dir}/flysim" +EDGE_BIN="$(dirname "$FLYSIM_BIN")/fly-edge" +if [ -x "$EDGE_BIN" ]; then + cp "$EDGE_BIN" "${release_dir}/fly-edge" + chmod 0755 "${release_dir}/fly-edge" +else + log "no fly-edge beside $FLYSIM_BIN; packaging without it (FLY_FEED_VIA=bus unavailable in this release)" +fi cp -a "$STAGE_DIR" "${release_dir}/stage" cp -a "$BRIDGE_DIR" "${release_dir}/bridge" diff --git a/infra/config/fly-tmpfiles.conf b/infra/config/fly-tmpfiles.conf index e568cb5..4aafd6a 100644 --- a/infra/config/fly-tmpfiles.conf +++ b/infra/config/fly-tmpfiles.conf @@ -6,6 +6,9 @@ d /run/fly 0750 fly fly - d /run/fly/pulse 0750 fly fly - d /run/fly/state 0750 fly fly - d /run/fly/wd 0750 fly fly - +# ADDED: the feed bus (FLY_FEED_VIA=bus, docs/design/flybus.md): flysim's +# router socket and artifact store. flysim also creates it, 0700, on start. +d /run/fly/bus 0700 fly fly - d /var/lib/fly 0750 fly fly - d /var/lib/fly/chrome 0700 fly fly - d /srv/fly/state 0750 fly fly - diff --git a/infra/env/example.env b/infra/env/example.env index fc19970..21afecc 100644 --- a/infra/env/example.env +++ b/infra/env/example.env @@ -312,6 +312,13 @@ CHAT_DENY_LIST=/srv/fly/chat-deny.txt # "palette" and "plan" are the two modes section 12 replaced; flysim still reads # either as "macros", with a warning, for one release. FLY_MACRO_MODE=raw +# --- feed path ---------------------------------------------------------------- +# Who serves ws://127.0.0.1:7400/feed (docs/design/flybus.md, "Feed over the +# bus"). direct: flysim binds it, as always. bus: flysim publishes every +# snapshot on its embedded feed bus (/run/fly/bus) and flyedge.service serves +# the same bytes; enable that unit by hand (its header has the steps). +# Watchdog check 2 follows this setting to the edge's counters by itself. +FLY_FEED_VIA=direct # How long a macro leaves a target alone after a walk to it aborted "blocked" or # "timeout" (macros.md section 12.1). Session state, so a restart offers every # target once more. Unset means the default, 10. diff --git a/infra/tests/lint.sh b/infra/tests/lint.sh index 0a45f9a..965d5f0 100755 --- a/infra/tests/lint.sh +++ b/infra/tests/lint.sh @@ -429,6 +429,83 @@ else fi rm -rf "$lint_tmp" +# --------------------------------------------------------------------------- +# 3b2. The feed bus edge (docs/design/flybus.md, "Feed over the bus"). +# +# flyedge.service is off unless the operator switches a container to +# FLY_FEED_VIA=bus by hand, and when it is on it must follow flysim, which +# owns the router. What would break that is statically visible: the unit +# ending up in fly.target or 07-enable's list, losing its ordering on +# flysim, or the deploy no longer writing the default. Watchdog check 2's +# choice of /metrics is driven for real against a fixture fly.env. +# --------------------------------------------------------------------------- +echo "--- flyedge.service: off by default, after and bound to flysim ---" +EDGE_UNIT="$INFRA_DIR/units/flyedge.service" +if [ ! -f "$EDGE_UNIT" ]; then + fail "units/flyedge.service is missing" +else + grep -qE '^After=.*\bflysim\.service\b' "$EDGE_UNIT" \ + && pass "flyedge.service orders itself After=flysim.service" \ + || fail "flyedge.service must be After=flysim.service: flysim owns the feed router" + grep -qE '^Requires=.*\bflysim\.service\b' "$EDGE_UNIT" \ + && pass "flyedge.service Requires=flysim.service" \ + || fail "flyedge.service must Require flysim.service, so a stop or restart of flysim takes the edge with it" + grep -qE '^ExecStart=/opt/fly/current/fly-edge$' "$EDGE_UNIT" \ + && pass "flyedge.service runs the release's fly-edge" \ + || fail "flyedge.service ExecStart must be /opt/fly/current/fly-edge" + grep -qE '^ConditionPathExists=/opt/fly/current/fly-edge$' "$EDGE_UNIT" \ + && pass "flyedge.service stays inactive on a release without fly-edge" \ + || fail "flyedge.service needs ConditionPathExists=/opt/fly/current/fly-edge (a release before it has none)" + grep -qE '^Environment=FLY_EDGE_METRICS_ADDR=127\.0\.0\.1:' "$EDGE_UNIT" \ + && pass "flyedge.service keeps its metrics on loopback" \ + || fail "flyedge.service FLY_EDGE_METRICS_ADDR must be a 127.0.0.1 address" +fi +if grep -E '^(Wants|Requires)=' "$INFRA_DIR/units/fly.target" | grep -q 'flyedge'; then + fail "fly.target pulls flyedge.service in; it must stay off until the operator enables it" +else + pass "fly.target does not pull flyedge.service in" +fi +if grep -E '^(ALWAYS_ON_UNITS|APP_UNITS)=' "$INFRA_DIR/07-enable.sh" "$INFRA_DIR/verify.sh" | grep -q 'flyedge'; then + fail "07-enable.sh or verify.sh lists flyedge.service as always-on" +else + pass "07-enable.sh and verify.sh leave flyedge.service alone" +fi +grep -qF 'echo "FLY_FEED_VIA=${FLY_FEED_VIA:-direct}"' "$INFRA_DIR/05-deploy.sh" \ + && pass "05-deploy.sh writes FLY_FEED_VIA with direct as the default" \ + || fail "05-deploy.sh must write FLY_FEED_VIA=\${FLY_FEED_VIA:-direct} into fly.env" +if grep -qE '^Environment=FLY_FEED_VIA' "$INFRA_DIR/units/flysim.service"; then + fail "flysim.service pins FLY_FEED_VIA; it belongs to fly.env so a box can be switched by deploy" +else + pass "flysim.service leaves FLY_FEED_VIA to fly.env" +fi + +echo "--- fly-watchdog check 2: the feed counters follow FLY_FEED_VIA ---" +if ! tail -n1 "$INFRA_DIR/bin/fly-watchdog" | grep -qE '^main "\$@"$'; then + fail "fly-watchdog: expected the last line to be 'main \"\$@\"' — the check-2 fixture strips it" +else + fe_fixture="$(mktemp -d "${TMPDIR:-/tmp}/fly-lint-edge.XXXXXX")" + sed '$d' "$INFRA_DIR/bin/fly-watchdog" > "$fe_fixture/wd.sh" + feed_url_case() { + local label="$1" env_line="$2" override="$3" want="$4" got + printf '%s\n' "$env_line" > "$fe_fixture/fly.env" + got="$(FLY_ENV_FILE="$fe_fixture/fly.env" FLY_FEED_METRICS_URL="$override" \ + FLY_METRICS_URL=http://sim FLY_EDGE_METRICS_URL=http://edge \ + WD_RUN_DIR="$fe_fixture/run" WD_STATE_DIR="$fe_fixture/state" \ + TEXTFILE_DIR="$fe_fixture/textfile" \ + bash -c "source '$fe_fixture/wd.sh'; feed_metrics_url" 2>&1 || true)" + if [ "$got" = "$want" ]; then + pass "check 2 feed metrics: $label -> $got" + else + fail "check 2 feed metrics: $label: got '$got', want '$want'" + fi + } + feed_url_case "direct" "FLY_FEED_VIA=direct" "" "http://sim" + feed_url_case "no FLY_FEED_VIA line (a fly.env before it)" "FLY_GAME=pokemon-red" "" "http://sim" + feed_url_case "bus" "FLY_FEED_VIA=bus" "" "http://edge" + feed_url_case "explicit override wins" "FLY_FEED_VIA=bus" "http://other" "http://other" + rm -rf "$fe_fixture" +fi + # --------------------------------------------------------------------------- # 3c. lib/common.sh cpuset_partition — the three-way cpuset split used by # 05-deploy.sh section 3b (flysim / page-capture / flycast). Run as its own diff --git a/infra/units/flyedge.service b/infra/units/flyedge.service new file mode 100644 index 0000000..a0d0f17 --- /dev/null +++ b/infra/units/flyedge.service @@ -0,0 +1,56 @@ +# infra/units/flyedge.service — pushed to /etc/systemd/system/flyedge.service. +# +# The feed WebSocket served from flysim's feed bus (docs/design/flybus.md, +# "Feed over the bus"; services/flysim/crates/fly-edge). DISABLED BY DEFAULT: +# it is in no target's Wants=/Requires= and 07-enable.sh does not enable it. +# With FLY_FEED_VIA=direct (the default, written into /etc/fly/fly.env by +# 05-deploy.sh) flysim binds 127.0.0.1:7400 itself and this unit has nothing +# to do. To move the feed onto the bus on one container: +# +# 1. FLY_FEED_VIA=bus in the env file, then 05-deploy.sh (rewrites fly.env); +# 2. systemctl enable --now flyedge.service; systemctl restart flysim.service +# (flysim stops binding :7400, the edge binds it once the first snapshot +# is on the bus); +# 3. nothing for the watchdog: check 2 reads FLY_FEED_VIA from fly.env and +# follows the feed counters to this unit's loopback /metrics. +# +# Back: FLY_FEED_VIA=direct, deploy, systemctl disable --now flyedge.service, +# restart flysim. +# +# Ordering (docs/design/flybus.md, amendment "Feed store lifecycle"): flysim +# owns the router and its store under /run/fly/bus, so it starts first and +# the edge follows it. Requires= makes an explicit stop or restart of flysim +# (the unstick rule's `systemctl restart flysim.service` included) stop or +# restart the edge with it. A crash-restart of flysim is covered by the edge +# itself: it drops its clients, unbinds :7400 and reconnects every 500 ms, +# so nothing here has to be restarted by hand. The edge holds no state; the +# store is flysim's and a new router removes the previous one's directory. +[Unit] +Description=flyedge: the feed WebSocket served from flysim's feed bus +After=flysim.service +Requires=flysim.service +# A release that predates fly-edge has no binary; stay cleanly inactive +# rather than restart-looping (the flybridge.service header explains why a +# Condition, not a start limit). +ConditionPathExists=/opt/fly/current/fly-edge + +[Service] +Type=simple +User=fly +# FLY_FEED_VIA, FLY_BUS_DIR and the rest of flysim's configuration: the edge +# reads the same file so the two cannot disagree about the port or the bus. +EnvironmentFile=/etc/fly/fly.env +Environment=FLY_FEED_BIND=127.0.0.1:7400 +Environment=FLY_BUS_DIR=/run/fly/bus +# Its own read-only /metrics and /healthz, for watchdog check 2 in bus mode. +# Loopback only: nothing off the container needs the edge's counters. +Environment=FLY_EDGE_METRICS_ADDR=127.0.0.1:9102 +ExecStart=/opt/fly/current/fly-edge +Restart=always +RestartSec=2 +# A few snapshots in flight and a WebSocket per client; the store itself is +# flysim's (tmpfs, bounded by feedbus::limits at 32 MiB). +MemoryMax=256M + +[Install] +WantedBy=fly.target diff --git a/infra/units/flysim.service b/infra/units/flysim.service index 8363bed..df2dc48 100644 --- a/infra/units/flysim.service +++ b/infra/units/flysim.service @@ -30,6 +30,10 @@ WatchdogSec=30 User=fly EnvironmentFile=/etc/fly/fly.env Environment=FLY_FEED_BIND=127.0.0.1:7400 +# Used only with FLY_FEED_VIA=bus (fly.env; default direct): the embedded +# feed router's socket and artifact store, on tmpfs. flyedge.service names +# the same directory. docs/design/flybus.md, "Feed over the bus". +Environment=FLY_BUS_DIR=/run/fly/bus Environment=FLY_CONTROL_BIND=127.0.0.1:7401 Environment=FLY_METRICS_ADDR=0.0.0.0:9101 Environment=FLY_STATE_HOT=/run/fly/state