infra: flyedge.service, off by default, and the feed bus through build, deploy and watchdog

flyedge.service runs /opt/fly/current/fly-edge After= and Requires= flysim.service, with its metrics on loopback :9102 and a ConditionPathExists so a release without the binary leaves it inactive. It is in no target and 07-enable.sh does not enable it; the header has the switch and the way back. build-flysim.sh also builds fly-edge beside the flysim binary and package-release.sh ships it when present. 05-deploy.sh writes FLY_FEED_VIA (default direct) into fly.env, flysim.service names FLY_BUS_DIR=/run/fly/bus and tmpfiles creates it. Watchdog check 2 reads the feed counters from whoever serves the feed: flyedge when fly.env says bus. lint.sh holds all of that, and drives check 2's choice against a fixture.
2026-09-23 08:19:51 +00:00 · 2026-09-23 08:19:51 +00:00 · 3d9a08d0be
commit 3d9a08d0be
parent 34c7a56b25
9 changed files with 203 additions and 1 deletions
--- a/infra/05-deploy.sh
+++ b/infra/05-deploy.sh
@ -499,6 +499,15 @@ trap 'rm -f "$tmp_fly_env" "$tmp_flypush_env"' EXIT
    # "palette"/"plan" as "macros" with a warning, and refuses an unrecognised
    # value outright.
    echo "FLY_MACRO_MODE=${FLY_MACRO_MODE:-raw}"
+    # Who serves the feed WebSocket (docs/design/flybus.md, "Feed over the
+    # bus"). "direct" is the default and is flysim binding :7400 itself, as
+    # every release before this knob. "bus" makes flysim publish on its
+    # embedded feed bus and leave :7400 to flyedge.service, which this script
+    # never enables: see that unit's header for the switch. Written
+    # unconditionally, like FLY_MACRO_MODE, so one grep says which a box runs.
+    # Watchdog check 2 reads this line to know whose /metrics carries the
+    # feed counters (flysim's :9101, or flyedge's loopback :9102).
+    echo "FLY_FEED_VIA=${FLY_FEED_VIA:-direct}"
    # How long a macro leaves a target alone after a walk to it aborted
    # (macros.md section 12.1, the Viridian stall). Only written when it is set,
    # because the default lives in the crate and a box that has not tuned it
--- a/infra/bin/fly-watchdog
+++ b/infra/bin/fly-watchdog
@ -32,6 +32,9 @@ log_info() {

 : "${FLY_CONTROL_URL:=http://127.0.0.1:7401}"
 : "${FLY_METRICS_URL:=http://127.0.0.1:9101}"
+# flyedge's loopback /metrics (units/flyedge.service), read by check 2 when
+# fly.env says FLY_FEED_VIA=bus.
+: "${FLY_EDGE_METRICS_URL:=http://127.0.0.1:9102}"
 : "${FLY_STATE_HOT:=/run/fly/state}"
 : "${FLY_MEDIA_DIR:=/srv/fly/media}"
 : "${MEDIAMTX_API:=http://127.0.0.1:9997}"
@ -335,10 +338,31 @@ check_flysim() {
 # read-only metrics listener (infra.md section 5; not superseded by the
 # feed/control contracts). Flat frames counter across two passes, or zero
 # clients, means the page is dead/frozen even though Chromium is alive.
+#
+# The two counters belong to whoever serves the feed: flysim itself, or with
+# FLY_FEED_VIA=bus in fly.env, flyedge (docs/design/flybus.md, "Feed over the
+# bus"), which exports them under the same names. FLY_FEED_METRICS_URL in the
+# watchdog's own environment overrides both.
 # ============================================================================
+feed_metrics_url() {
+    if [ -n "${FLY_FEED_METRICS_URL:-}" ]; then
+        echo "$FLY_FEED_METRICS_URL"
+        return
+    fi
+    local via=""
+    if [ -f "$FLY_ENV_FILE" ]; then
+        via="$(awk -F= '/^FLY_FEED_VIA=/ { print $2; exit }' "$FLY_ENV_FILE" 2>/dev/null | tr -d ' \r' || true)"
+    fi
+    if [ "$via" = "bus" ]; then
+        echo "$FLY_EDGE_METRICS_URL"
+    else
+        echo "$FLY_METRICS_URL"
+    fi
+}
+
 check_flystage() {
    local metrics frames clients ok=1
-    metrics="$(curl -fsS "${FLY_METRICS_URL}/metrics" 2>/dev/null || true)"
+    metrics="$(curl -fsS "$(feed_metrics_url)/metrics" 2>/dev/null || true)"
    if [ -z "$metrics" ]; then
        ok=0
    else
--- a/infra/build/build-flysim.sh
+++ b/infra/build/build-flysim.sh
@ -79,6 +79,12 @@ log "building in $crate_dir for target-cpu=haswell (the host is E5-2660 v3, Hasw
 (
    cd "$crate_dir"
    RUSTFLAGS="-C target-cpu=haswell" cargo build --release --target "$CARGO_TARGET" --bin flysim "${features_args[@]}"
+    # fly-edge (FLY_FEED_VIA=bus, docs/design/flybus.md): the feed WebSocket
+    # served from flysim's feed bus. Small, and no cargo features of its own;
+    # built every time so a release can switch a container onto the bus
+    # without a rebuild. It lands next to OUT_PATH, where package-release.sh
+    # looks for it.
+    RUSTFLAGS="-C target-cpu=haswell" cargo build --release --target "$CARGO_TARGET" --bin fly-edge
 )

 built="${crate_dir}/target/${CARGO_TARGET}/release/flysim"
@ -106,4 +112,10 @@ fi
 cp "$built" "$OUT_PATH"
 chmod 0755 "$OUT_PATH"
 log "built $OUT_PATH ($(du -h "$OUT_PATH" | cut -f1))"
+edge_built="${crate_dir}/target/${CARGO_TARGET}/release/fly-edge"
+[ -x "$edge_built" ] || die "expected binary not found after build: $edge_built"
+edge_out="$(dirname "$OUT_PATH")/fly-edge"
+cp "$edge_built" "$edge_out"
+chmod 0755 "$edge_out"
+log "built $edge_out ($(du -h "$edge_out" | cut -f1))"
 log "next: infra/build/package-release.sh VERSION $OUT_PATH <stage-dir> <bridge-dir> <out-dir>"
--- a/infra/build/package-release.sh
+++ b/infra/build/package-release.sh
@ -11,6 +11,9 @@
 #
 # Output: OUT_DIR/flybrain-<version>.tar.gz, laid out as
 #   flysim               (the binary, mode 0755)
+#   fly-edge              (the feed-bus edge, mode 0755, when build-flysim.sh
+#                          left one beside FLYSIM_BIN; flyedge.service stays
+#                          inactive on a release without it)
 #   stage/...             (apps/stage's build output)
 #   bridge/...             (services/bridge + node_modules)
 #   data/fafb-v783/...     (the connectome, from the repo; FLY_DATASET points here)
@ -72,6 +75,13 @@ mkdir -p "$release_dir"

 cp "$FLYSIM_BIN" "${release_dir}/flysim"
 chmod 0755 "${release_dir}/flysim"
+EDGE_BIN="$(dirname "$FLYSIM_BIN")/fly-edge"
+if [ -x "$EDGE_BIN" ]; then
+    cp "$EDGE_BIN" "${release_dir}/fly-edge"
+    chmod 0755 "${release_dir}/fly-edge"
+else
+    log "no fly-edge beside $FLYSIM_BIN; packaging without it (FLY_FEED_VIA=bus unavailable in this release)"
+fi
 cp -a "$STAGE_DIR" "${release_dir}/stage"
 cp -a "$BRIDGE_DIR" "${release_dir}/bridge"

--- a/infra/config/fly-tmpfiles.conf
+++ b/infra/config/fly-tmpfiles.conf
@ -6,6 +6,9 @@ d /run/fly            0750 fly  fly  -
 d /run/fly/pulse      0750 fly  fly  -
 d /run/fly/state      0750 fly  fly  -
 d /run/fly/wd         0750 fly  fly  -
+# ADDED: the feed bus (FLY_FEED_VIA=bus, docs/design/flybus.md): flysim's
+# router socket and artifact store. flysim also creates it, 0700, on start.
+d /run/fly/bus        0700 fly  fly  -
 d /var/lib/fly        0750 fly  fly  -
 d /var/lib/fly/chrome 0700 fly  fly  -
 d /srv/fly/state      0750 fly  fly  -
--- a/infra/env/example.env
+++ b/infra/env/example.env
@ -312,6 +312,13 @@ CHAT_DENY_LIST=/srv/fly/chat-deny.txt
 # "palette" and "plan" are the two modes section 12 replaced; flysim still reads
 # either as "macros", with a warning, for one release.
 FLY_MACRO_MODE=raw
+# --- feed path ----------------------------------------------------------------
+# Who serves ws://127.0.0.1:7400/feed (docs/design/flybus.md, "Feed over the
+# bus"). direct: flysim binds it, as always. bus: flysim publishes every
+# snapshot on its embedded feed bus (/run/fly/bus) and flyedge.service serves
+# the same bytes; enable that unit by hand (its header has the steps).
+# Watchdog check 2 follows this setting to the edge's counters by itself.
+FLY_FEED_VIA=direct
 # How long a macro leaves a target alone after a walk to it aborted "blocked" or
 # "timeout" (macros.md section 12.1). Session state, so a restart offers every
 # target once more. Unset means the default, 10.
--- a/infra/tests/lint.sh
+++ b/infra/tests/lint.sh
@ -429,6 +429,83 @@ else
 fi
 rm -rf "$lint_tmp"

+# ---------------------------------------------------------------------------
+# 3b2. The feed bus edge (docs/design/flybus.md, "Feed over the bus").
+#
+# flyedge.service is off unless the operator switches a container to
+# FLY_FEED_VIA=bus by hand, and when it is on it must follow flysim, which
+# owns the router. What would break that is statically visible: the unit
+# ending up in fly.target or 07-enable's list, losing its ordering on
+# flysim, or the deploy no longer writing the default. Watchdog check 2's
+# choice of /metrics is driven for real against a fixture fly.env.
+# ---------------------------------------------------------------------------
+echo "--- flyedge.service: off by default, after and bound to flysim ---"
+EDGE_UNIT="$INFRA_DIR/units/flyedge.service"
+if [ ! -f "$EDGE_UNIT" ]; then
+    fail "units/flyedge.service is missing"
+else
+    grep -qE '^After=.*\bflysim\.service\b' "$EDGE_UNIT" \
+        && pass "flyedge.service orders itself After=flysim.service" \
+        || fail "flyedge.service must be After=flysim.service: flysim owns the feed router"
+    grep -qE '^Requires=.*\bflysim\.service\b' "$EDGE_UNIT" \
+        && pass "flyedge.service Requires=flysim.service" \
+        || fail "flyedge.service must Require flysim.service, so a stop or restart of flysim takes the edge with it"
+    grep -qE '^ExecStart=/opt/fly/current/fly-edge$' "$EDGE_UNIT" \
+        && pass "flyedge.service runs the release's fly-edge" \
+        || fail "flyedge.service ExecStart must be /opt/fly/current/fly-edge"
+    grep -qE '^ConditionPathExists=/opt/fly/current/fly-edge$' "$EDGE_UNIT" \
+        && pass "flyedge.service stays inactive on a release without fly-edge" \
+        || fail "flyedge.service needs ConditionPathExists=/opt/fly/current/fly-edge (a release before it has none)"
+    grep -qE '^Environment=FLY_EDGE_METRICS_ADDR=127\.0\.0\.1:' "$EDGE_UNIT" \
+        && pass "flyedge.service keeps its metrics on loopback" \
+        || fail "flyedge.service FLY_EDGE_METRICS_ADDR must be a 127.0.0.1 address"
+fi
+if grep -E '^(Wants|Requires)=' "$INFRA_DIR/units/fly.target" | grep -q 'flyedge'; then
+    fail "fly.target pulls flyedge.service in; it must stay off until the operator enables it"
+else
+    pass "fly.target does not pull flyedge.service in"
+fi
+if grep -E '^(ALWAYS_ON_UNITS|APP_UNITS)=' "$INFRA_DIR/07-enable.sh" "$INFRA_DIR/verify.sh" | grep -q 'flyedge'; then
+    fail "07-enable.sh or verify.sh lists flyedge.service as always-on"
+else
+    pass "07-enable.sh and verify.sh leave flyedge.service alone"
+fi
+grep -qF 'echo "FLY_FEED_VIA=${FLY_FEED_VIA:-direct}"' "$INFRA_DIR/05-deploy.sh" \
+    && pass "05-deploy.sh writes FLY_FEED_VIA with direct as the default" \
+    || fail "05-deploy.sh must write FLY_FEED_VIA=\${FLY_FEED_VIA:-direct} into fly.env"
+if grep -qE '^Environment=FLY_FEED_VIA' "$INFRA_DIR/units/flysim.service"; then
+    fail "flysim.service pins FLY_FEED_VIA; it belongs to fly.env so a box can be switched by deploy"
+else
+    pass "flysim.service leaves FLY_FEED_VIA to fly.env"
+fi
+
+echo "--- fly-watchdog check 2: the feed counters follow FLY_FEED_VIA ---"
+if ! tail -n1 "$INFRA_DIR/bin/fly-watchdog" | grep -qE '^main "\$@"$'; then
+    fail "fly-watchdog: expected the last line to be 'main \"\$@\"' — the check-2 fixture strips it"
+else
+    fe_fixture="$(mktemp -d "${TMPDIR:-/tmp}/fly-lint-edge.XXXXXX")"
+    sed '$d' "$INFRA_DIR/bin/fly-watchdog" > "$fe_fixture/wd.sh"
+    feed_url_case() {
+        local label="$1" env_line="$2" override="$3" want="$4" got
+        printf '%s\n' "$env_line" > "$fe_fixture/fly.env"
+        got="$(FLY_ENV_FILE="$fe_fixture/fly.env" FLY_FEED_METRICS_URL="$override" \
+            FLY_METRICS_URL=http://sim FLY_EDGE_METRICS_URL=http://edge \
+            WD_RUN_DIR="$fe_fixture/run" WD_STATE_DIR="$fe_fixture/state" \
+            TEXTFILE_DIR="$fe_fixture/textfile" \
+            bash -c "source '$fe_fixture/wd.sh'; feed_metrics_url" 2>&1 || true)"
+        if [ "$got" = "$want" ]; then
+            pass "check 2 feed metrics: $label -> $got"
+        else
+            fail "check 2 feed metrics: $label: got '$got', want '$want'"
+        fi
+    }
+    feed_url_case "direct" "FLY_FEED_VIA=direct" "" "http://sim"
+    feed_url_case "no FLY_FEED_VIA line (a fly.env before it)" "FLY_GAME=pokemon-red" "" "http://sim"
+    feed_url_case "bus" "FLY_FEED_VIA=bus" "" "http://edge"
+    feed_url_case "explicit override wins" "FLY_FEED_VIA=bus" "http://other" "http://other"
+    rm -rf "$fe_fixture"
+fi
+
 # ---------------------------------------------------------------------------
 # 3c. lib/common.sh cpuset_partition — the three-way cpuset split used by
 # 05-deploy.sh section 3b (flysim / page-capture / flycast). Run as its own
--- a/infra/units/flyedge.service
+++ b/infra/units/flyedge.service
@ -0,0 +1,56 @@
+# infra/units/flyedge.service — pushed to /etc/systemd/system/flyedge.service.
+#
+# The feed WebSocket served from flysim's feed bus (docs/design/flybus.md,
+# "Feed over the bus"; services/flysim/crates/fly-edge). DISABLED BY DEFAULT:
+# it is in no target's Wants=/Requires= and 07-enable.sh does not enable it.
+# With FLY_FEED_VIA=direct (the default, written into /etc/fly/fly.env by
+# 05-deploy.sh) flysim binds 127.0.0.1:7400 itself and this unit has nothing
+# to do. To move the feed onto the bus on one container:
+#
+#   1. FLY_FEED_VIA=bus in the env file, then 05-deploy.sh (rewrites fly.env);
+#   2. systemctl enable --now flyedge.service; systemctl restart flysim.service
+#      (flysim stops binding :7400, the edge binds it once the first snapshot
+#      is on the bus);
+#   3. nothing for the watchdog: check 2 reads FLY_FEED_VIA from fly.env and
+#      follows the feed counters to this unit's loopback /metrics.
+#
+# Back: FLY_FEED_VIA=direct, deploy, systemctl disable --now flyedge.service,
+# restart flysim.
+#
+# Ordering (docs/design/flybus.md, amendment "Feed store lifecycle"): flysim
+# owns the router and its store under /run/fly/bus, so it starts first and
+# the edge follows it. Requires= makes an explicit stop or restart of flysim
+# (the unstick rule's `systemctl restart flysim.service` included) stop or
+# restart the edge with it. A crash-restart of flysim is covered by the edge
+# itself: it drops its clients, unbinds :7400 and reconnects every 500 ms,
+# so nothing here has to be restarted by hand. The edge holds no state; the
+# store is flysim's and a new router removes the previous one's directory.
+[Unit]
+Description=flyedge: the feed WebSocket served from flysim's feed bus
+After=flysim.service
+Requires=flysim.service
+# A release that predates fly-edge has no binary; stay cleanly inactive
+# rather than restart-looping (the flybridge.service header explains why a
+# Condition, not a start limit).
+ConditionPathExists=/opt/fly/current/fly-edge
+
+[Service]
+Type=simple
+User=fly
+# FLY_FEED_VIA, FLY_BUS_DIR and the rest of flysim's configuration: the edge
+# reads the same file so the two cannot disagree about the port or the bus.
+EnvironmentFile=/etc/fly/fly.env
+Environment=FLY_FEED_BIND=127.0.0.1:7400
+Environment=FLY_BUS_DIR=/run/fly/bus
+# Its own read-only /metrics and /healthz, for watchdog check 2 in bus mode.
+# Loopback only: nothing off the container needs the edge's counters.
+Environment=FLY_EDGE_METRICS_ADDR=127.0.0.1:9102
+ExecStart=/opt/fly/current/fly-edge
+Restart=always
+RestartSec=2
+# A few snapshots in flight and a WebSocket per client; the store itself is
+# flysim's (tmpfs, bounded by feedbus::limits at 32 MiB).
+MemoryMax=256M
+
+[Install]
+WantedBy=fly.target
--- a/infra/units/flysim.service
+++ b/infra/units/flysim.service
@ -30,6 +30,10 @@ WatchdogSec=30
 User=fly
 EnvironmentFile=/etc/fly/fly.env
 Environment=FLY_FEED_BIND=127.0.0.1:7400
+# Used only with FLY_FEED_VIA=bus (fly.env; default direct): the embedded
+# feed router's socket and artifact store, on tmpfs. flyedge.service names
+# the same directory. docs/design/flybus.md, "Feed over the bus".
+Environment=FLY_BUS_DIR=/run/fly/bus
 Environment=FLY_CONTROL_BIND=127.0.0.1:7401
 Environment=FLY_METRICS_ADDR=0.0.0.0:9101
 Environment=FLY_STATE_HOT=/run/fly/state