183 lines
8.8 KiB
Bash
Executable file
183 lines
8.8 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# infra/01-create-ct.sh ENVFILE
|
|
#
|
|
# Create the LXC container for one demo, guarded on `pct config` so a
|
|
# second run is a no-op. Run on the host as root. See docs/design/infra.md
|
|
# section 1, and docs/design/gpu.md sections 1 and 8 for the GPU=1 path
|
|
# (NVIDIA passthrough block + CPU pinning, converged into
|
|
# /etc/pve/lxc/$CTID.conf).
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=lib/common.sh
|
|
. "$SCRIPT_DIR/lib/common.sh"
|
|
|
|
[ $# -eq 1 ] || die "usage: $0 ENVFILE"
|
|
load_env "$1"
|
|
require_pve_host
|
|
need pct pveam
|
|
|
|
# ONBOOT defaults to 0 (do not start at host boot) when an env file does
|
|
# not set it — safe for a throwaway/dev container; the release env
|
|
# (<release-env>) sets ONBOOT=1 explicitly.
|
|
ONBOOT="${ONBOOT:-0}"
|
|
|
|
if ct_exists "$CTID"; then
|
|
log "01-create-ct: CT $CTID already exists (pct config succeeded), skipping pct create"
|
|
else
|
|
log "01-create-ct: creating CT $CTID ($HOSTNAME)"
|
|
|
|
# The template name carries a real patch level, so it cannot be hardcoded
|
|
# once and left alone (the literal "13.x" placeholder this used to hold was
|
|
# not a name `pct create` accepts, so it would have failed provision.sh
|
|
# step 1 for every env file; the host carries
|
|
# debian-13-standard_13.6-1_amd64.tar.zst). Resolve the newest Debian 13
|
|
# standard template actually present on this host, and let TEMPLATE= pin an
|
|
# exact one. Verified against the host's own `pveam list local` on 2026-09-15.
|
|
template="${TEMPLATE:-}"
|
|
if [ -z "$template" ]; then
|
|
template="$(pveam list local \
|
|
| awk '$1 ~ /^local:vztmpl\/debian-13-standard_.*_amd64\.tar\.zst$/ {print $1}' \
|
|
| sort -V | tail -1)"
|
|
[ -n "$template" ] || die "no debian-13-standard amd64 template found in 'pveam list local'. Download one first: pveam update && pveam available --section system | grep debian-13-standard && pveam download local <exact-name>"
|
|
log "01-create-ct: using template $template"
|
|
fi
|
|
startup_arg=""
|
|
if [ -n "${STARTUP_ORDER:-}" ] && [ -n "${STARTUP_UP:-}" ]; then
|
|
startup_arg="order=${STARTUP_ORDER},up=${STARTUP_UP}"
|
|
fi
|
|
|
|
# shellcheck disable=SC2086
|
|
pct create "$CTID" "$template" \
|
|
--hostname "$HOSTNAME" --ostype debian --unprivileged 1 \
|
|
--cores "$CORES" --memory "$MEMORY_MB" --swap "$SWAP_MB" \
|
|
--features nesting=1 \
|
|
--rootfs "local-zfs:${ROOTFS_GB}" \
|
|
--mp0 "local-zfs:${STATE_MP_GB},mp=/srv/fly/state" \
|
|
--mp1 "bulk-array:${MEDIA_MP_GB},mp=/srv/fly/media" \
|
|
--net0 name=eth0,bridge=vmbr0,ip=dhcp \
|
|
--onboot "$ONBOOT" \
|
|
${startup_arg:+--startup "$startup_arg"} \
|
|
--description "fly demo: flysim/flystage/flybridge/flycast (flybrain/infra), env=$1"
|
|
|
|
log "01-create-ct: created CT $CTID, waiting for it to be reachable"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# GPU passthrough and CPU pinning (docs/design/gpu.md sections 1 and 8).
|
|
#
|
|
# These are `lxc.*` raw keys, which `pct set` does not accept, so they go
|
|
# into /etc/pve/lxc/$CTID.conf as two sentinel blocks, read-modify-write,
|
|
# written only on a difference. They are only read at container START, so a
|
|
# change means the container has to be restarted — which is what the block
|
|
# below does, and why this runs before the "make sure it's started" step.
|
|
#
|
|
# The two blocks are gated INDEPENDENTLY (2026-09-16, the release container provisioning):
|
|
# the NVIDIA passthrough block on GPU=1, the cpuset block on CPUSET being
|
|
# set at all. They used to share one `if [ "$GPU" = 1 ]`, which silently
|
|
# skipped the CPU pinning on every CPU-only container — including the
|
|
# release box (<release-env>: GPU=0, CPUSET=eight whole cores), where
|
|
# pinning matters MORE, not less: P0 run 2 measured that without the
|
|
# partition flysim cannot hold real time and x11grab loses ~4 frames a
|
|
# second in both directions, and that whole measurement was taken with the
|
|
# x264 encoder. The GPU is the encoder's business; the cpuset is the sim's.
|
|
#
|
|
# TWO blocks, not one, and this is load-bearing: `fly-nvidia` is owned and
|
|
# regenerated by fly-nvidia-majors.service on the host at every boot (the
|
|
# uvm and caps majors are dynamic), so anything else living inside that
|
|
# block would be deleted on the next reboot. The cpuset therefore gets its
|
|
# own `fly-cpuset` block, which only this script writes. A small deviation
|
|
# from gpu.md section 8's "the block", for the reason above.
|
|
#
|
|
# The nvidia block is generated by invoking infra/host/fly-nvidia-majors.sh
|
|
# — the same script the host unit runs — rather than by a second copy of
|
|
# the same logic here, so the majors can never disagree between create time
|
|
# and boot time.
|
|
# ---------------------------------------------------------------------------
|
|
CONF_CHANGED=0
|
|
conf="/etc/pve/lxc/${CTID}.conf"
|
|
if [ "${GPU:-0}" = 1 ]; then
|
|
[ -f "$conf" ] || die "01-create-ct: GPU=1 but $conf does not exist (did pct create really succeed?)"
|
|
|
|
log "01-create-ct: GPU=1 — converging the fly-nvidia block via host/fly-nvidia-majors.sh (majors read live from /proc/devices)"
|
|
majors_out="$("$SCRIPT_DIR/host/fly-nvidia-majors.sh" "$CTID")"
|
|
printf '%s\n' "$majors_out"
|
|
if printf '%s\n' "$majors_out" | grep -qx "${CTID} changed"; then
|
|
CONF_CHANGED=1
|
|
fi
|
|
else
|
|
log "01-create-ct: GPU=0 (or unset) — no NVIDIA passthrough"
|
|
fi
|
|
|
|
if [ -n "${CPUSET:-}" ]; then
|
|
[ -f "$conf" ] || die "01-create-ct: CPUSET is set but $conf does not exist (did pct create really succeed?)"
|
|
|
|
log "01-create-ct: converging the fly-cpuset block (cpus=$CPUSET, mems=${CPUMEMS:-0})"
|
|
cpuset_blk="$(mktemp "${TMPDIR:-/tmp}/fly-cpuset.XXXXXX")"
|
|
{
|
|
# mems as well as cpus: the neuron sweep is memory-bandwidth-bound,
|
|
# so the pages must come from the socket the threads run on. Node 0
|
|
# is also the GPU-local socket (gpu.md section 1) and is the
|
|
# default, but it is NOT hardcoded any more: node 0 has only ten
|
|
# physical cores and the two prod containers now take eight WHOLE
|
|
# cores each, so fly-platformer lives on node 1 and sets CPUMEMS=1.
|
|
# A CPUSET on one socket with cpuset.mems pointing at the other is
|
|
# the worst of both worlds, which is exactly the mistake a
|
|
# hardcoded 0 invites.
|
|
echo "lxc.cgroup2.cpuset.cpus: ${CPUSET}"
|
|
echo "lxc.cgroup2.cpuset.mems: ${CPUMEMS:-0}"
|
|
} > "$cpuset_blk"
|
|
# `cores:` stays in the PVE config for lxcfs cpuinfo masking; this raw
|
|
# key is applied after PVE's own and wins. CONFIRMED on PVE 9.2.10
|
|
# (2026-09-16, the dev container): `taskset -pc 1` inside the container reports
|
|
# exactly CPUSET and cgroup2 cpuset.cpus.effective agrees, so the raw key
|
|
# does win. verify.sh asserts it on every run.
|
|
#
|
|
# The two cpuset keys are passed as OWNED_KEYs so that a re-partition
|
|
# replaces the old values instead of leaving a stale twin behind — and so
|
|
# that PVE's habit of hoisting the sentinel comments away from their
|
|
# content on every `pct start` does not read as a change (see
|
|
# converge_conf_block's header).
|
|
if [ "$(converge_conf_block "$conf" fly-cpuset "$cpuset_blk" infra/01-create-ct.sh \
|
|
"lxc.cgroup2.cpuset.cpus:" "lxc.cgroup2.cpuset.mems:")" = changed ]; then
|
|
CONF_CHANGED=1
|
|
fi
|
|
rm -f "$cpuset_blk"
|
|
else
|
|
log "01-create-ct: CPUSET is unset — no CPU pinning (PVE's automatic, non-whole-cored" \
|
|
"cpuset applies; see infra/README.md step 4 for why that is not good enough for a stream container)"
|
|
fi
|
|
|
|
# Idempotent regardless of create/skip: make sure it's started before later
|
|
# steps try to pct exec into it.
|
|
status="$(pct status "$CTID" | awk '{print $2}')"
|
|
if [ "$status" = "running" ] && [ "$CONF_CHANGED" -eq 1 ]; then
|
|
# lxc.* keys are read only at container start, so a running container is
|
|
# still on the OLD passthrough/cpuset until it is restarted. Do it here
|
|
# rather than leaving it as a runbook step nobody performs.
|
|
log "01-create-ct: CT $CTID conf changed and the container is running — restarting it so the lxc.* keys take effect"
|
|
pct stop "$CTID"
|
|
pct start "$CTID"
|
|
for _ in $(seq 1 30); do
|
|
pct exec "$CTID" -- true 2>/dev/null && break
|
|
sleep 2
|
|
done
|
|
status="$(pct status "$CTID" | awk '{print $2}')"
|
|
fi
|
|
if [ "$status" != "running" ]; then
|
|
log "01-create-ct: starting CT $CTID (status was: $status)"
|
|
pct start "$CTID"
|
|
# Wait for the guest agent / network, bounded.
|
|
for _ in $(seq 1 30); do
|
|
if pct exec "$CTID" -- true 2>/dev/null; then
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
fi
|
|
|
|
pct exec "$CTID" -- true || die "CT $CTID did not become reachable via pct exec"
|
|
|
|
log "01-create-ct: CT $CTID is up. VERIFY: pct list shows the expected IP; cross-check against" \
|
|
"the .3.2xx holder table in docs/design/infra.md section 0 before relying on it."
|
|
log "01-create-ct: done"
|