#!/usr/bin/env bash
# infra/01-create-ct.sh ENVFILE
#
# Create the LXC container for one demo, guarded on `pct config` so a
# second run is a no-op. Run on the host as root. See docs/design/infra.md
# section 1, and docs/design/gpu.md sections 1 and 8 for the GPU=1 path
# (NVIDIA passthrough block + CPU pinning, converged into
# /etc/pve/lxc/$CTID.conf).
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/common.sh
. "$SCRIPT_DIR/lib/common.sh"

[ $# -eq 1 ] || die "usage: $0 ENVFILE"
load_env "$1"
require_pve_host
need pct pveam

# ONBOOT defaults to 0 (do not start at host boot) when an env file does
# not set it — safe for a throwaway/dev container; the release env
# (<release-env>) sets ONBOOT=1 explicitly.
ONBOOT="${ONBOOT:-0}"

if ct_exists "$CTID"; then
    log "01-create-ct: CT $CTID already exists (pct config succeeded), skipping pct create"
else
    log "01-create-ct: creating CT $CTID ($HOSTNAME)"

    # The template name carries a real patch level, so it cannot be hardcoded
    # once and left alone (the literal "13.x" placeholder this used to hold was
    # not a name `pct create` accepts, so it would have failed provision.sh
    # step 1 for every env file; the host carries
    # debian-13-standard_13.6-1_amd64.tar.zst). Resolve the newest Debian 13
    # standard template actually present on this host, and let TEMPLATE= pin an
    # exact one. Verified against the host's own `pveam list local` on 2026-09-15.
    template="${TEMPLATE:-}"
    if [ -z "$template" ]; then
        template="$(pveam list local \
            | awk '$1 ~ /^local:vztmpl\/debian-13-standard_.*_amd64\.tar\.zst$/ {print $1}' \
            | sort -V | tail -1)"
        [ -n "$template" ] || die "no debian-13-standard amd64 template found in 'pveam list local'. Download one first: pveam update && pveam available --section system | grep debian-13-standard && pveam download local <exact-name>"
        log "01-create-ct: using template $template"
    fi
    startup_arg=""
    if [ -n "${STARTUP_ORDER:-}" ] && [ -n "${STARTUP_UP:-}" ]; then
        startup_arg="order=${STARTUP_ORDER},up=${STARTUP_UP}"
    fi

    # shellcheck disable=SC2086
    pct create "$CTID" "$template" \
        --hostname "$HOSTNAME" --ostype debian --unprivileged 1 \
        --cores "$CORES" --memory "$MEMORY_MB" --swap "$SWAP_MB" \
        --features nesting=1 \
        --rootfs "local-zfs:${ROOTFS_GB}" \
        --mp0 "local-zfs:${STATE_MP_GB},mp=/srv/fly/state" \
        --mp1 "bulk-array:${MEDIA_MP_GB},mp=/srv/fly/media" \
        --net0 name=eth0,bridge=vmbr0,ip=dhcp \
        --onboot "$ONBOOT" \
        ${startup_arg:+--startup "$startup_arg"} \
        --description "fly demo: flysim/flystage/flybridge/flycast (flybrain/infra), env=$1"

    log "01-create-ct: created CT $CTID, waiting for it to be reachable"
fi

# ---------------------------------------------------------------------------
# GPU passthrough and CPU pinning (docs/design/gpu.md sections 1 and 8).
#
# These are `lxc.*` raw keys, which `pct set` does not accept, so they go
# into /etc/pve/lxc/$CTID.conf as two sentinel blocks, read-modify-write,
# written only on a difference. They are only read at container START, so a
# change means the container has to be restarted — which is what the block
# below does, and why this runs before the "make sure it's started" step.
#
# The two blocks are gated INDEPENDENTLY (2026-09-16, the release container provisioning):
# the NVIDIA passthrough block on GPU=1, the cpuset block on CPUSET being
# set at all. They used to share one `if [ "$GPU" = 1 ]`, which silently
# skipped the CPU pinning on every CPU-only container — including the
# release box (<release-env>: GPU=0, CPUSET=eight whole cores), where
# pinning matters MORE, not less: P0 run 2 measured that without the
# partition flysim cannot hold real time and x11grab loses ~4 frames a
# second in both directions, and that whole measurement was taken with the
# x264 encoder. The GPU is the encoder's business; the cpuset is the sim's.
#
# TWO blocks, not one, and this is load-bearing: `fly-nvidia` is owned and
# regenerated by fly-nvidia-majors.service on the host at every boot (the
# uvm and caps majors are dynamic), so anything else living inside that
# block would be deleted on the next reboot. The cpuset therefore gets its
# own `fly-cpuset` block, which only this script writes. A small deviation
# from gpu.md section 8's "the block", for the reason above.
#
# The nvidia block is generated by invoking infra/host/fly-nvidia-majors.sh
# — the same script the host unit runs — rather than by a second copy of
# the same logic here, so the majors can never disagree between create time
# and boot time.
# ---------------------------------------------------------------------------
CONF_CHANGED=0
conf="/etc/pve/lxc/${CTID}.conf"
if [ "${GPU:-0}" = 1 ]; then
    [ -f "$conf" ] || die "01-create-ct: GPU=1 but $conf does not exist (did pct create really succeed?)"

    log "01-create-ct: GPU=1 — converging the fly-nvidia block via host/fly-nvidia-majors.sh (majors read live from /proc/devices)"
    majors_out="$("$SCRIPT_DIR/host/fly-nvidia-majors.sh" "$CTID")"
    printf '%s\n' "$majors_out"
    if printf '%s\n' "$majors_out" | grep -qx "${CTID} changed"; then
        CONF_CHANGED=1
    fi
else
    log "01-create-ct: GPU=0 (or unset) — no NVIDIA passthrough"
fi

if [ -n "${CPUSET:-}" ]; then
    [ -f "$conf" ] || die "01-create-ct: CPUSET is set but $conf does not exist (did pct create really succeed?)"

    log "01-create-ct: converging the fly-cpuset block (cpus=$CPUSET, mems=${CPUMEMS:-0})"
    cpuset_blk="$(mktemp "${TMPDIR:-/tmp}/fly-cpuset.XXXXXX")"
    {
        # mems as well as cpus: the neuron sweep is memory-bandwidth-bound,
        # so the pages must come from the socket the threads run on. Node 0
        # is also the GPU-local socket (gpu.md section 1) and is the
        # default, but it is NOT hardcoded any more: node 0 has only ten
        # physical cores and the two prod containers now take eight WHOLE
        # cores each, so fly-platformer lives on node 1 and sets CPUMEMS=1.
        # A CPUSET on one socket with cpuset.mems pointing at the other is
        # the worst of both worlds, which is exactly the mistake a
        # hardcoded 0 invites.
        echo "lxc.cgroup2.cpuset.cpus: ${CPUSET}"
        echo "lxc.cgroup2.cpuset.mems: ${CPUMEMS:-0}"
    } > "$cpuset_blk"
    # `cores:` stays in the PVE config for lxcfs cpuinfo masking; this raw
    # key is applied after PVE's own and wins. CONFIRMED on PVE 9.2.10
    # (2026-09-16, the dev container): `taskset -pc 1` inside the container reports
    # exactly CPUSET and cgroup2 cpuset.cpus.effective agrees, so the raw key
    # does win. verify.sh asserts it on every run.
    #
    # The two cpuset keys are passed as OWNED_KEYs so that a re-partition
    # replaces the old values instead of leaving a stale twin behind — and so
    # that PVE's habit of hoisting the sentinel comments away from their
    # content on every `pct start` does not read as a change (see
    # converge_conf_block's header).
    if [ "$(converge_conf_block "$conf" fly-cpuset "$cpuset_blk" infra/01-create-ct.sh \
            "lxc.cgroup2.cpuset.cpus:" "lxc.cgroup2.cpuset.mems:")" = changed ]; then
        CONF_CHANGED=1
    fi
    rm -f "$cpuset_blk"
else
    log "01-create-ct: CPUSET is unset — no CPU pinning (PVE's automatic, non-whole-cored" \
        "cpuset applies; see infra/README.md step 4 for why that is not good enough for a stream container)"
fi

# Idempotent regardless of create/skip: make sure it's started before later
# steps try to pct exec into it.
status="$(pct status "$CTID" | awk '{print $2}')"
if [ "$status" = "running" ] && [ "$CONF_CHANGED" -eq 1 ]; then
    # lxc.* keys are read only at container start, so a running container is
    # still on the OLD passthrough/cpuset until it is restarted. Do it here
    # rather than leaving it as a runbook step nobody performs.
    log "01-create-ct: CT $CTID conf changed and the container is running — restarting it so the lxc.* keys take effect"
    pct stop "$CTID"
    pct start "$CTID"
    for _ in $(seq 1 30); do
        pct exec "$CTID" -- true 2>/dev/null && break
        sleep 2
    done
    status="$(pct status "$CTID" | awk '{print $2}')"
fi
if [ "$status" != "running" ]; then
    log "01-create-ct: starting CT $CTID (status was: $status)"
    pct start "$CTID"
    # Wait for the guest agent / network, bounded.
    for _ in $(seq 1 30); do
        if pct exec "$CTID" -- true 2>/dev/null; then
            break
        fi
        sleep 2
    done
fi

pct exec "$CTID" -- true || die "CT $CTID did not become reachable via pct exec"

log "01-create-ct: CT $CTID is up. VERIFY: pct list shows the expected IP; cross-check against" \
    "the .3.2xx holder table in docs/design/infra.md section 0 before relying on it."
log "01-create-ct: done"