kit

freebsd_vm.sh (25373B)

---

 #!/usr/bin/env bash
 # Prepare and run FreeBSD release VMs used as execution targets for kit tests.
 #
 # Defaults target FreeBSD 15.0-RELEASE VM images:
 #   amd64   -> BASIC-CLOUDINIT UFS qcow2
 #   aarch64 -> BASIC-CLOUDINIT UFS qcow2
 #   riscv64 -> UFS qcow2 (no BASIC-CLOUDINIT image is published)
 #
 # Downloaded .xz images live under XDG cache by default so `make clean` does
 # not remove them. Expanded VM disks, firmware vars, and seed ISOs live under
 # build/freebsd-vm/ by default.
 
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 VM_ROOT="${KIT_FREEBSD_VM_DIR:-$ROOT/build/freebsd-vm}"
 RELEASE="${KIT_FREEBSD_RELEASE:-15.0-RELEASE}"
 XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}"
 DL_ROOT="${KIT_FREEBSD_DOWNLOAD_DIR:-$XDG_CACHE_HOME/kit/freebsd-vm/$RELEASE}"
 BASE_URL="${KIT_FREEBSD_BASE_URL:-https://download.freebsd.org/releases/VM-IMAGES/$RELEASE}"
 QEMU_SHARE="${KIT_QEMU_SHARE:-/opt/homebrew/share/qemu}"
 SSH_USER="${KIT_FREEBSD_SSH_USER:-kit}"
 # The SSH key lives in the download cache (alongside the golden disks) so
 # `make clean` does not wipe it -- the key is baked into the golden disk's
 # authorized_keys, so losing it would lock the cached disks out. (Sysroots are
 # built VM-free from base.txz; see scripts/freebsd_sysroot.sh.)
 SSH_KEY="${KIT_FREEBSD_SSH_KEY:-$DL_ROOT/ssh/id_ed25519}"
 
 usage() {
   cat <<EOF
 usage: scripts/freebsd_vm.sh <command> <arch> [args...]
 
 commands:
   doctor                 print local QEMU/image tool availability
   fetch <arch>           download image and checksum
   prepare <arch>         restore/build a provisioned, ready-to-run disk
   seed <arch>            create a NoCloud cidata ISO for cloud-init images
   provision <arch>       boot once offline to create user+sshd, cache golden disk
   run <arch> [qemu...]   run VM in foreground with host-only SSH forwarding
   wait-ssh <arch>        wait for SSH and print uname
   ssh <arch> [cmd...]    SSH into a running VM
   scp <arch> <src> <dst>  copy a file into the running VM (dst is remote path)
   run-batch <arch> <dir> <res>
                          ship a staging dir into the VM, run its run-remote.sh,
                          and bring per-binary res/<id>.{rc,out,err} back into
                          <res> (used by test/lib/exec_vm.sh)
 
 arches:
   amd64 | x64 | aarch64 | arm64 | rv64 | riscv64
 
 env:
   KIT_FREEBSD_VM_DIR     artifact directory (default: build/freebsd-vm)
   KIT_FREEBSD_DOWNLOAD_DIR
                          downloaded .xz/checksum cache
                          (default: \${XDG_CACHE_HOME:-\$HOME/.cache}/kit/freebsd-vm/<release>)
   KIT_FREEBSD_RELEASE    FreeBSD release (default: 15.0-RELEASE)
   KIT_FREEBSD_MEM        QEMU memory in MiB (default: 2048)
   KIT_FREEBSD_CPUS       QEMU CPU count (default: 2)
   KIT_FREEBSD_PROVISION_TIMEOUT
                          seconds to wait for the offline provisioning boot to
                          reach a login prompt (default: 1800)
   KIT_FREEBSD_ACCEL      QEMU accel override (default: hvf for aarch64 on arm64 macOS)
   KIT_FREEBSD_SSH_USER   cloud-init user (default: kit)
   KIT_FREEBSD_SSH_KEY    private SSH key path
 EOF
 }
 
 die() {
   printf 'freebsd-vm: %s\n' "$*" >&2
   exit 1
 }
 
 canon_arch() {
   case "${1:-}" in
     amd64|x64|x86_64) echo amd64 ;;
     aarch64|arm64|aa64) echo aarch64 ;;
     riscv64|rv64) echo riscv64 ;;
     *) die "unknown arch '${1:-}'" ;;
   esac
 }
 
 arch_dir() {
   case "$(canon_arch "$1")" in
     amd64) echo amd64 ;;
     aarch64) echo aarch64 ;;
     riscv64) echo riscv64 ;;
   esac
 }
 
 image_file() {
   case "$(canon_arch "$1")" in
     amd64) echo "FreeBSD-$RELEASE-amd64-BASIC-CLOUDINIT-ufs.qcow2.xz" ;;
     aarch64) echo "FreeBSD-$RELEASE-arm64-aarch64-BASIC-CLOUDINIT-ufs.qcow2.xz" ;;
     riscv64) echo "FreeBSD-$RELEASE-riscv-riscv64-ufs.qcow2.xz" ;;
   esac
 }
 
 cloudinit_supported() {
   case "$(canon_arch "$1")" in
     amd64|aarch64) return 0 ;;
     riscv64) return 1 ;;
   esac
 }
 
 ssh_port() {
   case "$(canon_arch "$1")" in
     amd64) echo "${KIT_FREEBSD_AMD64_SSH_PORT:-2222}" ;;
     aarch64) echo "${KIT_FREEBSD_AARCH64_SSH_PORT:-2223}" ;;
     riscv64) echo "${KIT_FREEBSD_RISCV64_SSH_PORT:-2224}" ;;
   esac
 }
 
 qemu_bin() {
   case "$(canon_arch "$1")" in
     amd64) echo qemu-system-x86_64 ;;
     aarch64) echo qemu-system-aarch64 ;;
     riscv64) echo qemu-system-riscv64 ;;
   esac
 }
 
 disk_path() {
   echo "$VM_ROOT/images/freebsd-$(canon_arch "$1").qcow2"
 }
 
 seed_path() {
   echo "$VM_ROOT/seeds/freebsd-$(canon_arch "$1")-seed.iso"
 }
 
 checksum_path() {
   echo "$DL_ROOT/$(canon_arch "$1")-CHECKSUM.SHA256"
 }
 
 download_path() {
   echo "$DL_ROOT/$(image_file "$1")"
 }
 
 host_is_darwin_arm64() {
   [ "$(uname -s 2>/dev/null)" = "Darwin" ] &&
     { [ "$(uname -m 2>/dev/null)" = "arm64" ] ||
       [ "$(uname -m 2>/dev/null)" = "aarch64" ]; }
 }
 
 qemu_accel_args() {
   local arch="$1" accel="${KIT_FREEBSD_ACCEL:-}"
   if [ -z "$accel" ] && [ "$(canon_arch "$arch")" = "aarch64" ] &&
      host_is_darwin_arm64; then
     accel=hvf
   fi
   [ -n "$accel" ] && printf '%s\n%s\n' -accel "$accel"
 }
 
 firmware_code() {
   case "$(canon_arch "$1")" in
     amd64) echo "$QEMU_SHARE/edk2-x86_64-code.fd" ;;
     aarch64) echo "$QEMU_SHARE/edk2-aarch64-code.fd" ;;
     riscv64) echo "$QEMU_SHARE/edk2-riscv-code.fd" ;;
   esac
 }
 
 firmware_vars_template() {
   case "$(canon_arch "$1")" in
     amd64) echo "$QEMU_SHARE/edk2-i386-vars.fd" ;;
     aarch64) echo "$QEMU_SHARE/edk2-arm-vars.fd" ;;
     riscv64) echo "$QEMU_SHARE/edk2-riscv-vars.fd" ;;
   esac
 }
 
 firmware_vars() {
   echo "$VM_ROOT/firmware/$(canon_arch "$1")-vars.fd"
 }
 
 ensure_ssh_key() {
   if [ ! -f "$SSH_KEY" ]; then
     mkdir -p "$(dirname "$SSH_KEY")"
     ssh-keygen -q -t ed25519 -N "" -f "$SSH_KEY"
   fi
 }
 
 fetch_arch() {
   local arch="$1" dir file url cksum
   arch="$(canon_arch "$arch")"
   dir="$(arch_dir "$arch")"
   file="$(image_file "$arch")"
   url="$BASE_URL/$dir/Latest/$file"
   mkdir -p "$DL_ROOT"
   printf 'fetch checksum: %s\n' "$BASE_URL/$dir/Latest/CHECKSUM.SHA256"
   curl -fL --retry 3 --retry-delay 2 -o "$(checksum_path "$arch")" \
     "$BASE_URL/$dir/Latest/CHECKSUM.SHA256"
   printf 'fetch image: %s\n' "$url"
   curl -fL --continue-at - --retry 3 --retry-delay 2 \
     -o "$(download_path "$arch")" "$url"
   cksum="$(awk -v f="$file" '$2 == "(" f ")" && $3 == "=" { print $4 }' \
     "$(checksum_path "$arch")")"
   [ -n "$cksum" ] || die "checksum entry missing for $file"
   verify_sha256 "$arch" "$cksum"
 }
 
 verify_sha256() {
   local arch="$1" expect="$2" got
   got="$(shasum -a 256 "$(download_path "$arch")" | awk '{ print $1 }')"
   [ "$got" = "$expect" ] ||
     die "checksum mismatch for $(image_file "$arch"): got $got want $expect"
   printf 'verified: %s\n' "$(image_file "$arch")"
 }
 
 # Produce a ready-to-run, provisioned disk: restore the cached golden disk if we
 # have one, otherwise download/expand the pristine image and provision it. The
 # heavy lifting lives in ensure_disk (defined below).
 prepare_arch() {
   ensure_disk "$1"
 }
 
 seed_arch() {
   local arch="$1" seed_dir seed key
   arch="$(canon_arch "$arch")"
   cloudinit_supported "$arch" ||
     die "$arch does not have a FreeBSD BASIC-CLOUDINIT VM image"
   ensure_ssh_key
   mkdir -p "$VM_ROOT/seeds"
   seed_dir="$VM_ROOT/seeds/$(canon_arch "$arch")"
   seed="$(seed_path "$arch")"
   key="$(cat "$SSH_KEY.pub")"
   rm -rf "$seed_dir"
   mkdir -p "$seed_dir"
   cat > "$seed_dir/meta-data" <<EOF
 local-hostname: kit-freebsd-$arch
 EOF
   # nuageinit (FreeBSD's native cloud-init) processes this #cloud-config.
   # runcmd only re-asserts sshd_enable (idempotent): sshd is already enabled in
   # the image rc.conf, so the base rc starts it exactly once. We deliberately do
   # NOT "service sshd start" here -- that races the base rc start and dies with
   # "Bind to port 22 ... Address already in use".
   cat > "$seed_dir/user-data" <<EOF
 #cloud-config
 users:
   - name: $SSH_USER
     gecos: Kit FreeBSD Test User
     groups: wheel
     homedir: /home/$SSH_USER
     shell: /bin/sh
     sudo: ALL=(ALL) NOPASSWD:ALL
     ssh_authorized_keys:
       - "$key"
 ssh_authorized_keys:
   - "$key"
 network:
   version: 2
   ethernets:
     vtnet0:
       dhcp4: true
 runcmd:
   - sysrc sshd_enable=YES
 EOF
   rm -f "$seed"
   hdiutil makehybrid -quiet -iso -joliet -default-volume-name cidata \
     -o "$seed" "$seed_dir"
   printf 'seed ready: %s\n' "$seed"
 }
 
 ensure_firmware_vars() {
   local arch="$1" code vars tmpl
   code="$(firmware_code "$arch")"
   vars="$(firmware_vars "$arch")"
   tmpl="$(firmware_vars_template "$arch")"
   [ -f "$code" ] || die "missing QEMU firmware code: $code"
   [ -f "$tmpl" ] || die "missing QEMU firmware vars template: $tmpl"
   mkdir -p "$(dirname "$vars")"
   # Always start from the pristine template. The EDK2 firmware persists its
   # boot order / NVRAM into this vars file; once it drifts (e.g. after a run
   # with a different device set) it can boot the built-in UEFI Shell instead
   # of chainloading the disk, which looks exactly like a hang. Resetting per
   # run keeps the boot path deterministic; nothing here relies on NVRAM
   # persisting across runs.
   cp "$tmpl" "$vars"
 }
 
 append_firmware_args() {
   local arch="$1" code vars
   ensure_firmware_vars "$arch"
   code="$(firmware_code "$arch")"
   vars="$(firmware_vars "$arch")"
   QEMU_ARGS=("${QEMU_ARGS[@]}"
     -drive "if=pflash,format=raw,readonly=on,file=$code"
     -drive "if=pflash,format=raw,file=$vars")
 }
 
 # append_common_args ARCH DISK PORT [RESTRICT] [SERIALSPEC]
 #   RESTRICT non-empty -> isolate the guest from the outside network. DHCP and
 #     the SSH hostfwd still work, but outbound connections fail fast. Used while
 #     provisioning so firstboot_freebsd_update cannot run its slow networked
 #     update (and force a reboot) on the first boot.
 #   SERIALSPEC defaults to "mon:stdio" (interactive foreground). Provisioning
 #     passes "file:<log>" so a backgrounded boot can be watched.
 append_common_args() {
   local arch="$1" disk="$2" port="$3" restrict="${4:-}" serialspec="${5:-mon:stdio}"
   local mem="${KIT_FREEBSD_MEM:-2048}" cpus="${KIT_FREEBSD_CPUS:-2}"
   local netopt="user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22"
   [ -n "$restrict" ] && netopt="user,id=net0,restrict=on,hostfwd=tcp:127.0.0.1:$port-:22"
   QEMU_ARGS=("${QEMU_ARGS[@]}"
     -m "$mem"
     -smp "$cpus"
     -drive "if=virtio,format=qcow2,file=$disk"
     -netdev "$netopt"
     -device "virtio-net-pci,netdev=net0"
     -device "virtio-rng-pci"
     -display none
     -serial "$serialspec")
 }
 
 append_seed_args() {
   local arch="$1" seed
   if cloudinit_supported "$arch"; then
     seed="$(seed_path "$arch")"
     [ -f "$seed" ] || seed_arch "$arch"
     QEMU_ARGS=("${QEMU_ARGS[@]}"
       -drive "if=virtio,format=raw,media=cdrom,readonly=on,file=$seed")
   fi
 }
 
 # Machine + firmware + accel selection, shared by run and provision. Appends to
 # an already-initialized QEMU_ARGS.
 append_machine_args() {
   local arch="$1" accel_args
   case "$arch" in
     amd64)
       # Drop the default q35 VGA so FreeBSD selects the serial console as
       # primary. Otherwise it picks the (headless) VGA as primary console and
       # all userland/cloud-init output plus the login getty go to a head we
       # never display, making a slow TCG boot indistinguishable from a hang.
       #
       # QEMU's default x86_64 TCG model is "qemu64" (~x86-64-v1, no
       # POPCNT/SSE4.2/BMI). kit targets a modern x86_64 baseline and emits e.g.
       # `popcnt` for @popcount, which #UDs (SIGILL -> exit 132) on qemu64. Add the
       # missing baseline-ish features explicitly: "-cpu max" enables enough that
       # the FreeBSD/EDK2 boot fails under TCG, so extend qemu64 minimally instead.
       QEMU_ARGS=("${QEMU_ARGS[@]}" -machine q35 -vga none \
         -cpu "qemu64,+popcnt,+sse4.1,+sse4.2")
       append_firmware_args "$arch"
       ;;
     aarch64)
       QEMU_ARGS=("${QEMU_ARGS[@]}" -machine virt)
       accel_args="$(qemu_accel_args "$arch" | tr '\n' ' ')"
       if [ -n "$accel_args" ]; then
         # shellcheck disable=SC2206
         QEMU_ARGS=("${QEMU_ARGS[@]}" $accel_args -cpu host)
       else
         QEMU_ARGS=("${QEMU_ARGS[@]}" -cpu cortex-a72)
       fi
       append_firmware_args "$arch"
       ;;
     riscv64)
       # acpi=off forces the edk2-riscv firmware to hand FreeBSD a flattened
       # device tree (FDT) instead of ACPI tables. FreeBSD/riscv64 is FDT-only;
       # with ACPI advertised, its loader finds "no valid device tree blob" and
       # the kernel boots blind (no console/disk).
       QEMU_ARGS=("${QEMU_ARGS[@]}" -machine virt,acpi=off -cpu rv64)
       append_firmware_args "$arch"
       ;;
   esac
 }
 
 # A "golden" disk is one whose first boot is already done: the login user and
 # sshd are provisioned and the firstboot services have run. The build/ marker
 # records that the working disk is golden; the DL_ROOT copy is the durable
 # cache that survives `make clean` (which wipes build/).
 provisioned_marker() {
   echo "$VM_ROOT/images/freebsd-$(canon_arch "$1").provisioned"
 }
 
 golden_cache_path() {
   echo "$DL_ROOT/freebsd-$(canon_arch "$1")-golden.qcow2"
 }
 
 # Save the (clean, provisioned) working disk to the durable golden cache,
 # compressed so the cache stays small.
 cache_golden() {
   local arch="$1" disk golden
   arch="$(canon_arch "$arch")"
   disk="$(disk_path "$arch")"
   golden="$(golden_cache_path "$arch")"
   mkdir -p "$DL_ROOT"
   printf 'cache golden disk: %s -> %s\n' "$disk" "$golden"
   qemu-img convert -O qcow2 -c "$disk" "$golden.tmp"
   mv "$golden.tmp" "$golden"
 }
 
 # Boot a freshly-expanded cloud-init disk once, offline, so nuageinit creates
 # the login user and starts sshd while firstboot_freebsd_update fails fast for
 # lack of a route to the update servers (its slow networked update would
 # otherwise run on the first boot and force a reboot -- painful under TCG).
 # Once the guest reaches its login prompt the disk is golden.
 provision_nuageinit() {
   local arch="$1" disk port qemu log qpid t0 w
   arch="$(canon_arch "$arch")"
   disk="$(disk_path "$arch")"
   port="$(ssh_port "$arch")"
   qemu="$(qemu_bin "$arch")"
   command -v "$qemu" >/dev/null 2>&1 || die "$qemu not found on PATH"
   [ -f "$disk" ] || die "no disk to provision: $disk (run prepare first)"
   [ -f "$(seed_path "$arch")" ] || seed_arch "$arch"
   ensure_ssh_key
   log="$VM_ROOT/images/freebsd-$arch.provision.log"
   rm -f "$log"
   QEMU_ARGS=()
   append_machine_args "$arch"
   append_common_args "$arch" "$disk" "$port" restrict "file:$log"
   append_seed_args "$arch"
   printf 'provision: booting %s offline to create user+sshd (log: %s)\n' "$arch" "$log"
   "$qemu" "${QEMU_ARGS[@]}" >/dev/null 2>&1 &
   qpid=$!
   t0=$(date +%s)
   while kill -0 "$qpid" 2>/dev/null; do
     grep -q "login:" "$log" 2>/dev/null && break
     if [ $(( $(date +%s) - t0 )) -gt "${KIT_FREEBSD_PROVISION_TIMEOUT:-1800}" ]; then
       kill "$qpid" 2>/dev/null; wait "$qpid" 2>/dev/null || true
       die "provision timed out for $arch after ${KIT_FREEBSD_PROVISION_TIMEOUT:-1800}s (see $log)"
     fi
     sleep 3
   done
   kill -0 "$qpid" 2>/dev/null || die "provision VM for $arch exited before login (see $log)"
   printf 'provision: %s reached login; shutting down cleanly\n' "$arch"
   # sshd's banner is delayed ~30s under restrict (it reverse-DNS-resolves the
   # client with no route out), so allow a generous connect timeout. A forced
   # stop would also recover (UFS+SUJ journals), but a clean halt avoids an fsck.
   sleep 5
   timeout 150 ssh -i "$SSH_KEY" -p "$port" \
     -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
     -o LogLevel=ERROR -o ConnectTimeout=90 \
     "$SSH_USER@127.0.0.1" 'sync; shutdown -p now' >/dev/null 2>&1 || true
   w=0
   while kill -0 "$qpid" 2>/dev/null && [ "$w" -lt 90 ]; do sleep 1; w=$((w+1)); done
   if kill -0 "$qpid" 2>/dev/null; then
     kill "$qpid" 2>/dev/null; wait "$qpid" 2>/dev/null || true
   fi
 }
 
 # Write the expect program that drives a first-boot setup over the serial
 # console. Kept quoted (no shell expansion); it reads user/pubkey/timeout from
 # the environment so the pubkey's `/ + =` never collide with shell quoting.
 write_serial_provision_expect() {
   cat > "$1" <<'EXP'
 set timeout $env(KIT_PROV_TIMEOUT)
 set user $env(KIT_PROV_USER)
 set pubkey $env(KIT_PROV_PUBKEY)
 spawn {*}$argv
 set qpid [exp_pid]
 proc bail {msg code} {
   puts "\nprovision-serial: $msg"
   catch { exec kill -9 $::qpid }
   exit $code
 }
 expect {
   timeout { bail "timed out waiting for login prompt" 2 }
   "login: "
 }
 send "root\r"
 expect {
   timeout { bail "timed out after sending root" 2 }
   "Login incorrect" { bail "root login refused (image expects a password)" 3 }
   -re {Password:} { send "\r"; exp_continue }
   -re {[#%$] $} {}
 }
 # Drop to /bin/sh and set a unique prompt. The `@@K""IT@@` we type evaluates to
 # @@KIT@@ in the prompt but the typed echo contains the quotes, so expecting the
 # bare marker only ever matches a real prompt, never the command echo.
 send "/bin/sh\r"
 send "PS1=@@K\"\"IT@@\r"
 expect {
   timeout { bail "timed out establishing shell prompt" 2 }
   "@@KIT@@"
 }
 proc step {cmd} {
   send "$cmd\r"
   expect {
     timeout { bail "timed out running: $cmd" 2 }
     "@@KIT@@"
   }
 }
 step "pw useradd -n $user -m -G wheel -s /bin/sh"
 step "mkdir -p /home/$user/.ssh"
 step "printf '%s\n' '$pubkey' > /home/$user/.ssh/authorized_keys"
 step "chown -R $user:$user /home/$user/.ssh"
 step "chmod 700 /home/$user/.ssh"
 step "chmod 600 /home/$user/.ssh/authorized_keys"
 step "sysrc sshd_enable=YES"
 step "sysrc ifconfig_vtnet0=DHCP"
 step "service sshd keygen"
 send "sync; shutdown -p now\r"
 expect {
   timeout { bail "powered down but qemu still running; killed" 0 }
   eof { exit 0 }
 }
 EXP
 }
 
 # Provision an image that has no cloud-init seed (FreeBSD publishes no
 # BASIC-CLOUDINIT riscv64 image). Drive the serial console with expect: log in
 # as root, create the SSH user, enable sshd + DHCP, then power off.
 provision_serial() {
   local arch="$1" disk port qemu expf rc
   arch="$(canon_arch "$arch")"
   disk="$(disk_path "$arch")"
   port="$(ssh_port "$arch")"
   qemu="$(qemu_bin "$arch")"
   command -v expect >/dev/null 2>&1 ||
     die "expect not found on PATH (needed to provision $arch over the serial console)"
   command -v "$qemu" >/dev/null 2>&1 || die "$qemu not found on PATH"
   [ -f "$disk" ] || die "no disk to provision: $disk (run prepare first)"
   ensure_ssh_key
   local mem="${KIT_FREEBSD_MEM:-2048}" cpus="${KIT_FREEBSD_CPUS:-2}"
   QEMU_ARGS=()
   append_machine_args "$arch"
   QEMU_ARGS=("${QEMU_ARGS[@]}"
     -m "$mem"
     -smp "$cpus"
     -drive "if=virtio,format=qcow2,file=$disk"
     -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22"
     -device "virtio-net-pci,netdev=net0"
     -device "virtio-rng-pci"
     -display none
     -serial stdio
     -monitor none)
   expf="$VM_ROOT/images/freebsd-$arch.provision.exp"
   write_serial_provision_expect "$expf"
   printf 'provision: booting %s to configure sshd over the serial console (slow under TCG)...\n' "$arch"
   KIT_PROV_USER="$SSH_USER" \
   KIT_PROV_PUBKEY="$(cat "$SSH_KEY.pub")" \
   KIT_PROV_TIMEOUT="${KIT_FREEBSD_PROVISION_TIMEOUT:-1800}" \
     expect "$expf" "$qemu" "${QEMU_ARGS[@]}"
   rc=$?
   rm -f "$expf"
   [ "$rc" -eq 0 ] || die "serial provisioning failed for $arch (expect rc=$rc)"
 }
 
 # Provision a freshly-expanded disk into a golden one (nuageinit for cloud-init
 # images, serial-console otherwise), then mark + cache it.
 provision_arch() {
   local arch="$1"
   arch="$(canon_arch "$arch")"
   if cloudinit_supported "$arch"; then
     provision_nuageinit "$arch"
   else
     provision_serial "$arch"
   fi
   touch "$(provisioned_marker "$arch")"
   cache_golden "$arch"
   printf 'provisioned: %s\n' "$(disk_path "$arch")"
 }
 
 # Make build/ hold a provisioned (golden) working disk for arch, preferring the
 # cheapest source: an already-provisioned working disk, then the durable golden
 # cache, then a from-scratch expand+provision.
 ensure_disk() {
   local arch="$1" disk marker golden src
   arch="$(canon_arch "$arch")"
   disk="$(disk_path "$arch")"
   marker="$(provisioned_marker "$arch")"
   golden="$(golden_cache_path "$arch")"
   mkdir -p "$VM_ROOT/images"
   [ -f "$disk" ] && [ -f "$marker" ] && return 0
   if [ -f "$golden" ]; then
     printf 'restore golden disk: %s -> %s\n' "$golden" "$disk"
     qemu-img convert -O qcow2 "$golden" "$disk.tmp"
     mv "$disk.tmp" "$disk"
     touch "$marker"
     return 0
   fi
   fetch_arch "$arch"
   src="$(download_path "$arch")"
   printf 'expand image: %s -> %s\n' "$src" "$disk"
   xz -dc "$src" > "$disk.tmp"
   mv "$disk.tmp" "$disk"
   rm -f "$marker"
   provision_arch "$arch"
 }
 
 run_arch() {
   local arch="$1" disk port qemu
   shift
   arch="$(canon_arch "$arch")"
   disk="$(disk_path "$arch")"
   port="$(ssh_port "$arch")"
   qemu="$(qemu_bin "$arch")"
   command -v "$qemu" >/dev/null 2>&1 || die "$qemu not found on PATH"
   ensure_disk "$arch"
   QEMU_ARGS=()
   append_machine_args "$arch"
   append_common_args "$arch" "$disk" "$port"
   append_seed_args "$arch"
   printf 'ssh forward: 127.0.0.1:%s -> %s:22\n' "$port" "$arch"
   printf 'run: %s\n' "$qemu"
   exec "$qemu" "${QEMU_ARGS[@]}" "$@"
 }
 
 ssh_args() {
   local arch="$1" port
   port="$(ssh_port "$arch")"
   printf '%s\n' -i "$SSH_KEY" \
     -p "$port" \
     -o StrictHostKeyChecking=no \
     -o UserKnownHostsFile=/dev/null \
     -o LogLevel=ERROR \
     -o ConnectTimeout=5
 }
 
 wait_ssh() {
   local arch="$1" port args
   arch="$(canon_arch "$arch")"
   port="$(ssh_port "$arch")"
   if [ ! -f "$(provisioned_marker "$arch")" ] && [ ! -f "$(golden_cache_path "$arch")" ]; then
     die "$arch is not provisioned yet; run 'prepare $arch' or 'provision $arch' first"
   fi
   [ -f "$SSH_KEY" ] || die "missing SSH key: $SSH_KEY"
   printf 'waiting for ssh on 127.0.0.1:%s as %s\n' "$port" "$SSH_USER"
   # shellcheck disable=SC2207
   args=($(ssh_args "$arch"))
   for _ in $(jot 120 1 120 2>/dev/null || seq 1 120); do
     if ssh "${args[@]}" "$SSH_USER@127.0.0.1" 'uname -a' 2>/dev/null; then
       return 0
     fi
     sleep 2
   done
   die "ssh did not become ready for $arch"
 }
 
 ssh_arch() {
   local arch="$1" args
   shift
   arch="$(canon_arch "$arch")"
   # shellcheck disable=SC2207
   args=($(ssh_args "$arch"))
   exec ssh "${args[@]}" "$SSH_USER@127.0.0.1" "$@"
 }
 
 scp_arch() {
   local arch="$1" src="$2" dst="$3" port args
   arch="$(canon_arch "$arch")"
   port="$(ssh_port "$arch")"
   # shellcheck disable=SC2207
   args=($(ssh_args "$arch"))
   exec scp -P "$port" -i "$SSH_KEY" \
     -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
     "$src" "$SSH_USER@127.0.0.1:$dst"
 }
 
 # run_batch ARCH STAGEDIR RESULTSDIR
 #   Run a whole staging dir in a single SSH session and bring per-binary results
 #   back. Pure transport: tar the stagedir in on stdin, the guest extracts it and
 #   runs its run-remote.sh entry script (authored by the caller; it writes
 #   res/<id>.{rc,out,err} per binary), then tars res/ back on stdout, which we
 #   extract flat into RESULTSDIR. One VM round-trip per arch. The VM must already
 #   be reachable (the caller boots it and waits for SSH).
 run_batch_arch() {
   local arch="$1" stagedir="$2" resultsdir="$3" args
   arch="$(canon_arch "$arch")"
   [ -d "$stagedir" ] || die "run-batch: no such staging dir: $stagedir"
   [ -f "$stagedir/run-remote.sh" ] || die "run-batch: missing $stagedir/run-remote.sh"
   [ -n "$resultsdir" ] || die "run-batch: results dir required"
   command -v tar >/dev/null 2>&1 || die "run-batch: tar not found on host"
   mkdir -p "$resultsdir"
   # shellcheck disable=SC2207
   args=($(ssh_args "$arch"))
   tar -C "$stagedir" -cf - . | ssh "${args[@]}" "$SSH_USER@127.0.0.1" \
     'd=$(mktemp -d /tmp/kit-vm.XXXXXX) || exit 1; tar -C "$d" -xf - || exit 1; ( cd "$d" && sh run-remote.sh ) </dev/null >/dev/null 2>&1; tar -C "$d/res" -cf - . 2>/dev/null; rm -rf "$d"' \
     | tar -C "$resultsdir" -xf - 2>/dev/null || true
 }
 
 doctor() {
   printf 'host: %s/%s\n' "$(uname -s 2>/dev/null)" "$(uname -m 2>/dev/null)"
   printf 'vm root: %s\n' "$VM_ROOT"
   printf 'download cache: %s\n' "$DL_ROOT"
   for tool in curl shasum xz hdiutil ssh-keygen ssh qemu-img \
               qemu-system-x86_64 qemu-system-aarch64 qemu-system-riscv64; do
     if command -v "$tool" >/dev/null 2>&1; then
       printf '  OK      %s (%s)\n' "$tool" "$(command -v "$tool")"
     else
       printf '  MISSING %s\n' "$tool"
     fi
   done
   printf 'ssh key: %s\n' "$([ -f "$SSH_KEY" ] && echo "$SSH_KEY" || echo "$SSH_KEY (absent)")"
   for arch in amd64 aarch64 riscv64; do
     printf '  %-7s cloudinit=%s ssh_port=%s golden=%s\n' \
       "$arch" \
       "$(cloudinit_supported "$arch" && echo yes || echo no)" \
       "$(ssh_port "$arch")" \
       "$([ -f "$(golden_cache_path "$arch")" ] && echo cached || echo none)"
   done
 }
 
 cmd="${1:-}"
 case "$cmd" in
   doctor) doctor ;;
   fetch) [ $# -eq 2 ] || { usage; exit 2; }; fetch_arch "$2" ;;
   prepare) [ $# -eq 2 ] || { usage; exit 2; }; prepare_arch "$2" ;;
   seed) [ $# -eq 2 ] || { usage; exit 2; }; seed_arch "$2" ;;
   provision) [ $# -eq 2 ] || { usage; exit 2; }; provision_arch "$2" ;;
   run) [ $# -ge 2 ] || { usage; exit 2; }; arch="$2"; shift 2; run_arch "$arch" "$@" ;;
   wait-ssh) [ $# -eq 2 ] || { usage; exit 2; }; wait_ssh "$2" ;;
   ssh) [ $# -ge 2 ] || { usage; exit 2; }; arch="$2"; shift 2; ssh_arch "$arch" "$@" ;;
   scp) [ $# -eq 4 ] || { usage; exit 2; }; scp_arch "$2" "$3" "$4" ;;
   run-batch) [ $# -eq 4 ] || { usage; exit 2; }; run_batch_arch "$2" "$3" "$4" ;;
   -h|--help|help|"") usage ;;
   *) usage; exit 2 ;;
 esac