lib-pipeline.sh (11221B)
1 # lib-pipeline.sh — driver-agnostic DSL for boot stage pipelines. 2 # 3 # A bootN.sh's "wiring" is a sequence of file→file program invocations 4 # in a flat namespace. This library exposes that as four primitives so 5 # the same wiring can run under different transports: 6 # 7 # podman — accumulate stages into one /work/run.sh, run once in a 8 # container against $IMAGE / $PLATFORM (env-set by caller). 9 # seed — run each stage as one qemu boot of seed-kernel via 10 # tier1-gate.sh's pattern (cpio /init + in/<inputs> on 11 # virtio-blk hd0, output dumped to virtio-blk hd1 as SEEDFS, 12 # extract). aarch64 only. 13 # 14 # Both drivers respect the `in/`+`out/` convention: inputs read from 15 # `in/<name>`, outputs written to `out/<name>`. The stage primitive 16 # rewrites argv tokens that match input/output names with the 17 # appropriate prefix; bare flag/literal tokens pass through untouched. 18 # 19 # DSL (source as `. boot/lib-pipeline.sh`): 20 # 21 # pipeline_init <staging-dir> <out-dir> <driver> 22 # pipeline_input <name> <host-path> # repeatable 23 # pipeline_input_from_src <subpath> [<name>] # from build/$ARCH/src/src/ 24 # stage <bin> <argv...> -- <inputs...> -- <outputs...> 25 # pipeline_export <name>... # one or more 26 # pipeline_run 27 # 28 # `stage` semantics: invoke `<bin>` with argv=[<bin>, <argv1>, ...]; the 29 # stage reads the listed input names and produces the listed output 30 # names. <bin> is also a name in the flat namespace — typically a 31 # pipeline_input, but may be the output of an earlier stage. 32 # 33 # Required env for podman driver: PLATFORM, IMAGE. 34 # Required env for seed driver: KERNEL_IMAGE, EXTRACT. 35 36 P_DRIVER= 37 P_STAGE_DIR= 38 P_OUT_DIR= 39 P_SCRIPT= 40 P_IDX=0 41 P_EXPORTS= 42 P_INPUT_NAMES= 43 P_PRODUCED_NAMES= 44 45 pipeline_init() { 46 P_STAGE_DIR=$1; P_OUT_DIR=$2; P_DRIVER=$3 47 rm -rf "$P_STAGE_DIR" 48 mkdir -p "$P_STAGE_DIR/in" "$P_STAGE_DIR/out" "$P_OUT_DIR" 49 P_IDX=0 50 P_EXPORTS= 51 P_INPUT_NAMES= 52 P_PRODUCED_NAMES= 53 case "$P_DRIVER" in 54 podman) 55 P_SCRIPT=$P_STAGE_DIR/run.sh 56 { 57 echo '#!/bin/sh' 58 echo 'set -eu' 59 # Stage everything in /tmp (RAM tmpfs) — the seed-stage tools 60 # do one syscall per byte, virtiofs round-trips would dominate. 61 # Mirror the in/ + out/ split so argv references resolve. 62 echo 'mkdir -p /tmp/in /tmp/out' 63 echo 'cp /work/in/* /tmp/in/' 64 echo 'cd /tmp' 65 } > "$P_SCRIPT" 66 ;; 67 seed) 68 mkdir -p "$P_STAGE_DIR/work" 69 : "${KERNEL_IMAGE:?lib-pipeline:seed: KERNEL_IMAGE not set}" 70 : "${EXTRACT:?lib-pipeline:seed: EXTRACT not set}" 71 ;; 72 *) 73 echo "lib-pipeline: unknown driver '$P_DRIVER'" >&2; exit 2 ;; 74 esac 75 } 76 77 pipeline_input() { 78 name=$1; src=$2 79 cp "$src" "$P_STAGE_DIR/in/$name" 80 if [ "$P_DRIVER" = "seed" ]; then 81 cp "$src" "$P_STAGE_DIR/work/$name" 82 fi 83 P_INPUT_NAMES="$P_INPUT_NAMES $name" 84 } 85 86 # pipeline_input_from_src <subpath> [<name>] 87 # Pull a file from the canonical generated source tree at 88 # build/$ARCH/src/src/<subpath>. Stages it under in/<name>; 89 # <name> defaults to basename(subpath). For the rare `bin/` case 90 # (the seed hex0-seed binary), call pipeline_input directly with 91 # build/$ARCH/src/bin/<file>. 92 pipeline_input_from_src() { 93 _subpath=$1; _name=${2:-} 94 [ -n "$_name" ] || _name=$(basename "$_subpath") 95 pipeline_input "$_name" "build/$ARCH/src/src/$_subpath" 96 } 97 98 # Look up a token: if it names an input, prefix `in/`; if it names a 99 # previously produced output, prefix `out/`; else leave unchanged. 100 _p_lookup() { 101 tok=$1 102 for n in $P_IN; do [ "$tok" = "$n" ] && { echo "in/$tok"; return; }; done 103 for n in $P_OUT; do [ "$tok" = "$n" ] && { echo "out/$tok"; return; }; done 104 echo "$tok" 105 } 106 107 # Resolve where the bin binary lives: in/ if it's a pipeline_input, out/ 108 # if a prior stage produced it. Stages with the same name as both an 109 # input and a produced output use the produced one. 110 _p_bin_path() { 111 b=$1 112 for n in $P_PRODUCED_NAMES; do [ "$b" = "$n" ] && { echo "out/$b"; return; }; done 113 for n in $P_INPUT_NAMES; do [ "$b" = "$n" ] && { echo "in/$b"; return; }; done 114 echo "$b" 115 } 116 117 # stage <bin> <argv...> -- <inputs...> -- <outputs...> 118 # 119 # The explicit input/output lists look redundant — most names already 120 # appear in <argv...> — but they are not. argv positions are tool- 121 # specific: a token like `M0.combined.hex2` is an output of one stage 122 # (catm produces it) and an input of the next (hex2 reads it). The 123 # framework cannot tell which from the token alone, so each stage 124 # declares both lists. Don't try to "simplify" by inferring from argv. 125 stage() { 126 bin=$1; shift 127 P_HEAD_RAW=""; P_IN=""; P_OUT=""; _s=head 128 while [ $# -gt 0 ]; do 129 if [ "$1" = "--" ]; then 130 case "$_s" in 131 head) _s=in ;; 132 in) _s=out ;; 133 *) echo "lib-pipeline: too many --" >&2; exit 2 ;; 134 esac 135 shift; continue 136 fi 137 case "$_s" in 138 head) P_HEAD_RAW="$P_HEAD_RAW $1" ;; 139 in) P_IN="$P_IN $1" ;; 140 out) P_OUT="$P_OUT $1" ;; 141 esac 142 shift 143 done 144 [ "$_s" = "out" ] || { echo "lib-pipeline: stage needs '<bin> argv... -- inputs... -- outputs...'" >&2; exit 2; } 145 146 # Rewrite head tokens with in/ or out/ prefixes. 147 P_HEAD="" 148 for tok in $P_HEAD_RAW; do 149 P_HEAD="$P_HEAD $(_p_lookup "$tok")" 150 done 151 P_BIN_PATH=$(_p_bin_path "$bin") 152 153 P_IDX=$((P_IDX + 1)) 154 case "$P_DRIVER" in 155 podman) _stage_podman ;; 156 seed) _stage_seed ;; 157 esac 158 159 # Track produced names so later stages can locate the binary if a 160 # subsequent `stage` uses one of these as its bin. 161 for o in $P_OUT; do P_PRODUCED_NAMES="$P_PRODUCED_NAMES $o"; done 162 } 163 164 _stage_podman() { 165 { 166 echo "# stage $P_IDX: $bin$P_HEAD" 167 echo "chmod +x ./$P_BIN_PATH" 168 echo "./$P_BIN_PATH$P_HEAD" 169 # Mirror this stage's outputs back into in/ so a later stage that 170 # declares one of them as an input finds it under in/<name>. 171 # (The seed driver does this naturally via its per-stage cpio.) 172 for o in $P_OUT; do 173 echo "cp -f out/$o in/$o" 174 done 175 } >> "$P_SCRIPT" 176 } 177 178 _stage_seed() { 179 cpio_dir=$P_STAGE_DIR/s$(printf '%02d' "$P_IDX") 180 rm -rf "$cpio_dir"; mkdir -p "$cpio_dir/cpio/in" 181 cp "$P_STAGE_DIR/work/$bin" "$cpio_dir/cpio/init" 182 chmod +x "$cpio_dir/cpio/init" 183 NAMES="init" 184 for inp in $P_IN; do 185 cp "$P_STAGE_DIR/work/$inp" "$cpio_dir/cpio/in/$inp" 186 NAMES="$NAMES 187 in/$inp" 188 done 189 ( cd "$cpio_dir/cpio" && printf '%s\n' "$NAMES" | cpio -o -H newc 2>/dev/null ) > "$cpio_dir/initramfs.cpio" 190 sz=$(wc -c < "$cpio_dir/initramfs.cpio") 191 pad=$(( (512 - sz % 512) % 512 )) 192 if [ "$pad" -gt 0 ]; then 193 head -c "$pad" /dev/zero >> "$cpio_dir/initramfs.cpio" 194 fi 195 mv "$cpio_dir/initramfs.cpio" "$cpio_dir/in.img" 196 truncate -s 256M "$cpio_dir/out.img" 197 198 APPEND="$bin$P_HEAD" 199 TRANSCRIPT=$cpio_dir/transcript.txt 200 echo "[lib-pipeline:seed] stage $P_IDX:$P_HEAD (bin=$bin)" >&2 201 seed_arch=${SEED_ARCH:-aarch64} 202 case "$seed_arch" in 203 aarch64) 204 qemu-system-aarch64 \ 205 -machine virt,gic-version=3,accel=hvf -cpu host -m 2048M \ 206 -nographic -no-reboot \ 207 -global virtio-mmio.force-legacy=false \ 208 -kernel "$KERNEL_IMAGE" \ 209 -drive file="$cpio_dir/in.img",if=none,format=raw,id=hd0,readonly=on \ 210 -device virtio-blk-device,drive=hd0 \ 211 -drive file="$cpio_dir/out.img",if=none,format=raw,id=hd1 \ 212 -device virtio-blk-device,drive=hd1 \ 213 -append "$APPEND" \ 214 > "$TRANSCRIPT" 2>&1 & 215 ;; 216 riscv64) 217 qemu-system-riscv64 \ 218 -machine virt -m 2048M \ 219 -nographic -no-reboot \ 220 -global virtio-mmio.force-legacy=false \ 221 -kernel "$KERNEL_IMAGE" \ 222 -drive file="$cpio_dir/in.img",if=none,format=raw,id=hd0,readonly=on \ 223 -device virtio-blk-device,drive=hd0 \ 224 -drive file="$cpio_dir/out.img",if=none,format=raw,id=hd1 \ 225 -device virtio-blk-device,drive=hd1 \ 226 -append "$APPEND" \ 227 > "$TRANSCRIPT" 2>&1 & 228 ;; 229 amd64) 230 qemu-system-x86_64 \ 231 -machine microvm,acpi=off,pic=off,pit=off,rtc=off,isa-serial=on,auto-kernel-cmdline=off \ 232 -cpu max -m 2048M \ 233 -nodefaults -display none -serial stdio -no-reboot \ 234 -global virtio-mmio.force-legacy=false \ 235 -device isa-debug-exit,iobase=0x501,iosize=2 \ 236 -kernel "$KERNEL_IMAGE" \ 237 -drive file="$cpio_dir/in.img",if=none,format=raw,id=hd0,readonly=on \ 238 -device virtio-blk-device,drive=hd0 \ 239 -drive file="$cpio_dir/out.img",if=none,format=raw,id=hd1 \ 240 -device virtio-blk-device,drive=hd1 \ 241 -append "$APPEND" \ 242 > "$TRANSCRIPT" 2>&1 & 243 ;; 244 *) echo "[lib-pipeline:seed] unsupported SEED_ARCH=$seed_arch" >&2; exit 2 ;; 245 esac 246 QPID=$! 247 ( sleep 240; kill -9 $QPID 2>/dev/null ) </dev/null >/dev/null 2>&1 & 248 WATCHER=$! 249 disown $WATCHER 2>/dev/null || true 250 wait $QPID 2>/dev/null || true 251 kill $WATCHER 2>/dev/null || true 252 253 mkdir -p "$cpio_dir/dump" 254 if ! "$EXTRACT" "$cpio_dir/dump" "$cpio_dir/out.img" >/dev/null 2>&1; then 255 echo "[lib-pipeline:seed] FAIL stage $P_IDX (bin=$bin): extract-blk failed" >&2 256 tail -40 "$TRANSCRIPT" >&2 257 exit 3 258 fi 259 260 for o in $P_OUT; do 261 if [ ! -f "$cpio_dir/dump/$o" ]; then 262 echo "[lib-pipeline:seed] FAIL stage $P_IDX: missing output '$o'" >&2 263 ls "$cpio_dir/dump" >&2 || true 264 exit 3 265 fi 266 cp "$cpio_dir/dump/$o" "$P_STAGE_DIR/work/$o" 267 done 268 } 269 270 pipeline_export() { 271 for _n in "$@"; do P_EXPORTS="$P_EXPORTS $_n"; done 272 } 273 274 pipeline_run() { 275 case "$P_DRIVER" in 276 podman) _run_podman ;; 277 seed) : ;; 278 esac 279 for n in $P_EXPORTS; do 280 case "$P_DRIVER" in 281 podman) cp "$P_STAGE_DIR/out/$n" "$P_OUT_DIR/$n" ;; 282 seed) cp "$P_STAGE_DIR/work/$n" "$P_OUT_DIR/$n" ;; 283 esac 284 chmod 0700 "$P_OUT_DIR/$n" 285 done 286 } 287 288 _run_podman() { 289 : "${PLATFORM:?lib-pipeline:podman: PLATFORM not set}" 290 : "${IMAGE:?lib-pipeline:podman: IMAGE not set}" 291 if [ -n "$P_EXPORTS" ]; then 292 cmd="cp" 293 for n in $P_EXPORTS; do cmd="$cmd out/$n"; done 294 cmd="$cmd /work/out/" 295 echo "$cmd" >> "$P_SCRIPT" 296 fi 297 chmod +x "$P_SCRIPT" 298 SDIR=$(cd "$P_STAGE_DIR" && pwd) 299 podman run --rm -i --pull=never --platform "$PLATFORM" \ 300 --tmpfs /tmp:size=512M \ 301 -v "$SDIR/run.sh:/work/run.sh:ro" \ 302 -v "$SDIR/in:/work/in:ro" \ 303 -v "$SDIR/out:/work/out:rw" \ 304 -w /work "$IMAGE" \ 305 sh -eu /work/run.sh 306 }