linux_bootstrap.sh (7115B)
1 #!/usr/bin/env bash 2 # Run the three-stage self-build (mk/bootstrap.mk) for aarch64-linux inside a 3 # Linux container, from a non-Linux dev host (e.g. an Apple-silicon Mac). 4 # 5 # The bootstrap is a NATIVE build: `make bootstrap` keys off the container's 6 # own uname (HOST_OS=linux + aarch64), so it selects the aarch64-linux ELF 7 # toolchain and reaches its fixed point entirely inside the container. No 8 # cross-compilation is involved -- the host only supplies podman + the repo. 9 # 10 # We run on the same arm64 Linux container family used by the hosted test 11 # suite: alpine (musl) or debian (glibc). The container needs a seed C 12 # compiler (clang) to build stage1; stages 2 and 3 are built by kit itself. 13 # 14 # usage: scripts/linux_bootstrap.sh [libc] [chain] 15 # libc musl (default, alpine) | glibc (debian) 16 # chain both (default) | debug (-O0) | release (-O1) 17 # 18 # Env overrides: 19 # KIT_LINUX_BOOT_ARCH aa64 (default) | x64 | rv64 — selects --platform + 20 # the default base image; non-host arches run the whole 21 # native 3-stage build under qemu emulation (binfmt). 22 # KIT_LINUX_BOOT_IMAGE container image (defaults per libc+arch, below) 23 # KIT_LINUX_BOOT_PLATFORM podman --platform (default derived from arch) 24 # KIT_LINUX_BOOT_TOY=1 also run the Toy corpus through the stage3 compiler 25 # KIT_LINUX_BOOT_SEED repo-relative path to a pre-built stage1 kit (the 26 # "hybrid" flow). When set, stage1 is this seed instead 27 # of an in-container clang build — so the seed can be 28 # cross-compiled on the host at native speed and only 29 # stages 2/3 run emulated. Intended for rv64, where an 30 # in-container clang stage1 would itself run emulated. 31 # Build one with: scripts/kit_cross.sh <target> --cc=clang 32 # 33 # The stage tree lands under build/linux-boot/<libc>/<arch>/ on the host 34 # (gitignored), so artifacts survive the run for inspection / per-object diffing. 35 36 set -eu 37 38 ROOT="$(cd "$(dirname "$0")/.." && pwd)" 39 LIBC="${1:-musl}" 40 CHAIN="${2:-both}" 41 42 case "${KIT_LINUX_BOOT_ARCH:-aarch64}" in 43 aa64|aarch64|arm64) ARCH=aarch64; DEF_PLATFORM=linux/arm64 ;; 44 x64|x86_64|amd64) ARCH=x86_64; DEF_PLATFORM=linux/amd64 ;; 45 rv64|riscv64) ARCH=riscv64; DEF_PLATFORM=linux/riscv64 ;; 46 *) echo "linux_bootstrap: unknown arch '$KIT_LINUX_BOOT_ARCH' (want aa64|x64|rv64)" >&2; exit 2 ;; 47 esac 48 PLATFORM="${KIT_LINUX_BOOT_PLATFORM:-$DEF_PLATFORM}" 49 50 case "$LIBC" in 51 musl) 52 case "$ARCH" in 53 aarch64) DEF_IMAGE="docker.io/library/alpine:3.23" ;; 54 x86_64) DEF_IMAGE="docker.io/library/alpine:3.23" ;; 55 riscv64) DEF_IMAGE="docker.io/riscv64/alpine:edge" ;; 56 esac ;; 57 glibc) 58 case "$ARCH" in 59 aarch64) DEF_IMAGE="docker.io/arm64v8/debian:bookworm-slim" ;; 60 x86_64) DEF_IMAGE="docker.io/amd64/debian:bookworm-slim" ;; 61 riscv64) DEF_IMAGE="docker.io/riscv64/debian:trixie-slim" ;; 62 esac ;; 63 *) echo "linux_bootstrap: unknown libc '$LIBC' (want musl|glibc)" >&2; exit 2 ;; 64 esac 65 IMAGE="${KIT_LINUX_BOOT_IMAGE:-$DEF_IMAGE}" 66 67 case "$CHAIN" in 68 both) TARGET="bootstrap" ;; 69 debug) TARGET="bootstrap-debug" ;; 70 release) TARGET="bootstrap-release" ;; 71 *) echo "linux_bootstrap: unknown chain '$CHAIN' (want both|debug|release)" >&2; exit 2 ;; 72 esac 73 74 TOY="${KIT_LINUX_BOOT_TOY:-0}" 75 SEED="${KIT_LINUX_BOOT_SEED:-}" 76 BUILD_DIR="build/linux-boot/$LIBC/$ARCH" 77 78 if [ -n "$SEED" ]; then 79 [ -x "$ROOT/$SEED" ] || { echo "linux_bootstrap: seed not found/executable: $ROOT/$SEED (build with scripts/kit_cross.sh)" >&2; exit 2; } 80 fi 81 82 # In-container provisioning + build. The package sets give: a seed clang/lld, 83 # make, the libc dev headers, binutils (ar/ranlib used by the host stage1 84 # link), the ASan/UBSan runtime + unwinder for the -O0 stage1, and perl (the 85 # `shasum` the bootstrap recipe prints with). detect_leaks=0 because kit is 86 # arena-allocated and never frees -- LeakSanitizer (absent on the macOS 87 # reference host, which is why this only bites on Linux) would otherwise abort 88 # every stage1 cc invocation. 89 # When seeded, stage1 is the host-built kit and stages 2/3 are compiled by kit 90 # itself (its own cc/ar/ld), so the container needs no clang/lld/binutils — only 91 # make, the libc dev headers/crt, and perl (shasum). Otherwise install the full 92 # seed-compiler set for the in-container clang stage1. 93 if [ -n "$SEED" ]; then 94 case "$LIBC" in 95 musl) PROVISION='apk add --no-cache make musl-dev perl-utils bash >/dev/null' ;; 96 glibc) PROVISION='export DEBIAN_FRONTEND=noninteractive; apt-get update -qq >/dev/null && apt-get install -y -qq make libc6-dev perl >/dev/null' ;; 97 esac 98 else 99 case "$LIBC" in 100 musl) PROVISION='apk add --no-cache clang lld make musl-dev binutils compiler-rt libgcc perl-utils bash >/dev/null' ;; 101 glibc) PROVISION='export DEBIAN_FRONTEND=noninteractive; apt-get update -qq >/dev/null && apt-get install -y -qq clang lld make libc6-dev binutils perl >/dev/null' ;; 102 esac 103 fi 104 105 # Hybrid (seeded) vs native stage1: a seed bypasses clang entirely and feeds 106 # mk/bootstrap.mk a pre-built stage1. The seeded flow is emulated (a foreign 107 # arch under binfmt), so it builds into container-local storage rather than the 108 # :Z mount: virtiofs/9p occasionally fails a new-file create under sustained 109 # emulated write load (hundreds of objects rapid-fire), which would derail an 110 # otherwise-good build. Source reads from /work stay reliable; only the stage 111 # kit binaries are copied back to the host BUILD_DIR for inspection. 112 if [ -n "$SEED" ]; then 113 EFF_BUILD_DIR="/root/boot" 114 BUILD_CMD="make $TARGET BUILD_DIR='$EFF_BUILD_DIR' BOOTSTRAP_SEED='/work/$SEED'" 115 SEED_BANNER="echo \"stage1 seed: /work/$SEED\"; /work/$SEED --help >/dev/null 2>&1 && echo 'seed runs' || echo 'seed FAILED to run'" 116 SEEDED=1 117 else 118 EFF_BUILD_DIR="$BUILD_DIR" 119 BUILD_CMD="make $TARGET BUILD_DIR='$EFF_BUILD_DIR' CC=clang AR=ar" 120 SEED_BANNER='clang --version | head -1' 121 SEEDED=0 122 fi 123 124 read -r -d '' REMOTE <<EOF || true 125 set -eu 126 $PROVISION 127 cd /work 128 EFF_BUILD_DIR='$EFF_BUILD_DIR' 129 echo "=== linux_bootstrap: \$(uname -m) / $LIBC / $TARGET ===" 130 $SEED_BANNER 131 $BUILD_CMD 132 if [ "$TOY" = "1" ]; then 133 for m in debug release; do 134 s3="\$EFF_BUILD_DIR/\$m/bootstrap/stage3/kit" 135 [ -x "\$s3" ] || continue 136 echo "=== Toy corpus through \$m stage3 ===" 137 KIT="\$(pwd)/\$s3" test/toy/run.sh || true 138 done 139 fi 140 if [ "$SEEDED" = 1 ]; then 141 for m in debug release; do 142 for st in stage2 stage3; do 143 k="\$EFF_BUILD_DIR/\$m/bootstrap/\$st/kit" 144 [ -x "\$k" ] || continue 145 mkdir -p "/work/$BUILD_DIR/\$m/bootstrap/\$st" 146 cp "\$k" "/work/$BUILD_DIR/\$m/bootstrap/\$st/kit" 147 done 148 done 149 echo "=== linux_bootstrap: copied stage kits to $BUILD_DIR ===" 150 fi 151 echo "=== linux_bootstrap: DONE $TARGET ===" 152 EOF 153 154 exec podman run --rm --platform "$PLATFORM" \ 155 -e ASAN_OPTIONS=halt_on_error=1:abort_on_error=1:detect_leaks=0 \ 156 -e UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ 157 -v "$ROOT":/work:Z \ 158 "$IMAGE" sh -c "$REMOTE"