release.sh (18096B)
1 #!/usr/bin/env bash 2 # scripts/release.sh — the multi-target kit release driver. 3 # 4 # This is the matrix counterpart to `make dist`: where `make dist` stages, 5 # packages, and signs the toolchain for the NATIVE host triple only, release.sh 6 # does the same for EVERY hosted (runnable) triple in scripts/hosted.sh's 7 # support set, signs 8 # each artifact with the REAL release key, and finally emits + signs one 9 # `kit-release 1` channel index covering all of them. See doc/plan/SELFDIST.md 10 # ("Release artifacts", "Channel index format", and the scripts/release.sh work 11 # item) and mk/dist.mk for the per-target staging/packaging contract this 12 # mirrors. 13 # 14 # For each hosted target it: 15 # 1. cross-builds the kit binary for the target (scripts/kit_cross.sh) 16 # 2. builds the per-target rt archive(s) (make rt-<variant>, target codegen) 17 # 3. stages the same self-contained tree `make dist` stages 18 # 4. packages a signed fat .kpkg, a signed portable .tar.gz, and a detached 19 # .tar.gz.minisig (the stock-minisign bootstrap path) 20 # then assembles one signed channel index over the whole matrix. 21 # 22 # Freestanding targets have no OS to run on, so they produce NO kit release 23 # artifact (SELFDIST.md: "the release set is the hosted triples only"); they are 24 # excluded by design. 25 # 26 # Prerequisites 27 # ------------- 28 # * A REAL signing key in $KIT_SIGN_KEY (see below). A release is signed with 29 # the real release secret or not at all. 30 # * Cross-build sysroots for every foreign-libc target, provisioned once with 31 # `make provision TARGET=<selector>` (doc/plan/SYSROOTS.md). kit_cross.sh 32 # errors (does not skip) when a requested target's sysroot is missing. 33 # * For the VM-validated targets (freebsd/windows) the usual VM/exec infra — 34 # drop them with KIT_VM=0 (below) on a VM-less CI runner. 35 # 36 # Environment knobs 37 # ----------------- 38 # KIT_SIGN_KEY (required) path to the real release minisign secret key. 39 # KIT_RELEASE_PUBKEYS (required) one or more production Minisign public-key 40 # files to embed in every released `kit update` binary. 41 # KIT_UPDATE_INDEX_URL 42 # (required) production stable channel-index URL compiled 43 # into every released binary. 44 # KIT_RELEASE_ALLOW_TEST_KEY 45 # (optional, default 0) set to 1 only for hermetic tests 46 # that intentionally sign with the in-tree non-release key. 47 # KIT_RELEASE_URL_BASE (optional) one or more space-separated mirror base URLs 48 # for the channel index `url` lines. Each artifact's URL 49 # is "<base>/<filename>". Defaults to a GitHub-Releases- 50 # style base: https://github.com/<org>/kit/releases/ 51 # download/v<VERSION>. Repeat (space-separated) for an 52 # ordered mirror list. 53 # KIT_RELEASE_TARGETS (optional) hosted.sh selector for the target set 54 # (default: all). Freestanding tokens are always dropped. 55 # KIT_VM (optional, default 1) 0 drops the VM-only targets 56 # (freebsd/windows) for a VM-less CI runner, same meaning 57 # as in scripts/hosted.sh. 58 # KIT_CHANNEL (optional, default "stable") channel name for the index 59 # and the index file basename (<channel>.index). 60 # KIT_RELEASE_OUT_DIR (optional, default build/release-dist) output directory 61 # for artifacts, staging trees, and the channel index. 62 # KIT (optional) path to a prebuilt native kit to use as the 63 # packaging tool (default: the build/release/kit produced 64 # by `make RELEASE=1 bin`). 65 66 set -euo pipefail 67 68 ROOT="$(cd "$(dirname "$0")/.." && pwd)" 69 cd "$ROOT" 70 HOSTED="$ROOT/scripts/hosted.sh" 71 72 die() { printf 'release: %s\n' "$*" >&2; exit 1; } 73 log() { printf 'release: %s\n' "$*" >&2; } 74 75 # ---- preconditions --------------------------------------------------------- 76 # A real signing key is mandatory; releasing with the test key is never valid. 77 [ -n "${KIT_SIGN_KEY:-}" ] || die \ 78 "KIT_SIGN_KEY is unset — set it to the real release minisign secret key" 79 [ -f "$KIT_SIGN_KEY" ] || die "KIT_SIGN_KEY=$KIT_SIGN_KEY is not a file" 80 [ -n "${KIT_RELEASE_PUBKEYS:-}" ] || die \ 81 "KIT_RELEASE_PUBKEYS is unset — provide production Minisign public-key file(s)" 82 for release_pubkey in $KIT_RELEASE_PUBKEYS; do 83 [ -f "$release_pubkey" ] || die \ 84 "KIT_RELEASE_PUBKEYS entry is not a file: $release_pubkey" 85 done 86 [ -n "${KIT_UPDATE_INDEX_URL:-}" ] || die \ 87 "KIT_UPDATE_INDEX_URL is unset — provide the production stable channel URL" 88 TEST_SIGN_KEY="$ROOT/test/dist/keys/nonrelease.key" 89 TEST_PUBKEY="$ROOT/test/dist/keys/nonrelease.pub" 90 TEST_SIGN_ID="$(sh "$ROOT/scripts/minisign_key_id.sh" "$TEST_SIGN_KEY")" 91 TEST_PUB_ID="$(sh "$ROOT/scripts/minisign_key_id.sh" "$TEST_PUBKEY")" 92 if [ "${KIT_RELEASE_ALLOW_TEST_KEY:-0}" != 1 ]; then 93 [ "$(sh "$ROOT/scripts/minisign_key_id.sh" "$KIT_SIGN_KEY")" != \ 94 "$TEST_SIGN_ID" ] || 95 die "KIT_SIGN_KEY contains the repository NON-RELEASE test key; set KIT_SIGN_KEY to the real release key" 96 for release_pubkey in $KIT_RELEASE_PUBKEYS; do 97 [ "$(sh "$ROOT/scripts/minisign_key_id.sh" "$release_pubkey")" != \ 98 "$TEST_PUB_ID" ] || 99 die "KIT_RELEASE_PUBKEYS contains the repository NON-RELEASE test anchor: $release_pubkey" 100 done 101 fi 102 103 [ -f "$ROOT/VERSION" ] || die "no VERSION file at repo root" 104 VERSION="$(tr -d ' \t\r\n' < "$ROOT/VERSION")" 105 [ -n "$VERSION" ] || die "VERSION file is empty" 106 107 # Release builds assert a clean git tree (SELFDIST.md reproducibility contract): 108 # the embedded build id must not be a "-dirty" hash, and the staged sources must 109 # match the committed tree. Override only for a deliberate dry run. 110 if [ "${KIT_RELEASE_ALLOW_DIRTY:-0}" != 1 ]; then 111 if ! git -C "$ROOT" diff --quiet HEAD 2>/dev/null; then 112 die "git tree is dirty — commit/stash first, or set KIT_RELEASE_ALLOW_DIRTY=1 for a dry run" 113 fi 114 fi 115 116 CHANNEL="${KIT_CHANNEL:-stable}" 117 118 # Default mirror base: GitHub-Releases-style download URL for this version. Can 119 # be overridden / extended (space-separated) via KIT_RELEASE_URL_BASE. 120 DEFAULT_URL_BASE="https://github.com/kit/kit/releases/download/v$VERSION" 121 URL_BASES="${KIT_RELEASE_URL_BASE:-$DEFAULT_URL_BASE}" 122 123 OUT_DIR="${KIT_RELEASE_OUT_DIR:-$ROOT/build/release-dist}" 124 INDEX_FILE="$OUT_DIR/$CHANNEL.index" 125 126 # ---- the native packaging tool --------------------------------------------- 127 # The packaging steps (kit pkg create / sign) run on the HOST, so they need a 128 # host-runnable kit. Build the release native binary first and use it as the 129 # packaging tool. (Mirrors mk/dist.mk, which packages with build/release/kit.) 130 log "building native bootstrap kit (make RELEASE=1 bin)" 131 make RELEASE=1 \ 132 KIT_UPDATE_INDEX_URL="$KIT_UPDATE_INDEX_URL" \ 133 KIT_RELEASE_PUBKEYS="$KIT_RELEASE_PUBKEYS" \ 134 KIT_RELEASE_ALLOW_TEST_KEY="${KIT_RELEASE_ALLOW_TEST_KEY:-0}" bin 135 KIT="${KIT:-$ROOT/build/release/kit}" 136 [ -x "$KIT" ] || die "native kit not found at $KIT after 'make RELEASE=1 bin'" 137 138 # ---- target set ------------------------------------------------------------ 139 # The release set is the hosted (runnable) triples: every target in the support 140 # set that has an OS to run on. scripts/hosted.sh's `expand --mode=cross` is the 141 # cross-buildable set (includes freestanding + non-host-arch macOS + android); 142 # we drop the freestanding tokens, which by design ship no kit artifact. KIT_VM 143 # is honored by hosted.sh itself (drops freebsd/windows when 0). 144 SELECTOR="${KIT_RELEASE_TARGETS:-all}" 145 TARGETS=() 146 while IFS= read -r tok; do 147 [ -n "$tok" ] || continue 148 case "$tok" in 149 freestanding-*) continue ;; # no host to run on → no release artifact 150 esac 151 TARGETS+=("$tok") 152 done < <(KIT_VM="${KIT_VM:-1}" bash "$HOSTED" expand "$SELECTOR" --mode=cross) 153 154 [ "${#TARGETS[@]}" -gt 0 ] || die "no hosted targets selected (selector='$SELECTOR', KIT_VM=${KIT_VM:-1})" 155 156 log "version=$VERSION channel=$CHANNEL targets=${TARGETS[*]}" 157 158 # ---- per-target rt variant mapping ----------------------------------------- 159 # Map a hosted token to the mk/rt.mk RT_VARIANTS entry whose target triple 160 # matches. The rt variant set is keyed by (arch,os) and is libc-agnostic on 161 # Linux (musl and glibc share the *-linux rt), so both linux libcs collapse to 162 # the same variant. Windows x64's rt variant is the msvc-ABI `x86_64-pc-windows` 163 # (the only x86_64 windows rt in the table); Android maps to the dedicated 164 # `aarch64-linux-android` variant. (Source of truth: mk/rt.mk RT_VARIANTS.) 165 rt_variant_of() { 166 case "$1" in 167 linux-glibc-aa64|linux-musl-aa64) echo aarch64-linux ;; 168 linux-glibc-x64|linux-musl-x64) echo x86_64-linux ;; 169 linux-glibc-rv64|linux-musl-rv64) echo riscv64-linux ;; 170 freebsd-aa64) echo aarch64-freebsd ;; 171 freebsd-x64) echo x86_64-freebsd ;; 172 freebsd-rv64) echo riscv64-freebsd ;; 173 windows-aa64) echo aarch64-windows ;; 174 windows-x64) echo x86_64-pc-windows ;; 175 android-aa64) echo aarch64-linux-android ;; 176 macos-aa64) echo aarch64-apple-darwin ;; 177 macos-x64) echo x86_64-apple-darwin ;; 178 *) die "no rt variant mapping for token '$1' (extend rt_variant_of + mk/rt.mk)" ;; 179 esac 180 } 181 182 # ---- output staging -------------------------------------------------------- 183 rm -rf "$OUT_DIR" 184 mkdir -p "$OUT_DIR" 185 186 # Accumulate one index "[host]" block per target into a temp dir, one file per 187 # target named by triple, so the final index can be assembled in sorted (i.e. 188 # canonical, strictly-ascending-by-target) order regardless of build order. 189 BLOCK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/kit-release-blocks.XXXXXX")" 190 cleanup() { rm -rf "$BLOCK_DIR"; } 191 trap cleanup EXIT 192 193 # Track artifacts produced, for the closing summary. 194 SUMMARY=() 195 196 # ---- per-target build/stage/package ---------------------------------------- 197 build_target() { 198 local token="$1" 199 local triple sysroot rt_variant 200 triple="$(bash "$HOSTED" triple "$token")" || die "bad target '$token'" 201 sysroot="$(bash "$HOSTED" path "$token" 2>/dev/null || true)" 202 rt_variant="$(rt_variant_of "$token")" 203 204 # macOS has no cross sysroot — compile against the host SDK. 205 local os="${token%%-*}" 206 if [ "$os" = macos ] && [ -z "$sysroot" ]; then 207 sysroot="$(xcrun --show-sdk-path 2>/dev/null || true)" 208 fi 209 210 log "=== $token (triple=$triple rt=$rt_variant) ===" 211 212 # 1) Cross-build the kit binary for the target. kit_cross.sh wipes its own 213 # build dir per run and writes the binary to build/kit-cross/kit/<token>/ 214 # (kit.exe for windows), so capture it immediately into the per-target 215 # staging tree before any later run can clobber it. 216 log "cross-building kit binary" 217 KIT_UPDATE_INDEX_URL="$KIT_UPDATE_INDEX_URL" \ 218 KIT_RELEASE_PUBKEYS="$KIT_RELEASE_PUBKEYS" \ 219 KIT_RELEASE_ALLOW_TEST_KEY="${KIT_RELEASE_ALLOW_TEST_KEY:-0}" \ 220 bash "$ROOT/scripts/kit_cross.sh" "$token" --cc=kit 221 local cross_dir="$ROOT/build/kit-cross/kit/$token" 222 local cross_bin bin_name 223 if [ "$os" = windows ]; then 224 cross_bin="$cross_dir/kit.exe"; bin_name="kit.exe" 225 else 226 cross_bin="$cross_dir/kit"; bin_name="kit" 227 fi 228 [ -x "$cross_bin" ] || die "$token: cross-built kit missing at $cross_bin" 229 230 # 2) Build the per-target rt archive(s). These are TARGET objects, so they 231 # must be produced by a HOST-runnable kit emitting target code — i.e. the 232 # native packaging kit driven with -target/--sysroot — NOT the just-built 233 # foreign binary (which cannot execute on the build host). This matches 234 # mk/dist.mk, where the rt is built by the native `$(BIN) cc`. The cross 235 # binary is the SHIPPED compiler; this host kit is only the BUILD tool. 236 # Each target gets an isolated RT_BUILD_DIR so per-arch objects never mix. 237 log "building rt variant $rt_variant" 238 local rt_build_dir="$ROOT/build/release-rt/$token" 239 local rt_tool_dir="$OUT_DIR/toolchain/$token" 240 rm -rf "$rt_build_dir" 241 rm -rf "$rt_tool_dir" 242 mkdir -p "$rt_tool_dir" 243 if [ -n "$sysroot" ]; then 244 cat > "$rt_tool_dir/cc" <<EOF 245 #!/bin/sh 246 exec "$KIT" cc -target "$triple" --sysroot "$sysroot" "\$@" 247 EOF 248 else 249 cat > "$rt_tool_dir/cc" <<EOF 250 #!/bin/sh 251 exec "$KIT" cc -target "$triple" "\$@" 252 EOF 253 fi 254 cat > "$rt_tool_dir/ar" <<EOF 255 #!/bin/sh 256 exec "$KIT" ar "\$@" 257 EOF 258 cat > "$rt_tool_dir/as" <<EOF 259 #!/bin/sh 260 exec "$KIT" as "\$@" 261 EOF 262 chmod +x "$rt_tool_dir/cc" "$rt_tool_dir/ar" "$rt_tool_dir/as" 263 make "rt-$rt_variant" \ 264 BIN="$KIT" \ 265 RT_BUILD_DIR="$rt_build_dir" \ 266 RT_CC="$rt_tool_dir/cc" \ 267 RT_AR="$rt_tool_dir/ar" \ 268 RT_AS="$rt_tool_dir/as" 269 local rt_archive="$rt_build_dir/$rt_variant/libkit_rt.a" 270 [ -f "$rt_archive" ] || die "$token: rt archive missing at $rt_archive" 271 272 # 3) Stage the self-contained tree, exactly as mk/dist.mk does: bin/kit[.exe], 273 # lib/libkit.a, include/, support/rt/{include,lib}, VERSION, README/license. 274 # libkit.a is the TARGET-arch static lib that kit_cross.sh produced next to 275 # the cross binary (its `make bin BUILD_DIR=$cross_dir` leaves both there), 276 # so the shipped lib matches the shipped binary. The rt archive is staged 277 # under support/rt/lib/ next to the rt sources, named for the target so a 278 # multi-arch tree stays unambiguous. 279 log "staging tree" 280 local stage="$OUT_DIR/stage/$triple" 281 local cross_lib="$cross_dir/libkit.a" 282 [ -f "$cross_lib" ] || die "$token: cross-built libkit.a missing at $cross_lib" 283 rm -rf "$stage" 284 mkdir -p "$stage/bin" "$stage/lib" "$stage/support/rt" 285 cp "$cross_bin" "$stage/bin/$bin_name" 286 cp "$cross_lib" "$stage/lib/libkit.a" 287 cp -r "$ROOT/include" "$stage/include" 288 cp -r "$ROOT/rt/include" "$stage/support/rt/include" 289 cp -r "$ROOT/rt/lib" "$stage/support/rt/lib" 290 cp "$ROOT/VERSION" "$stage/VERSION" 291 cp "$rt_archive" "$stage/support/rt/lib/libkit_rt-$rt_variant.a" 292 cp "$ROOT/README.md" "$stage/README.md" 293 sh "$ROOT/scripts/stage_docs.sh" "$ROOT" "$stage/doc" 294 local f 295 for f in LICENSE LICENSE.txt LICENSE.md NOTICE NOTICE.txt; do 296 [ -f "$ROOT/$f" ] && cp "$ROOT/$f" "$stage/$f" 297 done 298 : # keep set -e happy after the optional copies 299 300 # 4) Package + sign, mirroring mk/dist.mk's invocations (real key, no test-key 301 # fallback). Capture the package-id (`id <64hex>`) printed by pkg create. 302 local base="kit-$VERSION-$triple" 303 local kpkg="$OUT_DIR/$base.kpkg" 304 local targz="$OUT_DIR/$base.tar.gz" 305 306 log "packaging $base.kpkg (fat)" 307 local kpkg_out kpkg_id 308 kpkg_out="$("$KIT" pkg create --name kit --version "$VERSION" \ 309 --format kpkg --native-shape fat -s "$KIT_SIGN_KEY" \ 310 --root "$stage" -o "$kpkg")" 311 printf '%s\n' "$kpkg_out" >&2 312 kpkg_id="$(printf '%s\n' "$kpkg_out" | sed -n 's/.*id \([0-9a-f]\{64\}\).*/\1/p')" 313 [ -n "$kpkg_id" ] || die "$token: could not parse kpkg package-id from pkg create output" 314 315 log "packaging $base.tar.gz (portable)" 316 local targz_out targz_id 317 targz_out="$("$KIT" pkg create --name kit --version "$VERSION" \ 318 --format tar.gz -s "$KIT_SIGN_KEY" \ 319 --root "$stage" -o "$targz")" 320 printf '%s\n' "$targz_out" >&2 321 targz_id="$(printf '%s\n' "$targz_out" | sed -n 's/.*id \([0-9a-f]\{64\}\).*/\1/p')" 322 [ -n "$targz_id" ] || die "$token: could not parse tar.gz package-id from pkg create output" 323 324 log "signing $base.tar.gz.minisig (detached, bootstrap path)" 325 "$KIT" pkg sign -s "$KIT_SIGN_KEY" \ 326 -o "$targz.minisig" --comment "kit $VERSION $triple" \ 327 "$targz" 328 329 # kpkg byte size, for the index `size` field. 330 local kpkg_size 331 kpkg_size="$(wc -c < "$kpkg" | tr -d ' ')" 332 333 # 5) Emit this target's index [host] block into the sort accumulator. The 334 # block file is named by triple so the final assembly sorts canonically. 335 # Index field order matches test/api/release_index_test.c's golden: 336 # target, kpkg, targz, size, then one url line per mirror base. 337 { 338 printf '[host]\n' 339 printf 'target = %s\n' "$triple" 340 printf 'kpkg = %s\n' "$kpkg_id" 341 printf 'targz = %s\n' "$targz_id" 342 printf 'size = %s\n' "$kpkg_size" 343 for base_url in $URL_BASES; do 344 printf 'url = %s/%s.kpkg\n' "${base_url%/}" "$base" 345 done 346 } > "$BLOCK_DIR/$triple" 347 348 SUMMARY+=("$triple: kpkg=$kpkg ($kpkg_size bytes, id ${kpkg_id:0:12}…) targz=$targz minisig=$targz.minisig") 349 } 350 351 for token in "${TARGETS[@]}"; do 352 build_target "$token" 353 done 354 355 # ---- channel index --------------------------------------------------------- 356 # Assemble the canonical `kit-release 1` index: a fixed header, a blank line, 357 # then the [host] blocks sorted ASCENDING by target triple (so the file is 358 # byte-stable and parseable by kit_release_index_parse, which requires strictly 359 # ascending unique targets). Blocks are separated by a blank line. The block 360 # files are named by triple, so `sort` over the directory listing yields the 361 # canonical order. 362 log "emitting channel index $INDEX_FILE" 363 { 364 printf 'kit-release 1\n' 365 printf 'channel = %s\n' "$CHANNEL" 366 printf 'version = %s\n' "$VERSION" 367 printf 'hash = blake2b-256\n' 368 first=1 369 while IFS= read -r triple; do 370 [ -n "$triple" ] || continue 371 printf '\n' # blank line before each [host] block 372 cat "$BLOCK_DIR/$triple" 373 first=0 374 done < <(find "$BLOCK_DIR" -type f -exec basename {} \; | LC_ALL=C sort) 375 : "$first" 376 } > "$INDEX_FILE" 377 378 log "signing channel index $INDEX_FILE.minisig" 379 "$KIT" pkg sign -s "$KIT_SIGN_KEY" \ 380 -o "$INDEX_FILE.minisig" --comment "kit-release $CHANNEL $VERSION" \ 381 "$INDEX_FILE" 382 383 # ---- summary --------------------------------------------------------------- 384 printf '\nrelease: %s channel %s, version %s — %d target(s)\n' \ 385 "DONE" "$CHANNEL" "$VERSION" "${#TARGETS[@]}" >&2 386 for line in "${SUMMARY[@]}"; do 387 printf 'release: %s\n' "$line" >&2 388 done 389 printf 'release: index=%s\n' "$INDEX_FILE" >&2 390 printf 'release: index.sig=%s\n' "$INDEX_FILE.minisig" >&2 391 printf 'release: all artifacts under %s\n' "$OUT_DIR" >&2