run.sh (8904B)
1 #!/usr/bin/env bash 2 # test/ecosystem/run.sh — the C ecosystem gate. 3 # 4 # Builds real-world C projects (SQLite, Lua, cJSON, LZ4, miniz, tinyexpr, ...) 5 # from pinned upstream sources with kit: compile each translation unit, archive 6 # into a static library with `kit ar`, link a driver that exercises real 7 # functionality, run it, and assert the output two ways: 8 # 9 # <name>:<opt>:golden kit output == checked-in expected/<name>.txt 10 # <name>:<opt>:vs-clang kit output == output of the SAME program built with 11 # clang on this host (differential oracle) 12 # 13 # Sources come from the kit cache dir, provisioned offline first via 14 # scripts/ecosystem.sh fetch all (or: make provision-ecosystem) 15 # Projects that are not provisioned are SKIPPED with a hint, never failed. 16 # 17 # Per-project metadata + build/run recipe: test/ecosystem/recipes/<name>.recipe. 18 # 19 # Usage: 20 # test/ecosystem/run.sh [name ...] run all, or only the named projects 21 # test/ecosystem/run.sh --update [...] refresh expected/<name>.txt goldens 22 # (from the clang build, else the kit build) 23 # env: 24 # KIT kit binary (default build/kit) 25 # KIT_TEST_OPTS space-separated opt levels (default "O0 O1") 26 # KIT_ECO_NO_CLANG=1 skip the clang differential even if clang is present 27 28 set -u 29 30 SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) 31 REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd) 32 ECO_DIR="$SCRIPT_DIR" 33 RECIPE_DIR="$SCRIPT_DIR/recipes" 34 ECOSYS="$REPO_ROOT/scripts/ecosystem.sh" 35 36 KIT="${KIT:-$REPO_ROOT/build/kit}" 37 OPTS="${KIT_TEST_OPTS:-O0 O1}" 38 39 UPDATE=0 40 SEL="" 41 for a in "$@"; do 42 case "$a" in 43 --update) UPDATE=1 ;; 44 -*) echo "ecosystem: unknown flag $a" >&2; exit 2 ;; 45 *) SEL="$SEL $a" ;; 46 esac 47 done 48 49 if [ ! -x "$KIT" ]; then 50 echo "ecosystem: kit binary not found at $KIT (run: make bin)" >&2 51 exit 2 52 fi 53 54 # ---- host toolchain setup -------------------------------------------------- 55 # kit and clang both need the macOS SDK sysroot; on Linux they use the system 56 # libc directly. LINK_LC pulls libc at the link step for kit. 57 case "$(uname -s)" in 58 Darwin) 59 SDK="$(xcrun --sdk macosx --show-sdk-path 2>/dev/null || true)" 60 KIT_SYS="--sysroot $SDK" 61 CLANG_SYS="-isysroot $SDK" 62 ;; 63 *) 64 KIT_SYS="" 65 CLANG_SYS="" 66 ;; 67 esac 68 LINK_LC="-lc" 69 70 HAVE_CLANG=0 71 if [ "${KIT_ECO_NO_CLANG:-0}" != 1 ] && command -v clang >/dev/null 2>&1; then 72 HAVE_CLANG=1 73 fi 74 75 KIT_KIT_DIR="$REPO_ROOT/test/lib" 76 # shellcheck disable=SC1090 77 . "$REPO_ROOT/test/lib/kit_sh_kit.sh" 78 kit_report_init 79 80 WORK=$(mktemp -d "${TMPDIR:-/tmp}/kit-ecosystem.XXXXXX") 81 trap 'rm -rf "$WORK"' EXIT 82 83 # Failures are PRESERVED here (build/ is gitignored) so they can be dug into 84 # later: each failing build leaves its captured stderr, the source list, and an 85 # exact command log. Cleared at the start of every run. 86 FAILDIR="$REPO_ROOT/build/ecosystem-fail" 87 rm -rf "$FAILDIR" 88 89 # preserve_fail NAME OPT TAG OUTDIR — copy a failing build's artifacts out of the 90 # temp workdir into FAILDIR and print where they landed. 91 preserve_fail() { 92 pf="$FAILDIR/$1-$2-$3" 93 mkdir -p "$pf" 94 cp "$4/build.err" "$pf/build.err" 2>/dev/null || true 95 cp "$4/srcs" "$pf/srcs" 2>/dev/null || true 96 cp "$4/cmds.sh" "$pf/cmds.sh" 2>/dev/null || true 97 cp "$4/out" "$pf/out" 2>/dev/null || true 98 cp "$4/run.err" "$pf/run.err" 2>/dev/null || true 99 printf ' -> preserved: %s\n' "$pf" 100 } 101 102 # ---- generic build: compile lib TUs -> static lib -> link driver ----------- 103 # build_with TAG OPT OUTDIR (TAG = kit|clang). Uses ECO_* + SRC from caller. 104 # Returns 0 on a runnable $OUTDIR/app, non-zero (with $OUTDIR/build.err) on any 105 # compile/ar/link failure. 106 build_with() { 107 bw_tag=$1; bw_opt=$2; bw_od=$3 108 mkdir -p "$bw_od/obj" 109 : > "$bw_od/build.err" 110 : > "$bw_od/cmds.sh" 111 112 # Log each toolchain command (for the preserved repro) then run it. 113 log_run() { printf '%s\n' "$*" >> "$bw_od/cmds.sh"; "$@"; } 114 115 bw_inc="" 116 for i in $ECO_INCLUDES; do bw_inc="$bw_inc -I$SRC/$i"; done 117 118 # Library translation units (recipe-defined enumeration). 119 eco_lib_srcs > "$bw_od/srcs" 120 while IFS= read -r c; do 121 [ -z "$c" ] && continue 122 o="$bw_od/obj/$(printf '%s' "$c" | tr '/.' '__').o" 123 if [ "$bw_tag" = kit ]; then 124 log_run "$KIT" cc "-$bw_opt" $KIT_SYS $ECO_DEFINES $bw_inc -c "$c" -o "$o" 2>>"$bw_od/build.err" || return 1 125 else 126 log_run clang "-$bw_opt" $CLANG_SYS $ECO_DEFINES $bw_inc -c "$c" -o "$o" 2>>"$bw_od/build.err" || return 1 127 fi 128 done < "$bw_od/srcs" 129 130 # Archive. 131 if [ "$bw_tag" = kit ]; then 132 log_run "$KIT" ar rcs "$bw_od/lib.a" "$bw_od"/obj/*.o 2>>"$bw_od/build.err" || return 1 133 else 134 log_run ar rcs "$bw_od/lib.a" "$bw_od"/obj/*.o 2>>"$bw_od/build.err" || return 1 135 fi 136 137 # Driver main: "use/<f>.c" lives in test/ecosystem; anything else is the 138 # project's own main, relative to its source root. 139 case "$ECO_DRIVER" in 140 use/*) bw_drv="$ECO_DIR/$ECO_DRIVER" ;; 141 *) bw_drv="$SRC/$ECO_DRIVER" ;; 142 esac 143 144 if [ "$bw_tag" = kit ]; then 145 log_run "$KIT" cc "-$bw_opt" $KIT_SYS $ECO_DEFINES $bw_inc "$bw_drv" "$bw_od/lib.a" $ECO_LIBS $LINK_LC -o "$bw_od/app" 2>>"$bw_od/build.err" || return 1 146 else 147 log_run clang "-$bw_opt" $CLANG_SYS $ECO_DEFINES $bw_inc "$bw_drv" "$bw_od/lib.a" $ECO_LIBS -o "$bw_od/app" 2>>"$bw_od/build.err" || return 1 148 fi 149 return 0 150 } 151 152 # ---- per-project driver ---------------------------------------------------- 153 run_project() { 154 name=$1 155 recipe="$RECIPE_DIR/$name.recipe" 156 if [ ! -f "$recipe" ]; then kit_fail "$name" "no recipe"; return; fi 157 158 # Reset the recipe contract to defaults, then source. 159 ECO_VERSION=; ECO_URL=; ECO_SHA256=; ECO_SRCDIR=. 160 ECO_INCLUDES="."; ECO_DEFINES=""; ECO_LIBS=""; ECO_DRIVER=""; ECO_BUG="" 161 eco_lib_srcs() { :; }; eco_run() { :; } 162 # shellcheck disable=SC1090 163 . "$recipe" 164 export ECO_DIR ECO_SRC 165 166 SRC="$("$ECOSYS" srcdir "$name" 2>/dev/null || true)" 167 if [ -z "$SRC" ] || [ ! -d "$SRC" ]; then 168 kit_skip "$name" "not provisioned — run: make provision-ecosystem" 169 return 170 fi 171 ECO_SRC="$SRC"; export ECO_SRC 172 173 golden="$ECO_DIR/expected/$name.txt" 174 175 for opt in $OPTS; do 176 kod="$WORK/$name/kit-$opt"; mkdir -p "$kod" 177 178 # Failures are kept RED on purpose (no xfail) so known kit bugs stay visible 179 # until fixed; ECO_BUG only annotates the failure with a tracking pointer. 180 if ! build_with kit "$opt" "$kod"; then 181 [ -n "$ECO_BUG" ] && bug=" [known: $ECO_BUG]" || bug="" 182 kit_fail "$name:$opt:build" "kit build failed$bug" 183 tail -8 "$kod/build.err" | sed 's/^/ | /' 184 preserve_fail "$name" "$opt" build "$kod" 185 continue 186 fi 187 188 eco_run "$kod/app" > "$kod/out" 2>"$kod/run.err"; krc=$? 189 if [ "$krc" -ne 0 ]; then 190 [ -n "$ECO_BUG" ] && bug=" [known: $ECO_BUG]" || bug="" 191 kit_fail "$name:$opt:run" "exit=$krc$bug" 192 tail -6 "$kod/run.err" | sed 's/^/ | /' 193 preserve_fail "$name" "$opt" run "$kod" 194 continue 195 fi 196 197 # --update: (re)write the golden from clang (preferred) or kit output. 198 if [ "$UPDATE" = 1 ]; then 199 if [ "$HAVE_CLANG" = 1 ]; then 200 cod="$WORK/$name/clang-$opt"; mkdir -p "$cod" 201 if build_with clang "$opt" "$cod"; then 202 eco_run "$cod/app" > "$cod/out" 2>/dev/null && cp "$cod/out" "$golden" 203 else cp "$kod/out" "$golden"; fi 204 else cp "$kod/out" "$golden"; fi 205 kit_pass "$name:$opt:golden-updated" 206 continue 207 fi 208 209 # Golden assertion. 210 if [ -f "$golden" ]; then 211 if diff -u "$golden" "$kod/out" >/dev/null 2>&1; then 212 kit_pass "$name:$opt:golden" 213 else 214 kit_fail "$name:$opt:golden" 215 diff -u "$golden" "$kod/out" 2>&1 | head -20 | sed 's/^/ | /' 216 preserve_fail "$name" "$opt" golden "$kod" 217 cp "$golden" "$FAILDIR/$name-$opt-golden/golden.expected" 2>/dev/null || true 218 fi 219 else 220 kit_skip "$name:$opt:golden" "no golden (run --update)" 221 fi 222 223 # Differential vs clang. 224 if [ "$HAVE_CLANG" = 1 ]; then 225 cod="$WORK/$name/clang-$opt"; mkdir -p "$cod" 226 if build_with clang "$opt" "$cod"; then 227 eco_run "$cod/app" > "$cod/out" 2>/dev/null 228 if diff -u "$cod/out" "$kod/out" >/dev/null 2>&1; then 229 kit_pass "$name:$opt:vs-clang" 230 else 231 kit_fail "$name:$opt:vs-clang" 232 diff -u "$cod/out" "$kod/out" 2>&1 | head -20 | sed 's/^/ | /' 233 preserve_fail "$name" "$opt" vs-clang "$kod" 234 cp "$cod/out" "$FAILDIR/$name-$opt-vs-clang/clang.out" 2>/dev/null || true 235 fi 236 else 237 kit_skip "$name:$opt:vs-clang" "clang build failed" 238 fi 239 else 240 kit_skip_na "$name:$opt:vs-clang" "clang not available" 241 fi 242 done 243 } 244 245 # ---- drive ---------------------------------------------------------------- 246 PROJECTS="${SEL:-$("$ECOSYS" list)}" 247 for p in $PROJECTS; do 248 run_project "$p" 249 done 250 251 kit_summary "ecosystem" 252 kit_exit