kit_corpus.sh (14707B)
1 # test/lib/kit_corpus.sh — the unified corpus test harness engine. 2 # 3 # ONE harness for the lane-matrix corpus runners (parse/toy/asm/link/elf/wasm). 4 # A runner declares its corpus + lanes and calls kit_corpus_run; the engine owns 5 # discovery, the {case}x{opt}x{tuple} matrix, per-case workdirs, serial-or- 6 # parallel dispatch, event replay, deferred cross-arch exec, and per-lane 7 # timing. Reporting goes through kit_sh_report.sh's mode-transparent verbs. 8 # 9 # Requires bash (indexed arrays). Sources, relative to $KIT_LIB_DIR: 10 # kit_sh_report.sh (kit_pass/kit_fail/kit_skip/kit_skip_na/kit_time/summary) 11 # kit_skip.sh (kit_skip_sidecar/kit_skip_diag/kit_tuple_applicable) 12 # parallel.sh (kit_parallel_jobs/run/wait_all) 13 # exec_target.sh (exec_target_queue/flush/supported) [sourced by the runner if it uses path E] 14 # 15 # ============================================================================ 16 # RUNNER CONTRACT — set these before calling kit_corpus_run: 17 # KIT binary under test 18 # KIT_LABEL summary label (e.g. "toy") 19 # KIT_BUILD_DIR scratch root; per-item workdirs live under it 20 # KIT_CORPUS_GLOBS space-separated globs of case source files 21 # KIT_CORPUS_LIST optional exact manifest of cases to run. Entries are 22 # newline-separated, with blank lines and # comments 23 # ignored. Relative entries are resolved from the first 24 # corpus glob's directory; bare basenames may omit 25 # KIT_CORPUS_EXT. 26 # KIT_CORPUS_EXT extension stripped for the basename (e.g. "toy","c","s") 27 # KIT_SIDECAR_DIR dir holding sidecars (default: dir of the first glob) 28 # KIT_LANES active lane ids, space-separated (runner derives from KIT_TEST_PATHS) 29 # KIT_OPT_LEVELS opt levels to expand (default "$KIT_OPT_LEVELS" or "0 1"); 30 # set to "" for corpora with no opt axis (e.g. elf layers) 31 # KIT_TUPLES target tuples "<arch>-<obj>" (default "$KIT_DEFAULT_TUPLE") 32 # KIT_OPT0ONLY lanes that run only at opt 0 (space list; e.g. "C W") 33 # KIT_PARALLELIZABLE 1 to allow parallel dispatch (default 1); 0 forces serial 34 # KIT_EXPECTED_EXT sidecar suffix for the expected exit code (default ".expected") 35 # KIT_TARGETS_EXT sidecar suffix for whole-case tuple applicability (default ".targets"; empty disables) 36 # kit_lane_<ID>() per-lane hook (see below) 37 # KIT_READ_CASE optional fn, called per item after basics are set, to read 38 # bespoke markers (may override KIT_EXPECTED, set KIT_SKIP_CASE/KIT_SKIP_NA_CASE) 39 # KIT_FLUSH_VERIFY optional fn, called per queued-E item after flush: args 40 # (label, payload, rc); return 0 to keep pass, 1 to fail 41 # 42 # Debugging: 43 # KIT_CORPUS_TRACE=1 force serial dispatch and print each item/lane before it 44 # runs; deferred exec queues are flushed after each item. 45 # 46 # Per-item vars the engine sets before each kit_lane_<ID> call: 47 # KIT_BASE KIT_SRC KIT_WORK KIT_OPT KIT_LANE KIT_ARCH KIT_OBJ KIT_TUPLE KIT_EXPECTED 48 # KIT_NAME (display label "base" or "base/Oopt") KIT_SIDECAR_DIR 49 # A hook ends by calling exactly one of: kit_pass/kit_fail/kit_skip/kit_skip_na, OR 50 # kit_queue_e (deferred exec, resolved at flush). Hooks must write only under 51 # KIT_WORK and record only via these verbs — that is what makes a runner 52 # parallel-safe by construction (flip KIT_PARALLELIZABLE with no other change). 53 # ============================================================================ 54 55 KIT_LIB_DIR="${KIT_LIB_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" 56 . "$KIT_LIB_DIR/kit_sh_report.sh" 57 . "$KIT_LIB_DIR/kit_skip.sh" 58 . "$KIT_LIB_DIR/parallel.sh" 59 60 # Corpus runners treat an unexplained SKIP as a soft failure (gated by 61 # KIT_TEST_ALLOW_SKIP); the driver-scenario harnesses leave this 0. 62 KIT_SKIP_IS_FAILURE=1 63 64 # Millisecond clock for per-lane timing (python3 if present; else second res). 65 kit_now_ms() { 66 if command -v python3 >/dev/null 2>&1; then 67 python3 -c 'import time;print(int(time.time()*1000))' 68 else 69 echo $(( $(date +%s) * 1000 )) 70 fi 71 } 72 73 kit_corpus_trace_enabled() { 74 case "${KIT_CORPUS_TRACE:-0}" in 75 ""|0|false|FALSE|no|NO) return 1 ;; 76 *) return 0 ;; 77 esac 78 } 79 80 kit_corpus_trace() { 81 kit_corpus_trace_enabled || return 0 82 printf 'TRACE %s\n' "$*" >&2 83 } 84 85 # Deferred-exec bookkeeping (populated ONLY during serial execution / replay, 86 # never inside a worker — workers merely emit QUEUE_E events). 87 CFQ_LABELS=(); CFQ_RCS=(); CFQ_EXPS=(); CFQ_PAYLOADS=() 88 89 # kit_queue_e LABEL EXE OUT ERR RC_FILE EXPECTED EXEC_TAG [PAYLOAD] 90 # Record a case whose execution is deferred to a batched exec_target flush. 91 # In a worker: emit a QUEUE_E event (the parent does the actual queueing). 92 # In serial/replay: enqueue with exec_target and remember it for the flush. 93 kit_queue_e() { 94 if [ -n "${KIT_EV:-}" ]; then 95 printf 'QUEUE_E\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ 96 "$1" "$2" "$3" "$4" "$5" "$6" "$7" "${8:-}" >> "$KIT_EV" 97 return 98 fi 99 exec_target_queue "$7" "$1" "$2" "$3" "$4" "$5" 100 CFQ_LABELS+=("$1"); CFQ_RCS+=("$5"); CFQ_EXPS+=("$6"); CFQ_PAYLOADS+=("${8:-}") 101 } 102 103 # ---- discovery ------------------------------------------------------------- 104 kit_corpus_discover() { 105 KIT_CASES=() 106 local g f base manifest_base entry path 107 shopt -s nullglob 108 if [ -n "${KIT_CORPUS_LIST:-}" ]; then 109 if [ ! -f "$KIT_CORPUS_LIST" ]; then 110 echo "$KIT_LABEL: KIT_CORPUS_LIST not found: $KIT_CORPUS_LIST" >&2 111 exit 2 112 fi 113 set -- $KIT_CORPUS_GLOBS 114 manifest_base=$(dirname "$1") 115 while IFS= read -r entry || [ -n "$entry" ]; do 116 entry="${entry%%#*}" 117 entry="${entry#"${entry%%[![:space:]]*}"}" 118 entry="${entry%"${entry##*[![:space:]]}"}" 119 [ -z "$entry" ] && continue 120 case "$entry" in 121 /*) path="$entry" ;; 122 *) path="$manifest_base/$entry" ;; 123 esac 124 if [ ! -e "$path" ] && [ -n "$KIT_CORPUS_EXT" ]; then 125 path="$path.$KIT_CORPUS_EXT" 126 fi 127 if [ ! -e "$path" ]; then 128 echo "$KIT_LABEL: manifest case not found: $entry" >&2 129 exit 2 130 fi 131 base=$(basename "$path") 132 [ -n "$KIT_CORPUS_EXT" ] && base="${base%.$KIT_CORPUS_EXT}" 133 if [ -n "${KIT_TEST_FILTER:-}" ]; then 134 case "$base" in *"$KIT_TEST_FILTER"*) ;; *) continue ;; esac 135 fi 136 KIT_CASES+=("$path") 137 done < "$KIT_CORPUS_LIST" 138 return 139 fi 140 for g in $KIT_CORPUS_GLOBS; do 141 for f in $g; do 142 base=$(basename "$f") 143 [ -n "$KIT_CORPUS_EXT" ] && base="${base%.$KIT_CORPUS_EXT}" 144 if [ -n "${KIT_TEST_FILTER:-}" ]; then 145 case "$base" in *"$KIT_TEST_FILTER"*) ;; *) continue ;; esac 146 fi 147 KIT_CASES+=("$f") 148 done 149 done 150 } 151 152 # ---- per-item driver (runs every active lane for one case,opt,tuple) ------- 153 # Item encoding: "src|base|opt|tuple" (src/base never contain '|'). 154 kit_corpus_item() { 155 local item="$1" 156 KIT_SRC="${item%%|*}"; item="${item#*|}" 157 KIT_BASE="${item%%|*}"; item="${item#*|}" 158 KIT_OPT="${item%%|*}"; KIT_TUPLE="${item#*|}" 159 KIT_ARCH="${KIT_TUPLE%-*}"; KIT_OBJ="${KIT_TUPLE#*-}" 160 161 # KIT_WORK must be unique per work item. The matrix is case x opt x tuple 162 # (see kit_corpus_run), so the tuple is folded into the path — otherwise two 163 # tuples of the same (case, opt) share a dir and, under parallel dispatch, 164 # stomp each other's artifacts (one worker's `rm -rf $KIT_WORK` / rebuilt exe 165 # races another's exec, surfacing as spurious SIGILLs). KIT_NAME stays the 166 # human label; lanes that sweep multiple tuples prefix it with the arch. 167 if [ "$KIT_OPT" = "-" ]; then 168 KIT_NAME="$KIT_BASE"; KIT_WORK="$KIT_BUILD_DIR/$KIT_BASE/$KIT_TUPLE" 169 else 170 KIT_NAME="$KIT_BASE/O$KIT_OPT"; KIT_WORK="$KIT_BUILD_DIR/$KIT_BASE/O$KIT_OPT/$KIT_TUPLE" 171 fi 172 kit_corpus_trace "item $KIT_NAME tuple=$KIT_TUPLE src=$KIT_SRC" 173 rm -rf "$KIT_WORK"; mkdir -p "$KIT_WORK" 174 175 KIT_EXPECTED=0 176 if [ -f "$KIT_SIDECAR_DIR/$KIT_BASE$KIT_EXPECTED_EXT" ]; then 177 KIT_EXPECTED=$(tr -d '[:space:]' < "$KIT_SIDECAR_DIR/$KIT_BASE$KIT_EXPECTED_EXT") 178 fi 179 180 # Hook for bespoke per-case marker reading (link's expected/jit_only/targets, 181 # etc.). May set KIT_EXPECTED, KIT_SKIP_CASE (reason), KIT_SKIP_NA_CASE (1). 182 KIT_SKIP_CASE=; KIT_SKIP_NA_CASE= 183 if [ -n "${KIT_READ_CASE:-}" ] && command -v "$KIT_READ_CASE" >/dev/null 2>&1; then 184 "$KIT_READ_CASE" 185 fi 186 if [ -n "$KIT_SKIP_CASE" ]; then kit_skip "$KIT_NAME" "$KIT_SKIP_CASE"; return; fi 187 if [ -n "$KIT_SKIP_NA_CASE" ]; then kit_skip_na "$KIT_NAME"; return; fi 188 189 # Whole-case sidecar skip (one event for the item, not per lane). 190 local reason 191 if reason=$(kit_skip_sidecar "$KIT_SIDECAR_DIR" "$KIT_BASE" "$KIT_ARCH"); then 192 kit_skip "$KIT_NAME" "$reason"; return 193 fi 194 # Whole-case tuple applicability (SKIP-NA, uncounted). 195 if [ -n "$KIT_TARGETS_EXT" ] && 196 ! kit_tuple_applicable "$KIT_TUPLE" "$KIT_SIDECAR_DIR/$KIT_BASE$KIT_TARGETS_EXT"; then 197 kit_skip_na "$KIT_NAME"; return 198 fi 199 200 local lane 201 for lane in $KIT_LANES; do 202 KIT_LANE="$lane" 203 # opt-0-only lanes (C/W: backend ignores -O) run once, at opt 0. 204 case " $KIT_OPT0ONLY " in 205 *" $lane "*) [ "$KIT_OPT" = "0" ] || [ "$KIT_OPT" = "-" ] || continue ;; 206 esac 207 # per-lane sidecar skip (.<lane>.skip) 208 if reason=$(kit_skip_sidecar "$KIT_SIDECAR_DIR" "$KIT_BASE" "" "$lane"); then 209 kit_corpus_trace "lane $KIT_NAME/$lane skip-sidecar" 210 kit_skip "$KIT_NAME/$lane" "$reason"; continue 211 fi 212 kit_corpus_trace "lane $KIT_NAME/$lane" 213 "kit_lane_$lane" 214 done 215 } 216 217 # ---- worker (parallel mode): isolate output to the event file -------------- 218 # Runs in a background subshell; the ONLY parent-visible channel is the event 219 # file (+ captured stdout/stderr logs). Ends with a DONE sentinel so the parent 220 # can tell "produced no records" (fine) from "crashed mid-case" (fail). 221 kit_corpus_worker() { 222 KIT_EV="$3" 223 kit_corpus_item "$2" > "$4" 2> "$5" 224 printf 'DONE\n' >> "$KIT_EV" 225 } 226 227 # ---- replay one worker's event file through the counting verbs ------------- 228 kit_corpus_replay() { 229 local ev="$1" out="$2" err="$3" idx="$4" 230 if [ ! -s "$ev" ] || [ "$(tail -n1 "$ev" 2>/dev/null)" != "DONE" ]; then 231 kit_fail "internal/worker-$idx" "no DONE sentinel (worker crashed?)" 232 else 233 local kind a b c d e f g h 234 while IFS="$(printf '\t')" read -r kind a b c d e f g h; do 235 case "$kind" in 236 PASS) kit_pass "$a" ;; 237 FAIL) kit_fail "$a" "$b" ;; 238 SKIP) kit_skip "$a" "$b" ;; 239 SKIP_NA) kit_skip_na "$a" "$b" ;; 240 XFAIL) kit_xfail "$a" "$b" ;; 241 XPASS) kit_xpass "$a" ;; 242 TIME) kit_time "$a" "$b" ;; 243 QUEUE_E) kit_queue_e "$a" "$b" "$c" "$d" "$e" "$f" "$g" "$h" ;; 244 DONE) ;; 245 esac 246 done < "$ev" 247 fi 248 # Surface hook diagnostics (diffs/stderr) deterministically, in index order. 249 [ -s "$out" ] && cat "$out" 250 [ -s "$err" ] && cat "$err" >&2 251 return 0 252 } 253 254 # ---- deferred-exec flush + per-case verification --------------------------- 255 kit_corpus_flush_e() { 256 [ "${#CFQ_LABELS[@]}" -eq 0 ] && return 0 257 kit_corpus_trace "flush E queued=${#CFQ_LABELS[@]}" 258 exec_target_flush 259 local i rc ok 260 for i in "${!CFQ_LABELS[@]}"; do 261 rc=127; [ -f "${CFQ_RCS[$i]}" ] && rc=$(cat "${CFQ_RCS[$i]}") 262 rc=$((rc & 255)) 263 ok=0; [ "$rc" -eq "$(( ${CFQ_EXPS[$i]} & 255 ))" ] && ok=1 264 if [ "$ok" -eq 1 ] && [ -n "${KIT_FLUSH_VERIFY:-}" ] && 265 command -v "$KIT_FLUSH_VERIFY" >/dev/null 2>&1; then 266 "$KIT_FLUSH_VERIFY" "${CFQ_LABELS[$i]}" "${CFQ_PAYLOADS[$i]}" "$rc" || ok=0 267 fi 268 if [ "$ok" -eq 1 ]; then kit_pass "${CFQ_LABELS[$i]}" 269 else kit_fail "${CFQ_LABELS[$i]}" "expected ${CFQ_EXPS[$i]} got $rc"; fi 270 done 271 CFQ_LABELS=(); CFQ_RCS=(); CFQ_EXPS=(); CFQ_PAYLOADS=() 272 } 273 274 # ---- the entrypoint -------------------------------------------------------- 275 # Discovers + expands the matrix + dispatches (serial or parallel) + flushes 276 # deferred exec. Accumulates into the shared counters (so a runner may call it 277 # multiple times — e.g. elf's A/B/C layers — then summarize once). Does NOT 278 # print the summary or exit; the runner calls kit_summary "$KIT_LABEL"; kit_exit. 279 kit_corpus_run() { 280 : "${KIT_OPT_LEVELS=${KIT_OPT_LEVELS:-0 1}}" 281 : "${KIT_TUPLES:=${KIT_DEFAULT_TUPLE:-aarch64-elf}}" 282 : "${KIT_OPT0ONLY:=}" 283 : "${KIT_PARALLELIZABLE:=1}" 284 : "${KIT_EXPECTED_EXT:=.expected}" 285 # No-colon `=`: only default when UNSET. A caller-supplied empty value means 286 # "disable the whole-case tuple matcher" (see CONTRACT above) — the colon form 287 # `:=` would wrongly clobber that empty back to .targets and re-enable it. 288 : "${KIT_TARGETS_EXT=.targets}" 289 if [ -z "${KIT_SIDECAR_DIR:-}" ]; then 290 set -- $KIT_CORPUS_GLOBS; KIT_SIDECAR_DIR=$(dirname "$1") 291 fi 292 293 kit_corpus_discover 294 if [ "${#KIT_CASES[@]}" -eq 0 ]; then 295 echo "$KIT_LABEL: no cases under $KIT_CORPUS_GLOBS" >&2 296 exit 2 297 fi 298 299 # Build the flat work-item list = case x opt x tuple. 300 local opts="$KIT_OPT_LEVELS"; [ -z "$opts" ] && opts="-" 301 local f base o t 302 KIT_ITEMS=() 303 for f in "${KIT_CASES[@]}"; do 304 base=$(basename "$f"); [ -n "$KIT_CORPUS_EXT" ] && base="${base%.$KIT_CORPUS_EXT}" 305 for o in $opts; do 306 for t in $KIT_TUPLES; do 307 KIT_ITEMS+=("$f|$base|$o|$t") 308 done 309 done 310 done 311 312 local jobs; jobs="$(kit_parallel_jobs)" || jobs=1 313 if kit_corpus_trace_enabled; then 314 KIT_PARALLELIZABLE=0 315 jobs=1 316 kit_corpus_trace "dispatch serial items=${#KIT_ITEMS[@]} lanes=${KIT_LANES# }" 317 fi 318 if [ "$KIT_PARALLELIZABLE" = "1" ] && [ "$jobs" -gt 1 ] && [ "${#KIT_ITEMS[@]}" -gt 4 ]; then 319 kit_corpus_dispatch_parallel "$jobs" 320 else 321 local idx=0 322 for item in "${KIT_ITEMS[@]}"; do 323 KIT_EV=; kit_corpus_item "$item" 324 kit_corpus_trace_enabled && kit_corpus_flush_e 325 idx=$((idx + 1)) 326 done 327 fi 328 329 kit_corpus_flush_e 330 } 331 332 kit_corpus_dispatch_parallel() { 333 local jobs="$1" 334 local pdir="$KIT_BUILD_DIR/.parallel.$$" 335 rm -rf "$pdir"; mkdir -p "$pdir" 336 local evs=() outs=() errs=() idx=0 ev out err 337 for item in "${KIT_ITEMS[@]}"; do 338 ev="$pdir/$idx.events"; out="$pdir/$idx.out"; err="$pdir/$idx.err" 339 : > "$ev" 340 evs+=("$ev"); outs+=("$out"); errs+=("$err") 341 kit_parallel_run "$jobs" kit_corpus_worker "$idx" "$item" "$ev" "$out" "$err" 342 idx=$((idx + 1)) 343 done 344 kit_parallel_wait_all || true 345 # Serial replay in strict index order -> deterministic counts + output + 346 # exec_target queueing (which happens here, never in a worker). 347 KIT_EV= 348 local i=0 349 while [ "$i" -lt "${#evs[@]}" ]; do 350 kit_corpus_replay "${evs[$i]}" "${outs[$i]}" "${errs[$i]}" "$i" 351 i=$((i + 1)) 352 done 353 rm -rf "$pdir" 354 }