run.sh (7539B)
1 #!/bin/sh 2 # Scripted `kit dbg` transcript tests — Type K, mode G (golden transcript). 3 # 4 # Each case lives in test/dbg/cases/<name>/ and may contain: 5 # args one kit-dbg argument per line; @CASE@ expands to the case dir 6 # stdin commands fed to the REPL; @CASE@ expands to the case dir 7 # expected normalized stdout golden 8 # stderr optional exact stderr golden; absent means stderr must be empty 9 # xfail optional reason; expected failure until the feature is implemented 10 # 11 # dbg is mode G, but kit_scenario_case (the shared mode-G oracle) does not cover 12 # dbg's needs: per-case args/stdin templating, an stdin script piped to the 13 # REPL, a multi-stage stdout normalizer, separate stderr handling, and xfail/ 14 # xpass. So the per-case oracle (dbg_case) and the normalizer live lane-local 15 # here, while results, summary, and exit go through the shared Type-K kit. 16 # 17 # The stdout normalizer removes interactive prompts so goldens focus on 18 # debugger-visible events, masks run-to-run addresses, and re-tokenizes the 19 # case dir back to @CASE@. It intentionally leaves generation numbers and 20 # command output intact. 21 22 set -u 23 24 script_dir=$(cd "$(dirname "$0")" && pwd) 25 repo_root=$(cd "$script_dir/../.." && pwd) 26 cases_dir="$script_dir/cases" 27 28 KIT_KIT_DIR="$repo_root/test/lib" 29 . "$repo_root/test/lib/kit_sh_kit.sh" 30 31 KIT="${KIT:-$repo_root/build/kit}" 32 export KIT 33 kit_require_kit dbg 34 35 # dbg drives a JIT debugger that single-steps native code; only hosts whose 36 # native arch matches the JIT target can run it. (Preserved host gate.) 37 host_arch=$(uname -m 2>/dev/null || true) 38 host_os=$(uname -s 2>/dev/null || true) 39 case "$host_os:$host_arch" in 40 Darwin:arm64|Darwin:aarch64|Linux:arm64|Linux:aarch64) ;; 41 *) 42 printf 'SKIP test-dbg (unsupported host %s/%s)\n' "$host_os" "$host_arch" 43 exit 0 44 ;; 45 esac 46 47 kit_workdir dbg 48 49 # DBG_STRICT_XFAIL drives the report layer's strict-xfail gate: an xfail case 50 # that unexpectedly passes becomes a real failure (stale marker) under strict. 51 KIT_STRICT_XFAIL=${DBG_STRICT_XFAIL:-0} 52 53 kit_report_init 54 55 # normalize_stdout RAW CASE_DIR : multi-stage stdout normalizer (lane-local). 56 # Strips the "(kit) " prompt and continuation markers, re-tokenizes CASE_DIR 57 # to @CASE@, masks run-to-run hex addresses to 0xADDR, trims trailing 58 # whitespace, and elides blank lines. Idempotent on already-normalized goldens. 59 normalize_stdout() { 60 sed -e "s|$2|@CASE@|g" \ 61 -e 's/(kit) //g' -e 's/^ > //' -e 's/^expr > //' "$1" | 62 sed -E 's/0x[[:xdigit:]]{8,}/0xADDR/g' | 63 sed -E 's/[[:space:]]+$//' | 64 sed '/^$/d' 65 } 66 67 # case_selected NAME : honor the DBG_CASE comma-separated glob filter. With no 68 # filter, every case runs. (Preserved selection knob.) 69 case_selected() { 70 [ -n "${DBG_CASE:-}" ] || return 0 71 for pat in $(printf '%s' "$DBG_CASE" | tr ',' ' '); do 72 case "$1" in 73 $pat) return 0 ;; 74 esac 75 done 76 return 1 77 } 78 79 # dbg_case NAME CASE_DIR : run one transcript case and record exactly one 80 # verdict via the shared kit_* verbs (kit_pass/kit_fail/kit_xfail/kit_xpass). 81 dbg_case() { 82 dc_name=$1 83 dc_dir=$2 84 dc_stdin_file="$dc_dir/stdin" 85 dc_expected="$dc_dir/expected" 86 dc_expected_stderr="$dc_dir/stderr" 87 dc_expected_rc_file="$dc_dir/expected_rc" 88 dc_xfail_file="$dc_dir/xfail" 89 dc_raw_stdout="$KIT_WORK/$dc_name.stdout.raw" 90 dc_actual_stdout="$KIT_WORK/$dc_name.stdout" 91 dc_expected_norm="$KIT_WORK/$dc_name.expected" 92 dc_raw_stderr="$KIT_WORK/$dc_name.stderr" 93 dc_actual_stderr="$KIT_WORK/$dc_name.stderr.actual" 94 dc_case_stdin="$KIT_WORK/$dc_name.stdin" 95 dc_is_xfail=0 96 dc_reason= 97 dc_ok=1 98 dc_why= 99 100 if [ -e "$dc_xfail_file" ]; then 101 dc_is_xfail=1 102 dc_reason=$(sed -n '1p' "$dc_xfail_file") 103 fi 104 105 # Missing required inputs: a setup failure, routed through xfail when the 106 # case is marked (and not strict), else a plain failure. 107 if [ ! -e "$dc_stdin_file" ]; then 108 dc_ok=0 109 dc_why="missing stdin" 110 elif [ ! -e "$dc_expected" ]; then 111 dc_ok=0 112 dc_why="missing expected" 113 fi 114 if [ "$dc_ok" -eq 0 ]; then 115 if [ "$dc_is_xfail" -eq 1 ]; then 116 kit_xfail "$dc_name" "$dc_why" 117 else 118 kit_fail "$dc_name" "$dc_why" 119 fi 120 return 121 fi 122 123 # Build the dbg argv, templating @CASE@ -> case dir; skip blanks/#comments. 124 set -- 125 dc_args_file="$dc_dir/args" 126 if [ -e "$dc_args_file" ]; then 127 while IFS= read -r dc_arg || [ -n "$dc_arg" ]; do 128 [ -n "$dc_arg" ] || continue 129 case "$dc_arg" in 130 \#*) continue ;; 131 esac 132 dc_arg=$(printf '%s' "$dc_arg" | sed "s|@CASE@|$dc_dir|g") 133 set -- "$@" "$dc_arg" 134 done < "$dc_args_file" 135 fi 136 137 sed "s|@CASE@|$dc_dir|g" "$dc_stdin_file" > "$dc_case_stdin" 138 "$KIT" dbg "$@" < "$dc_case_stdin" > "$dc_raw_stdout" 2> "$dc_raw_stderr" 139 dc_rc=$? 140 141 # Normalize BOTH actual and the (already-normalized, so idempotent) golden. 142 normalize_stdout "$dc_raw_stdout" "$dc_dir" > "$dc_actual_stdout" 143 normalize_stdout "$dc_expected" "$dc_dir" > "$dc_expected_norm" 144 if [ -e "$dc_expected_stderr" ]; then 145 cp "$dc_raw_stderr" "$dc_actual_stderr" 146 else 147 : > "$dc_actual_stderr" 148 fi 149 150 if [ -e "$dc_expected_rc_file" ]; then 151 dc_want_rc=$(sed -n '1p' "$dc_expected_rc_file") 152 if [ "$dc_rc" -ne "$dc_want_rc" ]; then 153 dc_ok=0 154 dc_why="kit dbg exit=$dc_rc (expected $dc_want_rc)" 155 fi 156 elif [ "$dc_rc" -ne 0 ]; then 157 dc_ok=0 158 dc_why="kit dbg exit=$dc_rc" 159 fi 160 if [ "$dc_ok" -eq 1 ] && ! diff -u "$dc_expected_norm" "$dc_actual_stdout" >/dev/null 2>&1; then 161 dc_ok=0 162 dc_why="stdout" 163 fi 164 if [ "$dc_ok" -eq 1 ] && [ -e "$dc_expected_stderr" ]; then 165 if ! diff -u "$dc_expected_stderr" "$dc_actual_stderr" >/dev/null 2>&1; then 166 dc_ok=0 167 dc_why="stderr" 168 fi 169 elif [ "$dc_ok" -eq 1 ] && [ -s "$dc_raw_stderr" ]; then 170 dc_ok=0 171 dc_why="unexpected stderr" 172 fi 173 174 if [ "$dc_ok" -eq 1 ]; then 175 if [ "$dc_is_xfail" -eq 1 ]; then 176 # Marked xfail but passed (a stale xfail marker): kit_xpass is ALWAYS 177 # a failure in the report layer — matching the original's 178 # unconditional xpass-counts-as-fail behavior. 179 kit_xpass "$dc_name" 180 else 181 kit_pass "$dc_name" 182 fi 183 return 184 fi 185 186 if [ "$dc_is_xfail" -eq 1 ]; then 187 if [ -n "$dc_reason" ]; then 188 kit_xfail "$dc_name" "$dc_why: $dc_reason" 189 else 190 kit_xfail "$dc_name" "$dc_why" 191 fi 192 return 193 fi 194 195 kit_fail "$dc_name" "$dc_why" 196 case "$dc_why" in 197 stdout) 198 diff -u "$dc_expected_norm" "$dc_actual_stdout" || true 199 cp "$dc_actual_stdout" "$dc_dir/actual" 2>/dev/null || true 200 ;; 201 stderr) 202 diff -u "$dc_expected_stderr" "$dc_actual_stderr" || true 203 ;; 204 "unexpected stderr") 205 sed 's/^/ | /' "$dc_raw_stderr" 206 ;; 207 kit\ dbg\ exit=*) 208 sed 's/^/ stdout| /' "$dc_raw_stdout" 209 sed 's/^/ stderr| /' "$dc_raw_stderr" 210 ;; 211 esac 212 } 213 214 for case_dir in "$cases_dir"/*; do 215 [ -d "$case_dir" ] || continue 216 name=$(basename "$case_dir") 217 case_selected "$name" || continue 218 dbg_case "$name" "$case_dir" 219 done 220 221 kit_summary dbg 222 kit_exit