run.sh (9296B)
1 #!/bin/sh 2 # Scripted `kit dbg` transcript tests — Type K, mode G (golden transcript). 3 # 4 # Each case lives in test/dbg/cases/<name>/ and may contain: 5 # args one kit-dbg argument per line; @CASE@ expands to the case dir 6 # stdin commands fed to the REPL; @CASE@ expands to the case dir 7 # expected normalized stdout golden 8 # stderr optional exact stderr golden; absent means stderr must be empty 9 # xfail optional reason; expected failure until the feature is implemented 10 # 11 # dbg is mode G, but kit_scenario_case (the shared mode-G oracle) does not cover 12 # dbg's needs: per-case args/stdin templating, an stdin script piped to the 13 # REPL, a multi-stage stdout normalizer, separate stderr handling, and xfail/ 14 # xpass. So the per-case oracle (dbg_case) and the normalizer live lane-local 15 # here, while results, summary, and exit go through the shared Type-K kit. 16 # 17 # The stdout normalizer removes interactive prompts so goldens focus on 18 # debugger-visible events, masks run-to-run addresses, and re-tokenizes the 19 # case dir back to @CASE@. It intentionally leaves generation numbers and 20 # command output intact. 21 22 set -u 23 24 script_dir=$(cd "$(dirname "$0")" && pwd) 25 repo_root=$(cd "$script_dir/../.." && pwd) 26 cases_dir="$script_dir/cases" 27 28 KIT_KIT_DIR="$repo_root/test/lib" 29 . "$repo_root/test/lib/kit_sh_kit.sh" 30 31 KIT="${KIT:-$repo_root/build/kit}" 32 export KIT 33 kit_require_kit dbg 34 35 # dbg drives a JIT debugger that single-steps native code; only hosts whose 36 # native arch matches the JIT target can run it. (Preserved host gate.) 37 host_arch=$(uname -m 2>/dev/null || true) 38 host_os=$(uname -s 2>/dev/null || true) 39 case "$host_os:$host_arch" in 40 Darwin:arm64|Darwin:aarch64|Linux:arm64|Linux:aarch64) ;; 41 *) 42 printf 'SKIP test-dbg (unsupported host %s/%s)\n' "$host_os" "$host_arch" 43 exit 0 44 ;; 45 esac 46 47 kit_workdir dbg 48 49 # DBG_STRICT_XFAIL drives the report layer's strict-xfail gate: an xfail case 50 # that unexpectedly passes becomes a real failure (stale marker) under strict. 51 KIT_STRICT_XFAIL=${DBG_STRICT_XFAIL:-0} 52 53 kit_report_init 54 55 # Explicit script sources have command-style status semantics independent of 56 # transcript contents. Exercise them outside the golden matrix so failures can 57 # never be hidden by prompt normalization or an interactive EOF fallback. 58 dbg_expect_status() { 59 des_name=$1 60 des_want=$2 61 shift 2 62 "$@" </dev/null > "$KIT_WORK/$des_name.stdout" \ 63 2> "$KIT_WORK/$des_name.stderr" 64 des_got=$? 65 if [ "$des_got" -eq "$des_want" ]; then 66 kit_pass "$des_name" 67 else 68 kit_fail "$des_name" "status=$des_got expected=$des_want" 69 sed 's/^/ stdout| /' "$KIT_WORK/$des_name.stdout" 70 sed 's/^/ stderr| /' "$KIT_WORK/$des_name.stderr" 71 fi 72 } 73 74 printf 'set\n' > "$KIT_WORK/malformed.dbg" 75 awk 'BEGIN { for (i = 0; i < 1100; ++i) printf "x"; printf "\n" }' \ 76 > "$KIT_WORK/overlong.dbg" 77 printf 'q\n' > "$KIT_WORK/quit.dbg" 78 79 dbg_expect_status missing-script-batch 1 \ 80 "$KIT" dbg --batch --script "$KIT_WORK/does-not-exist.dbg" 81 dbg_expect_status missing-script-interactive 1 \ 82 "$KIT" dbg --script "$KIT_WORK/does-not-exist.dbg" 83 dbg_expect_status unreadable-script-batch 1 \ 84 "$KIT" dbg --batch --script "$KIT_WORK" 85 dbg_expect_status malformed-script-batch 1 \ 86 "$KIT" dbg --batch --script "$KIT_WORK/malformed.dbg" 87 dbg_expect_status malformed-script-interactive 1 \ 88 "$KIT" dbg --script "$KIT_WORK/malformed.dbg" 89 dbg_expect_status overlong-script-batch 1 \ 90 "$KIT" dbg --batch --script "$KIT_WORK/overlong.dbg" 91 dbg_expect_status overlong-script-interactive 1 \ 92 "$KIT" dbg --script "$KIT_WORK/overlong.dbg" 93 dbg_expect_status explicit-quit-batch 0 \ 94 "$KIT" dbg --batch --script "$KIT_WORK/quit.dbg" 95 dbg_expect_status explicit-quit-interactive 0 \ 96 "$KIT" dbg --script "$KIT_WORK/quit.dbg" 97 98 # normalize_stdout RAW CASE_DIR : multi-stage stdout normalizer (lane-local). 99 # Strips the "(kit) " prompt and continuation markers, re-tokenizes CASE_DIR 100 # to @CASE@, masks run-to-run hex addresses to 0xADDR, trims trailing 101 # whitespace, and elides blank lines. Idempotent on already-normalized goldens. 102 normalize_stdout() { 103 sed -e "s|$2|@CASE@|g" \ 104 -e 's/(kit) //g' -e 's/^ > //' -e 's/^expr > //' "$1" | 105 sed -E 's/0x[[:xdigit:]]{8,}/0xADDR/g' | 106 sed -E 's/[[:space:]]+$//' | 107 sed '/^$/d' 108 } 109 110 # case_selected NAME : honor the DBG_CASE comma-separated glob filter. With no 111 # filter, every case runs. (Preserved selection knob.) 112 case_selected() { 113 [ -n "${DBG_CASE:-}" ] || return 0 114 for pat in $(printf '%s' "$DBG_CASE" | tr ',' ' '); do 115 case "$1" in 116 $pat) return 0 ;; 117 esac 118 done 119 return 1 120 } 121 122 # dbg_case NAME CASE_DIR : run one transcript case and record exactly one 123 # verdict via the shared kit_* verbs (kit_pass/kit_fail/kit_xfail/kit_xpass). 124 dbg_case() { 125 dc_name=$1 126 dc_dir=$2 127 dc_stdin_file="$dc_dir/stdin" 128 dc_expected="$dc_dir/expected" 129 dc_expected_stderr="$dc_dir/stderr" 130 dc_expected_rc_file="$dc_dir/expected_rc" 131 dc_xfail_file="$dc_dir/xfail" 132 dc_raw_stdout="$KIT_WORK/$dc_name.stdout.raw" 133 dc_actual_stdout="$KIT_WORK/$dc_name.stdout" 134 dc_expected_norm="$KIT_WORK/$dc_name.expected" 135 dc_raw_stderr="$KIT_WORK/$dc_name.stderr" 136 dc_actual_stderr="$KIT_WORK/$dc_name.stderr.actual" 137 dc_case_stdin="$KIT_WORK/$dc_name.stdin" 138 dc_is_xfail=0 139 dc_reason= 140 dc_ok=1 141 dc_why= 142 143 if [ -e "$dc_xfail_file" ]; then 144 dc_is_xfail=1 145 dc_reason=$(sed -n '1p' "$dc_xfail_file") 146 fi 147 148 # Missing required inputs: a setup failure, routed through xfail when the 149 # case is marked (and not strict), else a plain failure. 150 if [ ! -e "$dc_stdin_file" ]; then 151 dc_ok=0 152 dc_why="missing stdin" 153 elif [ ! -e "$dc_expected" ]; then 154 dc_ok=0 155 dc_why="missing expected" 156 fi 157 if [ "$dc_ok" -eq 0 ]; then 158 if [ "$dc_is_xfail" -eq 1 ]; then 159 kit_xfail "$dc_name" "$dc_why" 160 else 161 kit_fail "$dc_name" "$dc_why" 162 fi 163 return 164 fi 165 166 # Build the dbg argv, templating @CASE@ -> case dir; skip blanks/#comments. 167 set -- 168 dc_args_file="$dc_dir/args" 169 if [ -e "$dc_args_file" ]; then 170 while IFS= read -r dc_arg || [ -n "$dc_arg" ]; do 171 [ -n "$dc_arg" ] || continue 172 case "$dc_arg" in 173 \#*) continue ;; 174 esac 175 dc_arg=$(printf '%s' "$dc_arg" | sed "s|@CASE@|$dc_dir|g") 176 set -- "$@" "$dc_arg" 177 done < "$dc_args_file" 178 fi 179 180 sed "s|@CASE@|$dc_dir|g" "$dc_stdin_file" > "$dc_case_stdin" 181 "$KIT" dbg "$@" < "$dc_case_stdin" > "$dc_raw_stdout" 2> "$dc_raw_stderr" 182 dc_rc=$? 183 184 # Normalize BOTH actual and the (already-normalized, so idempotent) golden. 185 normalize_stdout "$dc_raw_stdout" "$dc_dir" > "$dc_actual_stdout" 186 normalize_stdout "$dc_expected" "$dc_dir" > "$dc_expected_norm" 187 if [ -e "$dc_expected_stderr" ]; then 188 cp "$dc_raw_stderr" "$dc_actual_stderr" 189 else 190 : > "$dc_actual_stderr" 191 fi 192 193 if [ -e "$dc_expected_rc_file" ]; then 194 dc_want_rc=$(sed -n '1p' "$dc_expected_rc_file") 195 if [ "$dc_rc" -ne "$dc_want_rc" ]; then 196 dc_ok=0 197 dc_why="kit dbg exit=$dc_rc (expected $dc_want_rc)" 198 fi 199 elif [ "$dc_rc" -ne 0 ]; then 200 dc_ok=0 201 dc_why="kit dbg exit=$dc_rc" 202 fi 203 if [ "$dc_ok" -eq 1 ] && ! diff -u "$dc_expected_norm" "$dc_actual_stdout" >/dev/null 2>&1; then 204 dc_ok=0 205 dc_why="stdout" 206 fi 207 if [ "$dc_ok" -eq 1 ] && [ -e "$dc_expected_stderr" ]; then 208 if ! diff -u "$dc_expected_stderr" "$dc_actual_stderr" >/dev/null 2>&1; then 209 dc_ok=0 210 dc_why="stderr" 211 fi 212 elif [ "$dc_ok" -eq 1 ] && [ -s "$dc_raw_stderr" ]; then 213 dc_ok=0 214 dc_why="unexpected stderr" 215 fi 216 217 if [ "$dc_ok" -eq 1 ]; then 218 if [ "$dc_is_xfail" -eq 1 ]; then 219 # Marked xfail but passed (a stale xfail marker): kit_xpass is ALWAYS 220 # a failure in the report layer — matching the original's 221 # unconditional xpass-counts-as-fail behavior. 222 kit_xpass "$dc_name" 223 else 224 kit_pass "$dc_name" 225 fi 226 return 227 fi 228 229 if [ "$dc_is_xfail" -eq 1 ]; then 230 if [ -n "$dc_reason" ]; then 231 kit_xfail "$dc_name" "$dc_why: $dc_reason" 232 else 233 kit_xfail "$dc_name" "$dc_why" 234 fi 235 return 236 fi 237 238 kit_fail "$dc_name" "$dc_why" 239 case "$dc_why" in 240 stdout) 241 diff -u "$dc_expected_norm" "$dc_actual_stdout" || true 242 cp "$dc_actual_stdout" "$dc_dir/actual" 2>/dev/null || true 243 ;; 244 stderr) 245 diff -u "$dc_expected_stderr" "$dc_actual_stderr" || true 246 ;; 247 "unexpected stderr") 248 sed 's/^/ | /' "$dc_raw_stderr" 249 ;; 250 kit\ dbg\ exit=*) 251 sed 's/^/ stdout| /' "$dc_raw_stdout" 252 sed 's/^/ stderr| /' "$dc_raw_stderr" 253 ;; 254 esac 255 } 256 257 for case_dir in "$cases_dir"/*; do 258 [ -d "$case_dir" ] || continue 259 name=$(basename "$case_dir") 260 case_selected "$name" || continue 261 dbg_case "$name" "$case_dir" 262 done 263 264 kit_summary dbg 265 kit_exit