harness.sh (11739B)
1 #!/bin/sh 2 3 # Shared black-box release-audit primitives. This file is sourced by run.sh. 4 5 HARNESS_PASS=0 6 HARNESS_FAIL=0 7 HARNESS_BLOCKED=0 8 HARNESS_NOT_APPLICABLE=0 9 HARNESS_SELECTED=0 10 11 audit_match_filter() { 12 audit_match_values=$1 13 audit_match_spec=$2 14 15 [ -z "$audit_match_spec" ] && return 0 16 17 audit_match_old_ifs=$IFS 18 IFS=, 19 for audit_match_value in $audit_match_values; do 20 case "$audit_match_value" in 21 all|any) 22 IFS=$audit_match_old_ifs 23 return 0 24 ;; 25 esac 26 for audit_match_pattern in $audit_match_spec; do 27 IFS=$audit_match_old_ifs 28 case "$audit_match_value" in 29 $audit_match_pattern) return 0 ;; 30 esac 31 IFS=, 32 done 33 done 34 IFS=$audit_match_old_ifs 35 return 1 36 } 37 38 audit_case_selected() { 39 audit_match_filter "$CASE_MODULE" "$HARNESS_FILTER_MODULE" || return 1 40 audit_match_filter "$CASE_COMMAND" "$HARNESS_FILTER_COMMAND" || return 1 41 audit_match_filter "$CASE_WORKFLOW" "$HARNESS_FILTER_WORKFLOW" || return 1 42 audit_match_filter "$CASE_TARGET" "$HARNESS_FILTER_TARGET" || return 1 43 audit_match_filter "$CASE_ARCH" "$HARNESS_FILTER_ARCH" || return 1 44 audit_match_filter "$CASE_FORMAT" "$HARNESS_FILTER_FORMAT" || return 1 45 audit_match_filter "$CASE_SEVERITY" "$HARNESS_FILTER_SEVERITY" || return 1 46 return 0 47 } 48 49 audit_case_start() { 50 CASE_ID=$1 51 CASE_MODULE=$2 52 CASE_COMMAND=$3 53 CASE_WORKFLOW=$4 54 CASE_TARGET=$5 55 CASE_ARCH=$6 56 CASE_FORMAT=$7 57 CASE_SEVERITY=$8 58 CASE_FAILURES=0 59 CASE_FINISHED=0 60 CASE_ACTIVE_DIR= 61 CASE_STEP_COUNT=0 62 63 audit_case_selected || return 1 64 65 HARNESS_SELECTED=$((HARNESS_SELECTED + 1)) 66 CASE_DIR=$HARNESS_RUN_DIR/cases/$CASE_ID 67 if [ -e "$CASE_DIR" ]; then 68 printf 'audit: duplicate case id or existing case directory: %s\n' "$CASE_ID" >&2 69 exit 2 70 fi 71 72 mkdir -p \ 73 "$CASE_DIR/artifacts" \ 74 "$CASE_DIR/cache" \ 75 "$CASE_DIR/config" \ 76 "$CASE_DIR/home" \ 77 "$CASE_DIR/install" \ 78 "$CASE_DIR/tmp" \ 79 "$CASE_DIR/work" 80 : > "$CASE_DIR/stdin" 81 : > "$CASE_DIR/stdout" 82 : > "$CASE_DIR/stderr" 83 : > "$CASE_DIR/assertions.log" 84 printf '%s\n' NOT_RUN > "$CASE_DIR/command" 85 printf '%s\n' NOT_RUN > "$CASE_DIR/exit" 86 87 { 88 printf 'id=%s\n' "$CASE_ID" 89 printf 'module=%s\n' "$CASE_MODULE" 90 printf 'command=%s\n' "$CASE_COMMAND" 91 printf 'workflow=%s\n' "$CASE_WORKFLOW" 92 printf 'target=%s\n' "$CASE_TARGET" 93 printf 'arch=%s\n' "$CASE_ARCH" 94 printf 'format=%s\n' "$CASE_FORMAT" 95 printf 'severity=%s\n' "$CASE_SEVERITY" 96 } > "$CASE_DIR/metadata" 97 return 0 98 } 99 100 audit_case_exec() { 101 audit_exec_program=$1 102 shift 103 104 { 105 printf 'cwd=%s\n' "$CASE_DIR/work" 106 printf 'HOME=%s\n' "$CASE_DIR/home" 107 printf 'XDG_CACHE_HOME=%s\n' "$CASE_DIR/cache" 108 printf 'XDG_CONFIG_HOME=%s\n' "$CASE_DIR/config" 109 printf 'XDG_DATA_HOME=%s\n' "$CASE_DIR/home/.local/share" 110 printf 'TMPDIR=%s\n' "$CASE_DIR/tmp" 111 printf 'PATH=%s\n' "$HARNESS_PATH" 112 printf 'argv[0]=%s\n' "$audit_exec_program" 113 audit_exec_index=1 114 for audit_exec_arg in "$@"; do 115 printf 'argv[%s]=%s\n' "$audit_exec_index" "$audit_exec_arg" 116 audit_exec_index=$((audit_exec_index + 1)) 117 done 118 } > "$CASE_DIR/command" 119 120 ( 121 cd "$CASE_DIR/work" || exit 125 122 env -i \ 123 HOME="$CASE_DIR/home" \ 124 XDG_CACHE_HOME="$CASE_DIR/cache" \ 125 XDG_CONFIG_HOME="$CASE_DIR/config" \ 126 XDG_DATA_HOME="$CASE_DIR/home/.local/share" \ 127 TMPDIR="$CASE_DIR/tmp" \ 128 PATH="$HARNESS_PATH" \ 129 KIT_AUDIT_INSTALL="$CASE_DIR/install" \ 130 KIT_AUDIT_ARTIFACTS="$CASE_DIR/artifacts" \ 131 "$audit_exec_program" "$@" \ 132 < "$CASE_DIR/stdin" 133 ) > "$CASE_DIR/stdout" 2> "$CASE_DIR/stderr" 134 audit_exec_status=$? 135 printf '%s\n' "$audit_exec_status" > "$CASE_DIR/exit" 136 CASE_ACTIVE_DIR=$CASE_DIR 137 return 0 138 } 139 140 audit_case_exec_kit() { 141 audit_case_exec "$HARNESS_KIT" "$@" 142 } 143 144 audit_case_step_exec() { 145 audit_step_name=$1 146 audit_step_program=$2 147 shift 2 148 149 case "$audit_step_name" in 150 ''|*[!a-zA-Z0-9_.-]*) 151 printf 'audit: invalid step name: %s\n' "$audit_step_name" >&2 152 exit 2 153 ;; 154 esac 155 156 audit_step_dir=$CASE_DIR/steps/$audit_step_name 157 if [ -e "$audit_step_dir" ]; then 158 printf 'audit: duplicate step name in %s: %s\n' "$CASE_ID" "$audit_step_name" >&2 159 exit 2 160 fi 161 mkdir -p "$audit_step_dir" 162 163 if [ "$CASE_STEP_COUNT" -eq 0 ]; then 164 printf '%s\n' 'MULTI_STEP: see steps/' > "$CASE_DIR/command" 165 printf '%s\n' MULTI_STEP > "$CASE_DIR/exit" 166 fi 167 CASE_STEP_COUNT=$((CASE_STEP_COUNT + 1)) 168 169 { 170 printf 'cwd=%s\n' "$CASE_DIR/work" 171 printf 'HOME=%s\n' "$CASE_DIR/home" 172 printf 'XDG_CACHE_HOME=%s\n' "$CASE_DIR/cache" 173 printf 'XDG_CONFIG_HOME=%s\n' "$CASE_DIR/config" 174 printf 'XDG_DATA_HOME=%s\n' "$CASE_DIR/home/.local/share" 175 printf 'TMPDIR=%s\n' "$CASE_DIR/tmp" 176 printf 'PATH=%s\n' "$HARNESS_PATH" 177 printf 'argv[0]=%s\n' "$audit_step_program" 178 audit_step_index=1 179 for audit_step_arg in "$@"; do 180 printf 'argv[%s]=%s\n' "$audit_step_index" "$audit_step_arg" 181 audit_step_index=$((audit_step_index + 1)) 182 done 183 } > "$audit_step_dir/command" 184 185 ( 186 cd "$CASE_DIR/work" || exit 125 187 env -i \ 188 HOME="$CASE_DIR/home" \ 189 XDG_CACHE_HOME="$CASE_DIR/cache" \ 190 XDG_CONFIG_HOME="$CASE_DIR/config" \ 191 XDG_DATA_HOME="$CASE_DIR/home/.local/share" \ 192 TMPDIR="$CASE_DIR/tmp" \ 193 PATH="$HARNESS_PATH" \ 194 KIT_AUDIT_INSTALL="$CASE_DIR/install" \ 195 KIT_AUDIT_ARTIFACTS="$CASE_DIR/artifacts" \ 196 KIT_AUDIT_SENTINEL_LOG="$CASE_DIR/artifacts/sentinel.log" \ 197 "$audit_step_program" "$@" \ 198 < "$CASE_DIR/stdin" 199 ) > "$audit_step_dir/stdout" 2> "$audit_step_dir/stderr" 200 audit_step_status=$? 201 printf '%s\n' "$audit_step_status" > "$audit_step_dir/exit" 202 CASE_ACTIVE_DIR=$audit_step_dir 203 return 0 204 } 205 206 audit_case_step_exec_kit() { 207 audit_step_kit_name=$1 208 shift 209 audit_case_step_exec "$audit_step_kit_name" "$HARNESS_KIT" "$@" 210 } 211 212 audit_assert_dir() { 213 if [ -n "${CASE_ACTIVE_DIR:-}" ]; then 214 printf '%s\n' "$CASE_ACTIVE_DIR" 215 else 216 printf '%s\n' "$CASE_DIR" 217 fi 218 } 219 220 audit_read_exit() { 221 CASE_ACTUAL_EXIT=NOT_RUN 222 audit_read_dir=$(audit_assert_dir) 223 IFS= read -r CASE_ACTUAL_EXIT < "$audit_read_dir/exit" || : 224 } 225 226 audit_check_pass() { 227 printf 'PASS\t%s\n' "$1" >> "$CASE_DIR/assertions.log" 228 } 229 230 audit_check_fail() { 231 CASE_FAILURES=$((CASE_FAILURES + 1)) 232 printf 'FAIL\t%s\n' "$1" >> "$CASE_DIR/assertions.log" 233 } 234 235 audit_expect_exit() { 236 audit_read_exit 237 if [ "$CASE_ACTUAL_EXIT" = "$1" ]; then 238 audit_check_pass "exit status is $1" 239 else 240 audit_check_fail "expected exit $1, got $CASE_ACTUAL_EXIT" 241 fi 242 } 243 244 audit_expect_exit_one_of() { 245 audit_read_exit 246 audit_exit_wanted=$1 247 audit_exit_old_ifs=$IFS 248 IFS=, 249 for audit_exit_value in $audit_exit_wanted; do 250 IFS=$audit_exit_old_ifs 251 if [ "$CASE_ACTUAL_EXIT" = "$audit_exit_value" ]; then 252 audit_check_pass "exit status $CASE_ACTUAL_EXIT is one of $audit_exit_wanted" 253 return 0 254 fi 255 IFS=, 256 done 257 IFS=$audit_exit_old_ifs 258 audit_check_fail "expected exit in $audit_exit_wanted, got $CASE_ACTUAL_EXIT" 259 } 260 261 audit_expect_stdout_nonempty() { 262 audit_expect_dir=$(audit_assert_dir) 263 if [ -s "$audit_expect_dir/stdout" ]; then 264 audit_check_pass "stdout is non-empty" 265 else 266 audit_check_fail "expected non-empty stdout" 267 fi 268 } 269 270 audit_expect_stderr_nonempty() { 271 audit_expect_dir=$(audit_assert_dir) 272 if [ -s "$audit_expect_dir/stderr" ]; then 273 audit_check_pass "stderr is non-empty" 274 else 275 audit_check_fail "expected non-empty stderr" 276 fi 277 } 278 279 audit_expect_stderr_empty() { 280 audit_expect_dir=$(audit_assert_dir) 281 if [ ! -s "$audit_expect_dir/stderr" ]; then 282 audit_check_pass "stderr is empty" 283 else 284 audit_check_fail "expected empty stderr" 285 fi 286 } 287 288 audit_expect_stdout_contains() { 289 audit_expect_dir=$(audit_assert_dir) 290 if grep -F -q -- "$1" "$audit_expect_dir/stdout"; then 291 audit_check_pass "stdout contains: $1" 292 else 293 audit_check_fail "stdout does not contain: $1" 294 fi 295 } 296 297 audit_expect_stdout_regex() { 298 audit_expect_dir=$(audit_assert_dir) 299 if grep -E -q -- "$1" "$audit_expect_dir/stdout"; then 300 audit_check_pass "stdout matches: $1" 301 else 302 audit_check_fail "stdout does not match: $1" 303 fi 304 } 305 306 audit_reject_stdout_regex() { 307 audit_expect_dir=$(audit_assert_dir) 308 if grep -E -q -- "$1" "$audit_expect_dir/stdout"; then 309 audit_check_fail "stdout unexpectedly matches: $1" 310 else 311 audit_check_pass "stdout does not match: $1" 312 fi 313 } 314 315 audit_expect_stdout_equals_file() { 316 audit_expect_dir=$(audit_assert_dir) 317 if cmp -s "$audit_expect_dir/stdout" "$1"; then 318 audit_check_pass "stdout matches $1" 319 else 320 audit_check_fail "stdout differs from $1" 321 fi 322 } 323 324 audit_expect_stdout_equals() { 325 audit_expect_dir=$(audit_assert_dir) 326 audit_expect_text=$1 327 audit_expect_actual=$(cat "$audit_expect_dir/stdout") 328 if [ "$audit_expect_actual" = "$audit_expect_text" ]; then 329 audit_check_pass "stdout equals: $audit_expect_text" 330 else 331 audit_check_fail "stdout differs from expected text: $audit_expect_text" 332 fi 333 } 334 335 audit_expect_file_exists() { 336 if [ -f "$1" ]; then 337 audit_check_pass "file exists: $1" 338 else 339 audit_check_fail "file does not exist: $1" 340 fi 341 } 342 343 audit_expect_file_nonempty() { 344 if [ -s "$1" ]; then 345 audit_check_pass "file is non-empty: $1" 346 else 347 audit_check_fail "file is missing or empty: $1" 348 fi 349 } 350 351 audit_expect_file_empty() { 352 if [ -f "$1" ] && [ ! -s "$1" ]; then 353 audit_check_pass "file is empty: $1" 354 elif [ ! -e "$1" ]; then 355 audit_check_pass "file was not created: $1" 356 else 357 audit_check_fail "file is unexpectedly non-empty: $1" 358 fi 359 } 360 361 audit_record_result() { 362 audit_result_state=$1 363 audit_result_reason=$2 364 printf '%s\n' "$audit_result_state" > "$CASE_DIR/state" 365 printf '%s\n' "$audit_result_reason" > "$CASE_DIR/reason" 366 printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ 367 "$audit_result_state" "$CASE_SEVERITY" "$CASE_MODULE" \ 368 "$CASE_COMMAND" "$CASE_WORKFLOW" "$CASE_TARGET" "$CASE_ARCH" \ 369 "$CASE_FORMAT" "$CASE_ID" "$audit_result_reason" \ 370 >> "$HARNESS_SUMMARY_TSV" 371 372 case "$audit_result_state" in 373 PASS) HARNESS_PASS=$((HARNESS_PASS + 1)) ;; 374 FAIL) HARNESS_FAIL=$((HARNESS_FAIL + 1)) ;; 375 BLOCKED) HARNESS_BLOCKED=$((HARNESS_BLOCKED + 1)) ;; 376 NOT_APPLICABLE) HARNESS_NOT_APPLICABLE=$((HARNESS_NOT_APPLICABLE + 1)) ;; 377 *) printf 'audit: internal error: bad result state %s\n' "$audit_result_state" >&2; exit 2 ;; 378 esac 379 CASE_FINISHED=1 380 } 381 382 audit_case_finish() { 383 if [ "$CASE_FAILURES" -eq 0 ]; then 384 audit_record_result PASS "all assertions passed" 385 else 386 audit_record_result FAIL "$CASE_FAILURES assertion(s) failed; see assertions.log" 387 fi 388 } 389 390 audit_case_blocked() { 391 audit_record_result BLOCKED "$1" 392 } 393 394 audit_case_not_applicable() { 395 audit_record_result NOT_APPLICABLE "$1" 396 }