kit_sh_report.sh (7728B)
1 # test/lib/kit_sh_report.sh — the single shared reporting layer for every 2 # kit shell test harness (driver-scenario AND corpus runners). 3 # 4 # Sourced. POSIX sh (works under /bin/sh and bash 3.2): counters are scalars, 5 # name lists are space-joined strings, per-lane timing uses eval'd dynamic 6 # vars — no bash arrays here, so the /bin/sh driver harnesses can source it. 7 # (The corpus engine kit_corpus.sh adds the bash-only matrix/dispatch on top.) 8 # 9 # THE KEY SEAM — mode-transparent result verbs. kit_pass/kit_fail/kit_skip/ 10 # kit_skip_na/kit_time normally bump counters and print. But when KIT_EV is set 11 # (the harness ran the case in a background worker), they instead append a 12 # tab-separated event to "$KIT_EV"; the parent later replays those events in 13 # index order through the SAME verbs (KIT_EV unset during replay), so counts, 14 # failing-name order, and printed output are deterministic regardless of 15 # worker completion order. A lane hook calls kit_pass/... and never knows or 16 # cares which mode it is in — that is what makes parallelism a pure flag flip. 17 # 18 # Color is emitted only to a TTY and honors NO_COLOR, so piped/CI output is 19 # plain text. 20 21 if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then 22 _CF_RED=$(printf '\033[31m'); _CF_GRN=$(printf '\033[32m') 23 _CF_YEL=$(printf '\033[33m'); _CF_RST=$(printf '\033[0m') 24 else 25 _CF_RED=; _CF_GRN=; _CF_YEL=; _CF_RST= 26 fi 27 28 # ---- state ----------------------------------------------------------------- 29 KIT_PASS=0 30 KIT_FAIL=0 31 KIT_SKIP=0 32 KIT_SKIP_NA=0 # "not applicable" (e.g. arch tuple filtered) — tracked, never a failure 33 KIT_XFAIL=0 # expected failures (known-red cases that failed as expected) 34 KIT_XPASS=0 # xfail cases that unexpectedly passed (a real failure under KIT_STRICT_XFAIL) 35 KIT_FAILURES= 36 KIT_SKIPS= 37 KIT_TIME_LANES= # space-joined lane ids that have accrued time 38 # KIT_EV — when set, verbs emit events here instead of counting 39 # KIT_SKIP_IS_FAILURE — corpus runners set =1 so SKIP gates the exit (unless KIT_TEST_ALLOW_SKIP=1) 40 41 kit_report_init() { 42 KIT_PASS=0; KIT_FAIL=0; KIT_SKIP=0; KIT_SKIP_NA=0; KIT_XFAIL=0; KIT_XPASS=0 43 KIT_FAILURES=; KIT_SKIPS=; KIT_TIME_LANES= 44 } 45 46 # ---- mode-transparent result verbs ---------------------------------------- 47 48 kit_pass() { 49 if [ -n "${KIT_EV:-}" ]; then printf 'PASS\t%s\n' "$1" >> "$KIT_EV"; return; fi 50 KIT_PASS=$((KIT_PASS + 1)) 51 printf ' %sPASS%s %s\n' "$_CF_GRN" "$_CF_RST" "$1" 52 } 53 54 # kit_fail NAME [DETAIL] : NAME is recorded in the failures list; DETAIL (single 55 # line) is shown only on the printed line. 56 kit_fail() { 57 if [ -n "${KIT_EV:-}" ]; then printf 'FAIL\t%s\t%s\n' "$1" "${2:-}" >> "$KIT_EV"; return; fi 58 KIT_FAIL=$((KIT_FAIL + 1)) 59 KIT_FAILURES="$KIT_FAILURES $1" 60 if [ -n "${2:-}" ]; then 61 printf ' %sFAIL%s %s (%s)\n' "$_CF_RED" "$_CF_RST" "$1" "$2" 62 else 63 printf ' %sFAIL%s %s\n' "$_CF_RED" "$_CF_RST" "$1" 64 fi 65 } 66 67 kit_skip() { 68 if [ -n "${KIT_EV:-}" ]; then printf 'SKIP\t%s\t%s\n' "$1" "${2:-}" >> "$KIT_EV"; return; fi 69 KIT_SKIP=$((KIT_SKIP + 1)) 70 KIT_SKIPS="$KIT_SKIPS $1" 71 if [ -n "${2:-}" ]; then 72 printf ' %sSKIP%s %s — %s\n' "$_CF_YEL" "$_CF_RST" "$1" "$2" 73 else 74 printf ' %sSKIP%s %s\n' "$_CF_YEL" "$_CF_RST" "$1" 75 fi 76 } 77 78 # kit_skip_na : "not applicable" — a case/lane that structurally does not apply 79 # to the current target tuple. Counted separately and NEVER gates the exit. 80 kit_skip_na() { 81 if [ -n "${KIT_EV:-}" ]; then printf 'SKIP_NA\t%s\t%s\n' "$1" "${2:-}" >> "$KIT_EV"; return; fi 82 KIT_SKIP_NA=$((KIT_SKIP_NA + 1)) 83 # quiet by default; uncomment for verbose n/a tracing 84 [ -n "${KIT_VERBOSE_NA:-}" ] && printf ' %sn/a%s %s — %s\n' "$_CF_YEL" "$_CF_RST" "$1" "${2:-}" 85 return 0 86 } 87 88 # kit_xfail NAME [REASON] : an expected failure (a known-red case that failed as 89 # expected). Counted as xfail and does NOT gate the exit — UNLESS 90 # KIT_STRICT_XFAIL=1 (e.g. dbg's DBG_STRICT_XFAIL), under which even expected 91 # failures are promoted to a hard failure (no known-red allowed in strict mode). 92 kit_xfail() { 93 if [ -n "${KIT_EV:-}" ]; then printf 'XFAIL\t%s\t%s\n' "$1" "${2:-}" >> "$KIT_EV"; return; fi 94 if [ "${KIT_STRICT_XFAIL:-0}" = "1" ]; then 95 kit_fail "$1" "expected failure not allowed under strict xfail${2:+ ($2)}"; return 96 fi 97 KIT_XFAIL=$((KIT_XFAIL + 1)) 98 printf ' %sXFAIL%s %s%s\n' "$_CF_YEL" "$_CF_RST" "$1" "${2:+ — $2}" 99 } 100 101 # kit_xpass NAME : a case marked xfail that UNEXPECTEDLY passed — the xfail 102 # marker is stale and should be removed. This is ALWAYS a failure (kit_exit 103 # gates on KIT_XPASS), matching the lit/DejaGnu convention and dbg's original 104 # always-count-xpass-as-failure behavior. Tracked in its own bucket so the 105 # summary distinguishes it from a plain FAIL. 106 kit_xpass() { 107 if [ -n "${KIT_EV:-}" ]; then printf 'XPASS\t%s\n' "$1" >> "$KIT_EV"; return; fi 108 KIT_XPASS=$((KIT_XPASS + 1)) 109 printf ' %sXPASS%s %s (unexpected pass — xfail marker is stale)\n' "$_CF_RED" "$_CF_RST" "$1" 110 } 111 112 # kit_time LANE MS : accumulate per-lane wall time for the summary's Time line. 113 kit_time() { 114 if [ -n "${KIT_EV:-}" ]; then printf 'TIME\t%s\t%s\n' "$1" "$2" >> "$KIT_EV"; return; fi 115 case " $KIT_TIME_LANES " in 116 *" $1 "*) ;; 117 *) KIT_TIME_LANES="$KIT_TIME_LANES $1" ;; 118 esac 119 eval "kit_t_$1=\$(( \${kit_t_$1:-0} + $2 ))" 120 } 121 122 # ---- summary + exit -------------------------------------------------------- 123 124 # kit_summary LABEL : unified summary — failing/skipped name lists, the 125 # "P pass, F fail, S skip" line (+ " N n/a" when any), and a per-lane Time 126 # line when timing was recorded. 127 kit_summary() { 128 if [ -n "$KIT_FAILURES" ]; then printf '\n%s: failures:%s\n' "$1" "$KIT_FAILURES"; fi 129 if [ -n "$KIT_SKIPS" ]; then printf '%s: skipped:%s\n' "$1" "$KIT_SKIPS"; fi 130 kit_sm_extra= 131 [ "$KIT_SKIP_NA" -gt 0 ] && kit_sm_extra="$kit_sm_extra, $KIT_SKIP_NA n/a" 132 [ "$KIT_XFAIL" -gt 0 ] && kit_sm_extra="$kit_sm_extra, $KIT_XFAIL xfail" 133 [ "$KIT_XPASS" -gt 0 ] && kit_sm_extra="$kit_sm_extra, $KIT_XPASS xpass" 134 printf '\n%s: %d pass, %d fail, %d skip%s\n' "$1" "$KIT_PASS" "$KIT_FAIL" "$KIT_SKIP" "$kit_sm_extra" 135 if [ -n "$KIT_TIME_LANES" ]; then 136 kit_sm_t="$1: time" 137 for kit_sm_l in $KIT_TIME_LANES; do 138 eval "kit_sm_v=\${kit_t_$kit_sm_l:-0}" 139 kit_sm_t="$kit_sm_t $kit_sm_l=${kit_sm_v}ms" 140 done 141 printf '%s\n' "$kit_sm_t" 142 fi 143 } 144 145 # kit_exit : exit 1 on any failure or any XPASS (stale xfail); also exit 1 on 146 # skips when KIT_SKIP_IS_FAILURE=1 unless KIT_TEST_ALLOW_SKIP=1. SKIP_NA and 147 # XFAIL never gate (XFAIL only via kit_xfail's strict promotion to FAIL). 148 kit_exit() { 149 [ "$KIT_FAIL" -gt 0 ] && exit 1 150 [ "${KIT_XPASS:-0}" -gt 0 ] && exit 1 151 if [ "${KIT_SKIP_IS_FAILURE:-0}" = "1" ] && [ "$KIT_SKIP" -gt 0 ] && 152 [ "${KIT_TEST_ALLOW_SKIP:-0}" != "1" ]; then 153 exit 1 154 fi 155 exit 0 156 } 157 158 # ---- harness setup helpers ------------------------------------------------- 159 160 # kit_require_kit LABEL : ensure $KIT points at an executable, else exit 2. 161 kit_require_kit() { 162 if [ ! -x "${KIT:-}" ]; then 163 echo "$1: kit binary not found at ${KIT:-<unset>}" >&2 164 exit 2 165 fi 166 } 167 168 # kit_workdir TOOL : mktemp -d under TMPDIR with the canonical name, store it in 169 # the global KIT_WORK, and install an EXIT trap removing it. Call as 170 # `kit_workdir TOOL` (NOT `KIT_WORK=$(kit_workdir TOOL)`): the trap must be set in 171 # the caller's shell, so the function may not run inside a command-substitution 172 # subshell — there the EXIT trap fires (and deletes KIT_WORK) the moment the 173 # subshell closes, before the harness ever uses it. NOTE: the EXIT trap is a 174 # singleton; call once per harness. 175 kit_workdir() { 176 KIT_WORK=$(mktemp -d "${TMPDIR:-/tmp}/kit-$1-test.XXXXXX") 177 trap 'rm -rf "$KIT_WORK"' EXIT 178 }