kit_differential.sh (2053B)
1 # test/lib/kit_differential.sh — Type D (differential / agreement) helpers. 2 # 3 # Sourced AFTER kit_sh_report.sh (a D harness sources both). Type D's oracle 4 # is AGREEMENT, not a per-case fixed expectation: 5 # - baseline mode: a freshly-generated, normalized report must equal a 6 # checked-in baseline snapshot (which documents a known backlog). 7 # KIT_DIFF_UPDATE=1 regenerates the baseline instead of comparing. 8 # - reference mode: two independent producers (e.g. kit vs a third-party 9 # tool) must agree byte-for-byte, with explicit equivalence-skips for 10 # known-benign divergences (the caller kit_skip's those before comparing). 11 # Both record exactly one verdict through kit_pass/kit_fail/kit_skip, so summary, 12 # exit, color, and skip-gating match every other harness. A D harness still 13 # calls kit_report_init at the top and kit_summary LABEL + kit_exit at the end. 14 15 # kit_diff_baseline LABEL BASELINE ACTUAL 16 # ACTUAL is a freshly produced + already-normalized report file; it must 17 # equal the checked-in BASELINE. KIT_DIFF_UPDATE=1 rewrites BASELINE from 18 # ACTUAL and passes (the regen path). On mismatch, fail and show the unified 19 # delta so the new asymmetries are visible. 20 kit_diff_baseline() { 21 if [ "${KIT_DIFF_UPDATE:-0}" = "1" ]; then 22 cp "$3" "$2" 23 kit_pass "$1 (baseline updated)" 24 return 25 fi 26 if [ ! -f "$2" ]; then 27 kit_fail "$1" "missing baseline $(basename "$2")" 28 return 29 fi 30 if diff -u "$2" "$3" >/dev/null 2>&1; then 31 kit_pass "$1" 32 else 33 kit_fail "$1" "report drifted from baseline (run with KIT_DIFF_UPDATE=1 to refresh)" 34 diff -u "$2" "$3" || true 35 fi 36 } 37 38 # kit_diff_agree LABEL A B 39 # A and B (two producers' outputs over the same input) must be byte-equal. 40 # The caller is responsible for equivalence-skips: kit_skip the case BEFORE 41 # calling this when a known-benign divergence applies, and skip the call. 42 kit_diff_agree() { 43 if cmp -s "$2" "$3"; then 44 kit_pass "$1" 45 else 46 kit_fail "$1" "producers disagree" 47 cmp "$2" "$3" 2>&1 | head -3 || true 48 fi 49 }