cc_bench.sh (16344B)
1 #!/usr/bin/env bash 2 # -O0 C compile + link scaling benchmark for kit. 3 # 4 # Goal: kit is meant to be the *fastest* C compiler. This harness sweeps each 5 # input axis (see scripts/cc_bench_gen.py --list) over a geometric size series, 6 # times `kit cc -O0 -c/-E` (compile) and `kit ld` (link) best-of-N, records a 7 # clang -O0 reference for context, and samples the largest input per axis to 8 # produce a flat self-time hotspot table. The reporter (cc_bench_report.py) fits 9 # a scaling exponent per axis: ~1.0 is the linear goal, >=1.4 is an O(n^2) trap. 10 # 11 # Output under build/bench/cc/: scaling.csv (raw), plus scaling.md + hotspots.md 12 # written by the reporter. 13 # 14 # Env knobs: 15 # KIT kit binary (default build/bench/kit, else build/release/kit) 16 # CLANG clang binary (default clang) 17 # KIT_CC_BENCH_AXES space-separated subset of axes (default: all) 18 # KIT_CC_BENCH_SIZES override size series for ALL axes (default: per-axis) 19 # KIT_CC_BENCH_REPEATS best-of-N timing repeats (default 3) 20 # KIT_CC_BENCH_SKIP_CLANG 1 to skip the clang reference (default 0) 21 # KIT_CC_BENCH_SAMPLE 1 to sample hotspots (default 1) 22 # KIT_CC_BENCH_SAMPLE_AXES subset of axes to sample (default: all run) 23 # KIT_CC_BENCH_SAMPLE_MIN_MS skip sampling if max-size run is faster (default 400) 24 # KIT_CC_BENCH_DTRACE 1 to use dtrace flat histogram instead of sample (needs sudo) 25 # KIT_CC_BENCH_OUT output root (default build/bench/cc) 26 # KIT_SYSROOT sysroot passed to kit (default: macOS SDK via xcrun) 27 set -uo pipefail 28 29 ROOT="$(cd "$(dirname "$0")/.." && pwd)" 30 GEN="$ROOT/scripts/cc_bench_gen.py" 31 REPORT="$ROOT/scripts/cc_bench_report.py" 32 33 KIT="${KIT:-}" 34 if [ -z "$KIT" ]; then 35 if [ -x "$ROOT/build/bench/kit" ]; then KIT="$ROOT/build/bench/kit" 36 else KIT="$ROOT/build/release/kit"; fi 37 fi 38 CLANG="${CLANG:-clang}" 39 OUT_DIR="${KIT_CC_BENCH_OUT:-$ROOT/build/bench/cc}" 40 REPEATS="${KIT_CC_BENCH_REPEATS:-3}" 41 SKIP_CLANG="${KIT_CC_BENCH_SKIP_CLANG:-0}" 42 DO_SAMPLE="${KIT_CC_BENCH_SAMPLE:-1}" 43 SAMPLE_MIN_MS="${KIT_CC_BENCH_SAMPLE_MIN_MS:-400}" 44 USE_DTRACE="${KIT_CC_BENCH_DTRACE:-0}" 45 # Adaptive cap: once a size's kit time exceeds this, stop growing that axis. Keeps 46 # the run bounded even when an axis is badly superlinear (we still collect the 47 # smaller points needed to fit + flag the exponent). 0 disables. 48 MAX_MS="${KIT_CC_BENCH_MAX_MS:-4000}" 49 50 # kit needs a sysroot to find the host libc for linking. Match opt_bench.sh. 51 KIT_SYSROOT="${KIT_SYSROOT:-}" 52 if [ -z "$KIT_SYSROOT" ] && [ "$(uname -s)" = "Darwin" ] && command -v xcrun >/dev/null 2>&1; then 53 KIT_SYSROOT="$(xcrun --show-sdk-path 2>/dev/null || true)" 54 fi 55 KIT_SYSROOT_ARGS=() 56 [ -n "$KIT_SYSROOT" ] && KIT_SYSROOT_ARGS=(--sysroot "$KIT_SYSROOT") 57 # Homebrew clang ships a default config pointing at a (often nonexistent) SDK; 58 # pass the real one so the reference build is clean. Compile needs no headers 59 # (generated sources have none), but this silences the stale-sysroot warning. 60 CLANG_SYSROOT_ARGS=() 61 [ -n "$KIT_SYSROOT" ] && CLANG_SYSROOT_ARGS=(-isysroot "$KIT_SYSROOT") 62 63 CSV="$OUT_DIR/scaling.csv" 64 GEN_DIR="$OUT_DIR/gen" 65 RAW_DIR="$OUT_DIR/raw" 66 LOG_DIR="$OUT_DIR/logs" 67 68 if [ ! -x "$KIT" ]; then 69 printf 'cc-bench: kit binary not found/executable: %s\n' "$KIT" >&2 70 printf 'cc-bench: run `make bench-cc` (builds a profiling kit) or set KIT=...\n' >&2 71 exit 2 72 fi 73 74 rm -rf "$GEN_DIR" "$RAW_DIR" "$LOG_DIR" 75 mkdir -p "$OUT_DIR" "$GEN_DIR" "$RAW_DIR" "$LOG_DIR" 76 77 # --------------------------------------------------------------------------- 78 # Timing: run a command, capturing rc + wall-clock ms of exactly the child. 79 # python times only the subprocess, so its own startup is not counted. 80 # --------------------------------------------------------------------------- 81 time_cmd() { # $1=out $2=err ; rest=cmd ; sets TIME_RC, TIME_MS 82 local out="$1" err="$2"; shift 2 83 local line 84 line=$(python3 - "$out" "$err" "$@" <<'PY' 85 import subprocess, sys, time 86 out, err = sys.argv[1], sys.argv[2] 87 cmd = sys.argv[3:] 88 with open(out, "wb") as o, open(err, "wb") as e: 89 t0 = time.monotonic_ns() 90 rc = subprocess.call(cmd, stdout=o, stderr=e) 91 t1 = time.monotonic_ns() 92 print("%d %.3f" % (rc, (t1 - t0) / 1e6)) 93 PY 94 ) 95 TIME_RC="${line%% *}" 96 TIME_MS="${line#* }" 97 } 98 99 min_ms() { # echo min of $1,$2 (either may be "" meaning unset) 100 awk -v a="$1" -v b="$2" 'BEGIN{ 101 if (a=="") {print b; exit} 102 if (b=="") {print a; exit} 103 print (b+0 < a+0) ? b : a 104 }' 105 } 106 107 csv_field() { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; } 108 record() { # axis phase tool mode n unit time_ms status exit note 109 { csv_field "$1"; printf ','; csv_field "$2"; printf ','; csv_field "$3"; printf ','; 110 csv_field "$4"; printf ','; csv_field "$5"; printf ','; csv_field "$6"; printf ','; 111 csv_field "$7"; printf ','; csv_field "$8"; printf ','; csv_field "$9"; printf ','; 112 csv_field "${10}"; printf '\n'; } >>"$CSV" 113 } 114 115 # best-of-N: run "$@" REPEATS times, set BEST_MS (min over reps) + LAST_RC. 116 best_of() { # out_base, cmd... 117 local out_base="$1"; shift 118 local rep best="" 119 LAST_RC=0 120 for rep in $(seq 1 "$REPEATS"); do 121 time_cmd "$out_base.$rep.out" "$out_base.$rep.err" "$@" 122 LAST_RC="$TIME_RC" 123 [ "$TIME_RC" -ne 0 ] && { best=""; return 1; } 124 best="$(min_ms "$best" "$TIME_MS")" 125 done 126 BEST_MS="$best" 127 return 0 128 } 129 130 # --------------------------------------------------------------------------- 131 # kit / clang command builders (mode -> argv). kit links via `kit ld ... -l c`. 132 # --------------------------------------------------------------------------- 133 kit_compile_cmd() { KIT_CMD=("$KIT" cc -O0 -c "$1" -o "$2"); } 134 kit_preprocess_cmd() { KIT_CMD=("$KIT" cc -O0 -E "$@"); } # args: [-I dir] src -o out 135 kit_link_cmd() { KIT_CMD=("$KIT" ld "$@" -l c "${KIT_SYSROOT_ARGS[@]}"); } # args: objs... -o app 136 clang_compile_cmd() { CLANG_CMD=("$CLANG" "${CLANG_SYSROOT_ARGS[@]}" -O0 -c "$1" -o "$2"); } 137 clang_preprocess_cmd() { CLANG_CMD=("$CLANG" "${CLANG_SYSROOT_ARGS[@]}" -O0 -E "$@"); } 138 clang_link_cmd() { CLANG_CMD=("$CLANG" "${CLANG_SYSROOT_ARGS[@]}" "$@"); } # kit objs... -o app 139 140 # --------------------------------------------------------------------------- 141 # Axis runners. Each generates the instance, times kit (+clang ref), checks 142 # correctness, records rows. Sets KIT_BEST_MS for the sampling gate. 143 # --------------------------------------------------------------------------- 144 read_manifest() { # $1=manifest.json $2=key -> echo value (python) 145 python3 -c 'import json,sys; v=json.load(open(sys.argv[1])).get(sys.argv[2],""); print(v if not isinstance(v,list) else " ".join(map(str,v)))' "$1" "$2" 146 } 147 148 run_compile_axis() { # axis mode unit n dir 149 local axis="$1" mode="$2" unit="$3" n="$4" dir="$5" 150 local src="$dir/gen.c" incdir status="OK" 151 incdir="$(read_manifest "$dir/manifest.json" incdir)" 152 local inc_args=(); [ -n "$incdir" ] && inc_args=(-I "$dir/$incdir") 153 local logb="$LOG_DIR/$axis.n$n.kit" 154 155 # kit 156 if [ "$mode" = preprocess ]; then 157 kit_preprocess_cmd "${inc_args[@]}" "$src" -o "$dir/gen.i" 158 else 159 kit_compile_cmd "$src" "$dir/gen.o" 160 fi 161 KIT_BEST_MS="" 162 if best_of "$logb" "${KIT_CMD[@]}"; then 163 # correctness: output produced 164 local artifact="$dir/gen.o"; [ "$mode" = preprocess ] && artifact="$dir/gen.i" 165 [ -s "$artifact" ] || status="OUTPUT_FAIL" 166 KIT_BEST_MS="$BEST_MS" 167 else 168 status="KIT_FAIL" 169 fi 170 record "$axis" compile kit "$mode" "$n" "$unit" "${BEST_MS:-NA}" "$status" "$LAST_RC" "$logb.1.err" 171 172 # clang reference 173 if [ "$SKIP_CLANG" != 1 ]; then 174 local clogb="$LOG_DIR/$axis.n$n.clang" cstatus="OK" 175 if [ "$mode" = preprocess ]; then 176 clang_preprocess_cmd "${inc_args[@]}" "$src" -o "$dir/gen.clang.i" 177 else 178 clang_compile_cmd "$src" "$dir/gen.clang.o" 179 fi 180 if best_of "$clogb" "${CLANG_CMD[@]}"; then :; else cstatus="CLANG_FAIL"; fi 181 record "$axis" compile clang "$mode" "$n" "$unit" "${BEST_MS:-NA}" "$cstatus" "$LAST_RC" "$clogb.1.err" 182 fi 183 } 184 185 run_link_axis() { # axis unit n dir 186 local axis="$1" unit="$2" n="$3" dir="$4" status="OK" 187 local prebuild link_order expected logb="$LOG_DIR/$axis.n$n.kit" 188 prebuild="$(read_manifest "$dir/manifest.json" prebuild)" 189 link_order="$(read_manifest "$dir/manifest.json" link_order)" 190 expected="$(read_manifest "$dir/manifest.json" expected_exit)" 191 192 # Pre-build kit objects (UNTIMED). Shared by both linkers. 193 local s o objs=() 194 for s in $prebuild; do 195 o="$dir/${s%.c}.o" 196 if ! "$KIT" cc -O0 -c "$dir/$s" -o "$o" 2>"$LOG_DIR/$axis.n$n.prebuild.err"; then 197 record "$axis" link kit link "$n" "$unit" NA PREBUILD_FAIL 1 "$LOG_DIR/$axis.n$n.prebuild.err" 198 KIT_BEST_MS=""; return 199 fi 200 done 201 for o in $link_order; do objs+=("$dir/$o"); done 202 203 # kit ld (TIMED): relink the prebuilt objects best-of-N. 204 kit_link_cmd "${objs[@]}" -o "$dir/app" 205 KIT_BEST_MS="" 206 if best_of "$logb" "${KIT_CMD[@]}"; then 207 "$dir/app" >/dev/null 2>&1; local got=$? 208 [ "$got" = "$expected" ] || status="OUTPUT_FAIL($got!=$expected)" 209 KIT_BEST_MS="$BEST_MS" 210 else 211 status="KIT_FAIL" 212 fi 213 record "$axis" link kit link "$n" "$unit" "${BEST_MS:-NA}" "$status" "$LAST_RC" "$logb.1.err" 214 215 # System-ld reference: link the SAME kit objects through the clang driver. 216 # (kit's Mach-O objects no longer carry a raw __eh_frame, so ld64 accepts 217 # them.) If the system linker still rejects an object, it's recorded N/A and 218 # the run continues. 219 if [ "$SKIP_CLANG" != 1 ]; then 220 local clogb="$LOG_DIR/$axis.n$n.clang" cstatus="OK" 221 clang_link_cmd "${objs[@]}" -o "$dir/app.clang" 222 if best_of "$clogb" "${CLANG_CMD[@]}"; then 223 "$dir/app.clang" >/dev/null 2>&1 224 local cgot=$? 225 [ "$cgot" = "$expected" ] || cstatus="OUTPUT_FAIL($cgot!=$expected)" 226 else 227 cstatus="CLANG_LINK_NA" 228 fi 229 record "$axis" link clang link "$n" "$unit" "${BEST_MS:-NA}" "$cstatus" \ 230 "$LAST_RC" "$clogb.1.err" 231 fi 232 } 233 234 # --------------------------------------------------------------------------- 235 # Hotspot sampling: run kit on the largest input in the background, sample by 236 # PID until it exits. No sudo. dtrace path (flat histogram) is opt-in. 237 # --------------------------------------------------------------------------- 238 sample_cmd() { # axis raw_path cmd... 239 local axis="$1" raw="$2"; shift 2 240 if [ "$USE_DTRACE" = 1 ]; then 241 # Flat user-function histogram for exactly the target process. 242 sudo dtrace -x ustackframes=64 -o "$raw" \ 243 -n 'profile-997 /pid == $target/ { @[ufunc(arg1)] = count(); } END { printa(@); }' \ 244 -c "$(printf '%q ' "$@")" >/dev/null 2>"$raw.dtrace.err" || \ 245 printf 'cc-bench: dtrace sample failed for %s (see %s.dtrace.err)\n' "$axis" "$raw" >&2 246 return 247 fi 248 "$@" >/dev/null 2>"$RAW_DIR/$axis.sample.cmd.err" & 249 local pid=$! 250 # Attach and sample at 1ms until the process dies (-mayDie). 3600s upper bound. 251 sample "$pid" 3600 1 -mayDie -file "$raw" >/dev/null 2>"$RAW_DIR/$axis.sample.err" 252 wait "$pid" 2>/dev/null 253 } 254 255 # =========================================================================== 256 printf 'axis,phase,tool,mode,n,unit,time_ms,status,exit_code,note\n' >"$CSV" 257 258 printf 'cc-bench: kit: %s\n' "$KIT" 259 "$KIT" --version 2>/dev/null | head -1 | sed 's/^/cc-bench: /' || true 260 printf 'cc-bench: clang: %s (skip=%s)\n' "$CLANG" "$SKIP_CLANG" 261 printf 'cc-bench: out: %s\n' "$OUT_DIR" 262 printf 'cc-bench: repeats=%s sample=%s dtrace=%s\n' "$REPEATS" "$DO_SAMPLE" "$USE_DTRACE" 263 264 # Fixed per-invocation overhead: empty TU compile, empty TU preprocess, and a 265 # trivial 1-object link. Subtracted by the reporter so process startup doesn't 266 # masquerade as sub-linear scaling. 267 OV_DIR="$GEN_DIR/__overhead__"; mkdir -p "$OV_DIR" 268 printf 'int kit_bench_empty(void){return 0;}\n' >"$OV_DIR/empty.c" 269 printf 'int main(void){return 0;}\n' >"$OV_DIR/main.c" 270 # Warm up: the very first kit invocation pays dyld/page-in cold-start cost that 271 # would inflate the overhead baseline. Discard one run first. 272 "$KIT" cc -O0 -c "$OV_DIR/empty.c" -o "$OV_DIR/empty.o" >/dev/null 2>&1 273 best_of "$LOG_DIR/ov.compile" "$KIT" cc -O0 -c "$OV_DIR/empty.c" -o "$OV_DIR/empty.o" && \ 274 record __overhead__ compile kit compile 0 functions "$BEST_MS" OK 0 "" 275 best_of "$LOG_DIR/ov.pp" "$KIT" cc -O0 -E "$OV_DIR/empty.c" -o "$OV_DIR/empty.i" && \ 276 record __overhead__ compile kit preprocess 0 headers "$BEST_MS" OK 0 "" 277 "$KIT" cc -O0 -c "$OV_DIR/main.c" -o "$OV_DIR/main.o" 2>/dev/null 278 best_of "$LOG_DIR/ov.link" "$KIT" ld "$OV_DIR/main.o" -l c "${KIT_SYSROOT_ARGS[@]}" -o "$OV_DIR/app" && \ 279 record __overhead__ link kit link 0 objects "$BEST_MS" OK 0 "" 280 281 # Axis catalog (JSON) -> drive the sweep. 282 AXES_JSON="$(python3 "$GEN" --list)" 283 ALL_AXES="$(printf '%s' "$AXES_JSON" | python3 -c 'import json,sys; print(" ".join(a["axis"] for a in json.load(sys.stdin)))')" 284 SEL_AXES="${KIT_CC_BENCH_AXES:-$ALL_AXES}" 285 SAMPLE_AXES="${KIT_CC_BENCH_SAMPLE_AXES:-$SEL_AXES}" 286 287 declare -A AXIS_MAXMS AXIS_MAXN AXIS_PHASE AXIS_MODE AXIS_UNIT 288 for axis in $SEL_AXES; do 289 phase="$(printf '%s' "$AXES_JSON" | python3 -c 'import json,sys; a={x["axis"]:x for x in json.load(sys.stdin)}["'"$axis"'"]; print(a["phase"])')" 290 mode="$(printf '%s' "$AXES_JSON" | python3 -c 'import json,sys; a={x["axis"]:x for x in json.load(sys.stdin)}["'"$axis"'"]; print(a["mode"])')" 291 unit="$(printf '%s' "$AXES_JSON" | python3 -c 'import json,sys; a={x["axis"]:x for x in json.load(sys.stdin)}["'"$axis"'"]; print(a["unit"])')" 292 sizes="${KIT_CC_BENCH_SIZES:-$(printf '%s' "$AXES_JSON" | python3 -c 'import json,sys; a={x["axis"]:x for x in json.load(sys.stdin)}["'"$axis"'"]; print(" ".join(map(str,a["sizes"])))')}" 293 AXIS_PHASE[$axis]="$phase"; AXIS_MODE[$axis]="$mode"; AXIS_UNIT[$axis]="$unit" 294 maxn=0 295 printf '\n===== %s (%s, %s) sizes: %s =====\n' "$axis" "$phase" "$mode" "$sizes" 296 for n in $sizes; do 297 dir="$GEN_DIR/$axis/n$n"; mkdir -p "$dir" 298 python3 "$GEN" --axis "$axis" --n "$n" --out "$dir" >"$dir/manifest.json" 299 if [ "$phase" = link ]; then 300 run_link_axis "$axis" "$unit" "$n" "$dir" 301 else 302 run_compile_axis "$axis" "$mode" "$unit" "$n" "$dir" 303 fi 304 printf ' n=%-7s kit=%-10s\n' "$n" "${KIT_BEST_MS:-FAIL}" 305 [ "$n" -gt "$maxn" ] && { maxn="$n"; AXIS_MAXMS[$axis]="${KIT_BEST_MS:-}"; } 306 # Adaptive cap: stop growing this axis once a run blows past the ceiling. 307 if [ "$MAX_MS" != 0 ] && [ -n "${KIT_BEST_MS:-}" ] && \ 308 awk -v m="$KIT_BEST_MS" -v c="$MAX_MS" 'BEGIN{exit !(m+0 > c+0)}'; then 309 printf ' (cap: %.0fms > %sms -> stop growing %s)\n' "$KIT_BEST_MS" "$MAX_MS" "$axis" 310 break 311 fi 312 done 313 AXIS_MAXN[$axis]="$maxn" 314 done 315 316 # ---- sampling pass: largest input per axis ---- 317 if [ "$DO_SAMPLE" = 1 ]; then 318 printf '\ncc-bench: sampling hotspots (largest input per axis)\n' 319 for axis in $SAMPLE_AXES; do 320 [ -n "${AXIS_PHASE[$axis]:-}" ] || continue 321 maxn="${AXIS_MAXN[$axis]}"; maxms="${AXIS_MAXMS[$axis]:-0}" 322 raw="$RAW_DIR/$axis.sample.txt" 323 if [ -z "$maxms" ]; then printf ' %-14s SKIP (run failed)\n' "$axis"; continue; fi 324 if awk -v m="$maxms" -v t="$SAMPLE_MIN_MS" 'BEGIN{exit !(m+0 < t+0)}'; then 325 printf ' %-14s SKIP (%.0fms < %sms)\n' "$axis" "$maxms" "$SAMPLE_MIN_MS" 326 printf 'skipped: %s ran in %sms (< %sms), too short to sample\n' "$axis" "$maxms" "$SAMPLE_MIN_MS" >"$raw" 327 continue 328 fi 329 dir="$GEN_DIR/$axis/n$maxn" 330 if [ "${AXIS_PHASE[$axis]}" = link ]; then 331 link_order="$(read_manifest "$dir/manifest.json" link_order)" 332 objs=(); for o in $link_order; do objs+=("$dir/$o"); done # objects already built in sweep 333 printf ' %-14s sampling kit ld (n=%s, ~%.0fms)\n' "$axis" "$maxn" "$maxms" 334 sample_cmd "$axis" "$raw" "$KIT" ld "${objs[@]}" -l c "${KIT_SYSROOT_ARGS[@]}" -o "$dir/app.sample" 335 else 336 incdir="$(read_manifest "$dir/manifest.json" incdir)" 337 inc_args=(); [ -n "$incdir" ] && inc_args=(-I "$dir/$incdir") 338 printf ' %-14s sampling kit cc (n=%s, ~%.0fms)\n' "$axis" "$maxn" "$maxms" 339 if [ "${AXIS_MODE[$axis]}" = preprocess ]; then 340 sample_cmd "$axis" "$raw" "$KIT" cc -O0 -E "${inc_args[@]}" "$dir/gen.c" -o "$dir/gen.sample.i" 341 else 342 sample_cmd "$axis" "$raw" "$KIT" cc -O0 -c "$dir/gen.c" -o "$dir/gen.sample.o" 343 fi 344 fi 345 done 346 fi 347 348 printf '\ncc-bench: wrote %s\n' "$CSV" 349 python3 "$REPORT" "$OUT_DIR" || true