kit

commit 9c3ce7acc8fe15631b220d004e0e5406340fa775
parent f7a46307b63b2cbb7a3420afcb8acf2b9545fefd
Author: Ryan Sepassi <rsepassi@gmail.com>
Date:   Thu, 11 Jun 2026 09:43:38 -0700

perf(bench): add focused single-axis A/B compile timer

scripts/perf_axis_time.py times kit cc -O0 -c/-E on one synthetic bench
axis for a golden vs candidate kit (best-of-N), for attributing each
perf change's marginal speedup on a quiet machine (parallel worktree
builds make in-worktree timing meaningless).

Diffstat:
A
scripts/perf_axis_time.py
 | 
85
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

1 file changed, 85 insertions(+), 0 deletions(-)
diff --git a/scripts/perf_axis_time.py b/scripts/perf_axis_time.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""perf_axis_time.py — focused single-axis A/B timer for the perf sprint.
+
+Times `kit cc -O0 -c` (compile) or `kit cc -O0 -E` (preprocess) on one synthetic
+bench axis for a GOLDEN and a CAND kit, best-of-N, and prints the per-axis
+speedup. Used to attribute each candidate's win on a quiet machine (the parallel
+worktree builds make in-worktree timing meaningless; this runs centrally after).
+
+  scripts/perf_axis_time.py --golden G/kit --cand C/kit --axis body-size --n 64000
+  scripts/perf_axis_time.py --golden G/kit --cand C/kit --axis pp-macro --n 4000 --mode preprocess
+
+Reports golden best-ms, cand best-ms, delta, and % speedup (positive = cand faster).
+"""
+import argparse, json, os, shutil, subprocess, sys, tempfile, time
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+GEN = os.path.join(HERE, "cc_bench_gen.py")
+
+
+def gen_axis(axis, n, outdir):
+    # The generator writes the instance files into outdir and prints the JSON
+    # manifest to stdout.
+    r = subprocess.run([sys.executable, GEN, "--axis", axis, "--n", str(n), "--out", outdir],
+                       check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    return json.loads(r.stdout.decode())
+
+
+def best_ms(kit, args, cwd, repeats):
+    best = float("inf")
+    for _ in range(repeats):
+        t0 = time.perf_counter()
+        r = subprocess.run([kit, "cc"] + args, cwd=cwd,
+                           stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+        dt = (time.perf_counter() - t0) * 1000.0
+        if r.returncode != 0:
+            sys.stderr.write(f"  ! {os.path.basename(os.path.dirname(kit))} rc={r.returncode}: "
+                             f"{r.stderr.decode()[:200]}\n")
+            return None
+        best = min(best, dt)
+    return best
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--golden", required=True)
+    ap.add_argument("--cand", required=True)
+    ap.add_argument("--axis", required=True)
+    ap.add_argument("--n", type=int, required=True)
+    ap.add_argument("--mode", choices=["compile", "preprocess"], default="compile")
+    ap.add_argument("--repeats", type=int, default=9)
+    ap.add_argument("--target", default="")
+    a = ap.parse_args()
+
+    a.golden = os.path.abspath(a.golden)
+    a.cand = os.path.abspath(a.cand)
+    work = tempfile.mkdtemp(prefix="perfaxis_")
+    try:
+        outdir = os.path.join(work, "gen")
+        m = gen_axis(a.axis, a.n, outdir)
+        src = m.get("source", "gen.c")
+        tflag = (["-target", a.target] if a.target else [])
+        if a.mode == "preprocess":
+            args = tflag + ["-O0", "-E", src, "-o", "out.i"]
+        else:
+            args = tflag + ["-O0", "-c", src, "-o", "out.o"]
+        # warm caches once each (untimed)
+        for k in (a.golden, a.cand):
+            subprocess.run([k, "cc"] + args, cwd=outdir,
+                           stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        g = best_ms(a.golden, args, outdir, a.repeats)
+        c = best_ms(a.cand, args, outdir, a.repeats)
+        if g is None or c is None:
+            print(f"{a.axis:16s} n={a.n:<8} FAILED (golden={g} cand={c})")
+            return 2
+        spd = (g - c) / g * 100.0
+        flag = "  <-- faster" if spd > 1.5 else ("  ~same" if abs(spd) <= 1.5 else "  <-- SLOWER")
+        print(f"{a.axis:16s} n={a.n:<8} golden={g:8.2f}ms  cand={c:8.2f}ms  "
+              f"delta={g-c:+7.2f}ms  speedup={spd:+6.2f}%{flag}")
+        return 0
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())