perf_axis_time.py (3573B)
1 #!/usr/bin/env python3 2 """perf_axis_time.py — focused single-axis A/B timer for the perf sprint. 3 4 Times `kit cc -O0 -c` (compile) or `kit cc -O0 -E` (preprocess) on one synthetic 5 bench axis for a GOLDEN and a CAND kit, best-of-N, and prints the per-axis 6 speedup. Used to attribute each candidate's win on a quiet machine (the parallel 7 worktree builds make in-worktree timing meaningless; this runs centrally after). 8 9 scripts/perf_axis_time.py --golden G/kit --cand C/kit --axis body-size --n 64000 10 scripts/perf_axis_time.py --golden G/kit --cand C/kit --axis pp-macro --n 4000 --mode preprocess 11 12 Reports golden best-ms, cand best-ms, delta, and % speedup (positive = cand faster). 13 """ 14 import argparse, json, os, shutil, subprocess, sys, tempfile, time 15 16 HERE = os.path.dirname(os.path.abspath(__file__)) 17 GEN = os.path.join(HERE, "cc_bench_gen.py") 18 19 20 def gen_axis(axis, n, outdir): 21 # The generator writes the instance files into outdir and prints the JSON 22 # manifest to stdout. 23 r = subprocess.run([sys.executable, GEN, "--axis", axis, "--n", str(n), "--out", outdir], 24 check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) 25 return json.loads(r.stdout.decode()) 26 27 28 def best_ms(kit, args, cwd, repeats): 29 best = float("inf") 30 for _ in range(repeats): 31 t0 = time.perf_counter() 32 r = subprocess.run([kit, "cc"] + args, cwd=cwd, 33 stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) 34 dt = (time.perf_counter() - t0) * 1000.0 35 if r.returncode != 0: 36 sys.stderr.write(f" ! {os.path.basename(os.path.dirname(kit))} rc={r.returncode}: " 37 f"{r.stderr.decode()[:200]}\n") 38 return None 39 best = min(best, dt) 40 return best 41 42 43 def main(): 44 ap = argparse.ArgumentParser() 45 ap.add_argument("--golden", required=True) 46 ap.add_argument("--cand", required=True) 47 ap.add_argument("--axis", required=True) 48 ap.add_argument("--n", type=int, required=True) 49 ap.add_argument("--mode", choices=["compile", "preprocess"], default="compile") 50 ap.add_argument("--repeats", type=int, default=9) 51 ap.add_argument("--target", default="") 52 a = ap.parse_args() 53 54 a.golden = os.path.abspath(a.golden) 55 a.cand = os.path.abspath(a.cand) 56 work = tempfile.mkdtemp(prefix="perfaxis_") 57 try: 58 outdir = os.path.join(work, "gen") 59 m = gen_axis(a.axis, a.n, outdir) 60 src = m.get("source", "gen.c") 61 tflag = (["-target", a.target] if a.target else []) 62 if a.mode == "preprocess": 63 args = tflag + ["-O0", "-E", src, "-o", "out.i"] 64 else: 65 args = tflag + ["-O0", "-c", src, "-o", "out.o"] 66 # warm caches once each (untimed) 67 for k in (a.golden, a.cand): 68 subprocess.run([k, "cc"] + args, cwd=outdir, 69 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 70 g = best_ms(a.golden, args, outdir, a.repeats) 71 c = best_ms(a.cand, args, outdir, a.repeats) 72 if g is None or c is None: 73 print(f"{a.axis:16s} n={a.n:<8} FAILED (golden={g} cand={c})") 74 return 2 75 spd = (g - c) / g * 100.0 76 flag = " <-- faster" if spd > 1.5 else (" ~same" if abs(spd) <= 1.5 else " <-- SLOWER") 77 print(f"{a.axis:16s} n={a.n:<8} golden={g:8.2f}ms cand={c:8.2f}ms " 78 f"delta={g-c:+7.2f}ms speedup={spd:+6.2f}%{flag}") 79 return 0 80 finally: 81 shutil.rmtree(work, ignore_errors=True) 82 83 84 if __name__ == "__main__": 85 sys.exit(main())