kit

opt_bench_compare.py (7510B)

---

 #!/usr/bin/env python3
 """Compare kit tools vs a baseline (default: gcc-15 -O0).
 
 The fixed compilers (gcc/clang/MIR) don't change as kit evolves, so their
 timings are measured once into a cached CSV (scripts/opt_bench_baseline.csv via
 `KIT_OPT_BENCH_MODE=baseline scripts/opt_bench.sh`). This script auto-merges
 that cache with the fresh kit-only results so comparisons still work even when
 the results CSV contains only kit rows.
 
 Usage:
     python3 scripts/opt_bench_compare.py [results.csv]
     python3 scripts/opt_bench_compare.py results.csv --base-tool gcc-15 --base-opt 0
     python3 scripts/opt_bench_compare.py results.csv --baseline-csv path.csv
     python3 scripts/opt_bench_compare.py results.csv --no-baseline
 """
 import csv
 import math
 import os
 import sys
 
 DEFAULT_BASELINE_CSV = os.path.join(
     os.path.dirname(os.path.abspath(__file__)), "opt_bench_baseline.csv"
 )
 
 
 def fnum(v):
     try:
         return float(v)
     except (TypeError, ValueError):
         return None
 
 
 def speedup_label(s):
     if s is None:
         return "NA"
     if abs(s - 1.0) < 0.03:
         return "~tied"
     return f"{s:.2f}x faster" if s > 1 else f"{1/s:.2f}x slower"
 
 
 def compare(val, base):
     """Return speedup ratio: >1 means val is faster (lower time) than base."""
     if not val or not base or base <= 0 or val <= 0:
         return None
     return base / val
 
 
 def geo(xs):
     xs = [x for x in xs if x and x > 0]
     if not xs:
         return None
     return math.exp(sum(math.log(x) for x in xs) / len(xs))
 
 
 def compile_total(r):
     cm = fnum(r.get("compile_ms"))
     cg = fnum(r.get("codegen_ms")) or 0.0
     return (cm + cg) if cm is not None else None
 
 
 def main():
     args = sys.argv[1:]
     csv_path = None
     base_tool_arg = None
     base_opt = "0"
     baseline_csv = DEFAULT_BASELINE_CSV
     i = 0
     while i < len(args):
         if args[i] == "--base-tool" and i + 1 < len(args):
             base_tool_arg = args[i + 1]
             i += 2
         elif args[i] == "--base-opt" and i + 1 < len(args):
             base_opt = args[i + 1]
             i += 2
         elif args[i] == "--baseline-csv" and i + 1 < len(args):
             baseline_csv = args[i + 1]
             i += 2
         elif args[i] == "--no-baseline":
             baseline_csv = None
             i += 1
         else:
             csv_path = args[i]
             i += 1
 
     if csv_path is None:
         root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         csv_path = os.path.join(root, "build", "bench", "opt", "results.csv")
 
     if not os.path.exists(csv_path):
         sys.exit(f"compare: CSV not found: {csv_path}")
 
     with open(csv_path, newline="") as f:
         ok = [r for r in csv.DictReader(f) if r["status"] == "OK"]
 
     # Benches measured by this run (typically kit-only); used to scope output.
     result_benches = {r["bench"] for r in ok}
 
     # Merge cached baseline timings (gcc/clang/MIR) unless disabled. Rows already
     # present in the fresh CSV win, so an explicit baseline run still overrides.
     if (
         baseline_csv
         and os.path.exists(baseline_csv)
         and os.path.abspath(baseline_csv) != os.path.abspath(csv_path)
     ):
         seen = {(r["tool"], r["opt"], r["bench"]) for r in ok}
         with open(baseline_csv, newline="") as f:
             for r in csv.DictReader(f):
                 if r["status"] != "OK":
                     continue
                 key = (r["tool"], r["opt"], r["bench"])
                 if key not in seen:
                     ok.append(r)
                     seen.add(key)
 
     if not ok:
         sys.exit("compare: no OK rows in CSV")
 
     # Find base tool
     base_tool = base_tool_arg
     if not base_tool:
         for prefix in ("gcc-15", "gcc"):
             base_tool = next((r["tool"] for r in ok if r["tool"].startswith(prefix)), None)
             if base_tool:
                 break
     if not base_tool:
         sys.exit("compare: no gcc tool found; use --base-tool TOOL")
 
     # Baseline map: bench -> (compile_total_ms, runtime_ms)
     baseline = {}
     for r in ok:
         if r["tool"] == base_tool and r["opt"] == base_opt:
             baseline[r["bench"]] = (compile_total(r), fnum(r["runtime_ms"]))
 
     if not baseline:
         sys.exit(f"compare: no rows for tool={base_tool} opt={base_opt}")
 
     idx = {(r["tool"], r["opt"], r["bench"]): r for r in ok}
     # Only report benches this run covered (and that the baseline has). When the
     # CSV is itself a baseline dump (no separate result benches), show them all.
     shown = (result_benches & set(baseline)) if result_benches else set(baseline)
     all_benches = sorted(shown) or sorted(baseline)
     all_opts = sorted(
         {r["opt"] for r in ok},
         key=lambda x: (int(x) if x.isdigit() else 99, x),
     )
 
     # Show order: base row first, then kit/kit-run at each opt level
     show = [(base_tool, base_opt)]
     seen = {(base_tool, base_opt)}
     for t in ("kit", "kit-run", "mir-c2m"):
         for o in all_opts:
             key = (t, o)
             if key not in seen and any((t, o, b) in idx for b in all_benches):
                 show.append(key)
                 seen.add(key)
 
     print(f"kit/mir vs {base_tool} -O{base_opt}  [{os.path.basename(csv_path)}]")
     print()
 
     # Per-benchmark table
     bench_label = all_benches[0] if len(all_benches) == 1 else f"{len(all_benches)} benches"
     print(f"Per-benchmark: {bench_label}")
     print()
     hdr = (
         f"{'bench':<16} {'tool':<12} opt  "
         f"{'compile_ms':>10}  {'compile vs base':>16}  "
         f"{'runtime_ms':>10}  {'runtime vs base':>16}"
     )
     print(hdr)
     print("-" * len(hdr))
 
     for bench in all_benches:
         bcm, brt = baseline[bench]
         first = True
         for tool, opt in show:
             r = idx.get((tool, opt, bench))
             if r is None:
                 continue
             ct = compile_total(r)
             rt = fnum(r["runtime_ms"])
             is_base = tool == base_tool and opt == base_opt
             cl = "baseline" if is_base else speedup_label(compare(ct, bcm))
             rl = "baseline" if is_base else speedup_label(compare(rt, brt))
             bench_col = bench if first else ""
             first = False
             ct_s = f"{ct:.1f}" if ct is not None else "NA"
             rt_s = f"{rt:.1f}" if rt is not None else "NA"
             print(
                 f"{bench_col:<16} {tool:<12} O{opt:<2}  "
                 f"{ct_s:>10}  {cl:>16}  "
                 f"{rt_s:>10}  {rl:>16}"
             )
         print()
 
     if len(all_benches) <= 1:
         return
 
     # Geomean summary
     print(f"Geomean ({len(all_benches)} benches)")
     print()
     hdr2 = f"{'tool':<12} opt  {'compile speedup':>16}  {'runtime speedup':>16}"
     print(hdr2)
     print("-" * len(hdr2))
     for tool, opt in show:
         crs, rrs = [], []
         for bench in all_benches:
             bcm, brt = baseline.get(bench, (None, None))
             r = idx.get((tool, opt, bench))
             if r is None:
                 continue
             s = compare(compile_total(r), bcm)
             if s is not None:
                 crs.append(s)
             s = compare(fnum(r["runtime_ms"]), brt)
             if s is not None:
                 rrs.append(s)
         is_base = tool == base_tool and opt == base_opt
         cl = "baseline" if is_base else speedup_label(geo(crs))
         rl = "baseline" if is_base else speedup_label(geo(rrs))
         print(f"{tool:<12} O{opt:<2}  {cl:>16}  {rl:>16}")
 
 
 if __name__ == "__main__":
     main()