commit d4eeb8400c5efb6c824ffa48eba4f7d24b823279
parent 9f3f9a01597ac46d82d5c3369e4c5851a43b57eb
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 00:16:57 -0700
opt: inline-pressure cap at O1 (O1.md W4)
Diffstat:
3 files changed, 185 insertions(+), 1 deletion(-)
diff --git a/mk/test.mk b/mk/test.mk
@@ -859,7 +859,7 @@ test-macho: lib $(TEST_RT_DEP) $(ROUNDTRIP_BIN_MACHO) $(LINK_EXE_RUNNER) $(JIT_R
OPT_TEST_BIN = build/test/cg_ir_lower_test
TINY_INLINE_TEST_BIN = build/test/tiny_inline_test
-test-opt: bin $(OPT_TEST_BIN) test-opt-tiny-inline test-opt-inline test-opt-zero-arg test-opt-static-prune-aa64 test-opt-aa64-tail test-opt-x64-win-tail-sret test-opt-prologue-tier test-opt-whole-program-inline test-opt-lto-phase1 test-opt-redundant-copy-ext test-opt-redundant-frame-sub test-opt-o1-branch-cleanup test-opt-hot-slot-order test-opt-o1-remat test-opt-aa64-x29-bottom test-opt-o1-coalesce test-opt-o1-switch-imm
+test-opt: bin $(OPT_TEST_BIN) test-opt-tiny-inline test-opt-inline test-opt-zero-arg test-opt-static-prune-aa64 test-opt-aa64-tail test-opt-x64-win-tail-sret test-opt-prologue-tier test-opt-whole-program-inline test-opt-lto-phase1 test-opt-redundant-copy-ext test-opt-redundant-frame-sub test-opt-o1-branch-cleanup test-opt-hot-slot-order test-opt-o1-remat test-opt-aa64-x29-bottom test-opt-o1-coalesce test-opt-o1-switch-imm test-opt-o1-inline-cap
$(OPT_TEST_BIN)
@@ -911,6 +911,12 @@ test-opt-o1-coalesce: bin
test-opt-o1-switch-imm: bin
@KIT=$(abspath $(BIN)) bash test/opt/o1_switch_imm.sh
+# Behavioral check: the O1 whole-program inliner backs off inlining into an
+# already-huge / high-pressure caller, while small + always_inline still fuse.
+.PHONY: test-opt-o1-inline-cap
+test-opt-o1-inline-cap: bin
+ @KIT=$(abspath $(BIN)) bash test/opt/o1_inline_cap.sh
+
test-opt-tiny-inline: bin $(TINY_INLINE_TEST_BIN)
$(TINY_INLINE_TEST_BIN)
diff --git a/src/opt/pass_inline.c b/src/opt/pass_inline.c
@@ -37,6 +37,27 @@ typedef struct InlineOrderCtx {
#define INLINE_ABS_GROWTH_LIMIT 64u
#define INLINE_HINT_ABS_GROWTH_LIMIT 128u
+/* Inline-pressure cap (O1.md W4). The existing cost/growth gates bound how much
+ * a single inline *grows* a caller, but at -O1 there is no SSA post-inline
+ * cleanup (GVN/DSE/redundant-load elim are O2-only), so inlining into a caller
+ * whose live-set is already far past the register file just deepens the spilling
+ * it does. The whole-program inliner walks callers bottom-up to a fixpoint, so a
+ * single hot caller can accrete dozens of small bodies and climb to hundreds of
+ * simultaneously-live pseudo-registers. This is a cheap, per-call-site back-off:
+ * once the estimated post-inline live-set (caller pregs + callee pregs, an upper
+ * bound — param/return materialization actually reuses some) would sit past this
+ * cap, refuse a DEFAULT/HINT inline. ALWAYS still bypasses it (the frontend's
+ * explicit request wins, matching the cost-cap behavior).
+ *
+ * The cap is set well above any target's register file (aa64/rv64 = 32 GPRs,
+ * x64 = 16; allocatable ~16-28 after reserves), so it only fires on callers that
+ * are *already* spilling heavily and can only spill worse — it is deliberately
+ * conservative and never trims a moderate-pressure hot loop. Measured on the
+ * ecosystem corpus (aarch64/Darwin, -O1): byte-identical on the inline-win files
+ * (lz4 0.39x, yyjson 0.88x, lapi, cjson, miniz) and a small spill reduction on
+ * sqlite (the inline-heavy / huge-function case the doc names). See O1.md W4. */
+#define INLINE_PRESSURE_CAP 512u
+
/* Streaming O1 tiny-inline: a much smaller budget than the whole-program
* inliner. DEFAULT/HINT callees must fit under this cost; ALWAYS bypasses it.
* Bounded by max passes since an inlined straightline body never introduces a
@@ -654,6 +675,18 @@ static int caller_growth_ok(FuncSet* fs, Func* caller, const u32* base_cost,
return func_inline_cost(caller) + callee_cost <= base + growth_limit;
}
+/* O1.md W4: back off when the estimated post-inline live-set would sit past the
+ * pressure cap (see INLINE_PRESSURE_CAP). Cheap and linear — one read of the two
+ * functions' pseudo-register counts, no analysis. ALWAYS bypasses, matching the
+ * cost/growth gates: an explicit always_inline request wins over the heuristic.
+ * The callee's pregs are an upper bound on the bodies it adds (param/return
+ * materialization reuses some), so the estimate never *under*-counts pressure. */
+static int caller_pressure_ok(const Func* caller, const Func* callee,
+ KitCgInlinePolicy policy) {
+ if (policy == KIT_CG_INLINE_ALWAYS) return 1;
+ return (u64)caller->npregs + (u64)callee->npregs < INLINE_PRESSURE_CAP;
+}
+
static int try_inline_call(FuncSet* fs, Func* caller, u32 b, u32 i,
const u32* base_cost) {
Inst* in = &caller->blocks[b].insts[i];
@@ -679,6 +712,12 @@ static int try_inline_call(FuncSet* fs, Func* caller, u32 b, u32 i,
metrics_count(caller->c, "opt.inline.refuse_growth", 1);
return 0;
}
+ if (!caller_pressure_ok(caller, callee, policy)) {
+ /* Folded into the growth-refusal bucket (no schema change): both are
+ * "refuse because the caller would get too big". */
+ metrics_count(caller->c, "opt.inline.refuse_growth", 1);
+ return 0;
+ }
if (!inline_rewrite_supported(callee, in)) {
metrics_count(caller->c, "opt.inline.refuse_rewrite_shape", 1);
return 0;
@@ -883,6 +922,12 @@ static int try_tiny_inline_call(FuncSet* fs, Func* caller, u32 b, u32 i) {
metrics_count(caller->c, "opt.tiny_inline.refuse_budget", 1);
return 0;
}
+ /* No pressure cap here: the streaming tiny-inliner only fuses callees of cost
+ * <= INLINE_TINY_COST_LIMIT, which add a handful of pregs at most, so a tiny
+ * fuse never meaningfully grows the live-set. The W4 pressure cap targets the
+ * whole-program inliner (try_inline_call), which is where many small bodies
+ * accrete into one caller; applying it here measurably *hurt* (it blocked
+ * beneficial tiny fuses into large-but-not-pressured callers). */
if (!inline_rewrite_supported(callee, in)) {
metrics_count(caller->c, "opt.tiny_inline.refuse_rewrite_shape", 1);
return 0;
diff --git a/test/opt/o1_inline_cap.sh b/test/opt/o1_inline_cap.sh
@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+# Inline-pressure cap (O1.md W4).
+#
+# At -O1 there is no SSA post-inline cleanup, so the whole-program inliner
+# (opt_inline) backs off when the estimated post-inline live-set
+# (caller pregs + callee pregs) would sit past INLINE_PRESSURE_CAP (512). The cap
+# is set far above any target's register file, so it only fires on callers that
+# are already spilling heavily; a small, low-pressure caller still inlines the
+# exact same callee. This is one target-independent decision, so the structural
+# checks run identically for aarch64, x86_64, and riscv64.
+#
+# Green:
+# 1. A non-tiny callee (cost above the streaming tiny cap, so only the
+# whole-program inliner handles it) fuses into a SMALL caller — no call
+# left, proving the cap does not over-fire on ordinary code.
+# 2. The SAME callee called from a HUGE / high-pressure caller is NOT fused —
+# the call survives, proving the cap backs off past its threshold.
+# 3. An always_inline callee fuses even into the huge caller (ALWAYS bypasses
+# the cap, matching the cost/growth gates).
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+KIT="${KIT:-$ROOT/build/kit}"
+WORK="$ROOT/build/test/opt/o1_inline_cap"
+mkdir -p "$WORK"
+
+call_mnemonics='\b(bl|blr|callq?|jalr?)\b'
+
+# Count calls inside one function body. The disassembly has no blank separators,
+# so walk from the function's `<name>:` header to the next symbol header.
+calls_in() { # $1=disfile $2=funcname
+ awk -v f="<$2>:" '$0 ~ f {g = 1; next} /<[A-Za-z_].*>:/ {g = 0} g' "$1" \
+ | grep -cE "$call_mnemonics" || true
+}
+
+# A callee whose cost sits above the streaming tiny-inline cap (8) but below the
+# DEFAULT whole-program cap (20): only the whole-program inliner can fuse it, so
+# this isolates the W4 cap on opt_inline.
+HELPER='static int helper(int x){ int a=x*3; int b=a+7; int c=b^a;'\
+' int d=c*x; int e=d-b; return a+b+c+d+e+x; }'
+# always_inline twin: must fuse regardless of pressure.
+AHELPER='__attribute__((always_inline)) static int ahelper(int x){'\
+' int a=x*3; int b=a+7; int c=b^a; int d=c*x; int e=d-b;'\
+' return a+b+c+d+e+x; }'
+
+# Emit a caller that holds ~N*2 temporaries live across the helper call, pushing
+# the inline-time pregs estimate well past the 512 cap. N=300 -> ~600 pregs.
+emit_huge_caller() { # $1=name $2=callee $3=N
+ printf 'int %s(int *p){\n int s = %s(p[0])' "$1" "$2"
+ local i
+ for ((i = 0; i < $3; i++)); do printf ' + (p[%d]*%d ^ p[%d])' "$i" "$((i + 1))" "$((i + 2))"; done
+ printf ';\n return s;\n}\n'
+}
+
+build_src() { # writes the test TU to $1
+ {
+ printf '%s\n' "$HELPER"
+ printf '%s\n' "$AHELPER"
+ # Small, low-pressure caller of the plain helper: must fuse.
+ printf 'int small(int *p){ return helper(p[0]) + p[1]; }\n'
+ # Huge, high-pressure caller of the plain helper: must NOT fuse.
+ emit_huge_caller huge helper 300
+ # Huge caller of the always_inline helper: must fuse despite pressure.
+ emit_huge_caller huge_always ahelper 300
+ } > "$1"
+}
+
+check_arch() {
+ local triple=$1 tag=$2
+ local src="$WORK/$tag.c" obj="$WORK/$tag.o" dis="$WORK/$tag.dis"
+ build_src "$src"
+ "$KIT" cc -target "$triple" -O1 -ffreestanding -std=c11 -c "$src" -o "$obj" \
+ > "$WORK/$tag.cc.out" 2>&1
+ "$KIT" objdump -d "$obj" > "$dis" 2>&1
+
+ local cs ch ca
+ cs=$(calls_in "$dis" small)
+ ch=$(calls_in "$dis" huge)
+ ca=$(calls_in "$dis" huge_always)
+
+ if [ "$cs" -ne 0 ]; then
+ printf 'o1-inline-cap FAILED: %s small caller did NOT inline helper (%s call[s]) — cap over-fired\n' \
+ "$tag" "$cs" >&2
+ awk '/<small>:/{g=1;next} /<[A-Za-z_].*>:/{g=0} g' "$dis" | sed 's/^/ | /' >&2
+ exit 1
+ fi
+ if [ "$ch" -eq 0 ]; then
+ printf 'o1-inline-cap FAILED: %s huge/high-pressure caller inlined helper anyway — pressure cap did not back off\n' \
+ "$tag" >&2
+ exit 1
+ fi
+ if [ "$ca" -ne 0 ]; then
+ printf 'o1-inline-cap FAILED: %s always_inline callee NOT fused into huge caller — cap ignored ALWAYS policy\n' \
+ "$tag" >&2
+ exit 1
+ fi
+ printf 'o1-inline-cap %-8s small fused, huge backed off, always_inline fused\n' "$tag"
+}
+
+check_arch aarch64-linux-gnu aa64
+check_arch x86_64-linux-gnu x64
+check_arch riscv64-linux-gnu rv64
+
+# Metric corroboration: the whole-program inliner must report a growth/pressure
+# refusal at -O1 for the huge caller (the cap is folded into refuse_growth), and
+# must still inline the small caller — i.e. the cap fires selectively, not
+# blanket. Run via `kit run` so KIT_METRICS surfaces the finalize-sweep counters.
+read -r -d '' RUN_SRC <<'EOF' || true
+__attribute__((noinline)) static int sink(int x){ return x; }
+EOF
+build_src "$WORK/metric.c"
+printf 'int main(void){ int p[512]; for(int i=0;i<512;i++) p[i]=i;'\
+' return (small(p) | huge(p) | huge_always(p)) ? 0 : 0; }\n' >> "$WORK/metric.c"
+if ! KIT_METRICS=1 "$KIT" run -O1 "$WORK/metric.c" >"$WORK/metric.out" 2>"$WORK/metric.err"; then
+ printf 'o1-inline-cap FAILED: `kit run -O1` on the metric TU did not exit 0\n' >&2
+ sed 's/^/ | /' "$WORK/metric.err" >&2
+ exit 1
+fi
+# Some inlining happened (the small caller), and some was refused (the huge one,
+# via the growth/pressure bucket).
+if ! grep -q 'opt.inline.inlined' "$WORK/metric.err"; then
+ printf 'o1-inline-cap FAILED: opt.inline.inlined metric absent at -O1\n' >&2
+ sed -n '1,80p' "$WORK/metric.err" >&2
+ exit 1
+fi
+if ! grep -q 'opt.inline.refuse_growth' "$WORK/metric.err"; then
+ printf 'o1-inline-cap FAILED: no growth/pressure refusal recorded (cap never fired)\n' >&2
+ sed -n '1,80p' "$WORK/metric.err" >&2
+ exit 1
+fi
+printf 'o1-inline-cap run inlined some, refused the over-pressure caller, exit 0\n'
+
+printf 'o1-inline-cap: ok\n'
