commit 98b1f010ea090a8d18360d194880ea658b26296c
parent 2b1b03c21d1b949107cf692febb5a709969608f0
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 07:07:19 -0700
perf(cg): -O0 register-resident call args — spill only the live set
nd_call flushed the entire value cache before every call and sourced each
argument from its frame home, emitting `<compute>; stur <arg>,[home]; ldur
x0,[home]; bl` — a spill+reload round-trip per argument plus dead writebacks of
every cached local.
Now an argument provably dead after the call (new CGCallDesc.arg_dead_mask, set
from the sound api_temp_dead in api_finish_call) is sourced straight from its
live register into the ABI arg register; the backends' existing parallel-copy
scheduler (native_arg_shuffle) already resolves reg<->arg-reg conflicts. Only
the live-across set — non-arg cached locals plus any live/shared arg — is
spilled (nd_flush_all_except_kept_args). Kept args are pinned across the callee
materialize and dropped without write-back afterward (their registers are
call-clobbered and they are dead).
viacall: add w8,w9,w10; stur w8,[home]; ldur w0,[home]; bl
-> add w8,w9,w10; mov w0,w8; bl
A live or shared arg (`y = g(t) + t`) stays clear in the mask and is spilled, so
its value survives the call (verified against clang).
sqlite3.c -c -O0 (arm64-macOS): compile instrs 2,145.0M -> 2,134.5M (-10.4M);
emitted insns 518,757 -> 479,263 (-39,494); object 2,469,528 -> 2,311,552 B
(-6.4%). Byte-deterministic (compile x2 identical).
Green: test-toy 1392/0, test-parse 3920+129/0, test-smoke-x64 3/0,
test-smoke-rv64 3/0, test-cg-api 0 fail, test-libc 9/0; sqlite e2e 84|2;
call-correctness probe matches clang (shared-arg-live-across, swap, indirect,
nested).
Diffstat:
3 files changed, 74 insertions(+), 5 deletions(-)
diff --git a/src/cg/call.c b/src/cg/call.c
@@ -133,6 +133,14 @@ static int api_tail_decide(KitCg* g, const CGCallDesc* desc,
static void api_finish_call(KitCg* g, CGCallDesc* desc, int want_tail,
int emit_tail) {
if (!emit_tail) api_call_clobber_boundary(g, desc);
+ /* Flag arguments that are dead after the call (no live value-stack reference)
+ * so the -O0 backend can keep them register-resident across the call instead
+ * of round-tripping through their homes. api_temp_dead is sound — true only
+ * when the transient is provably dead — so a live or shared arg stays clear
+ * and is spilled like any other live-across value. */
+ desc->arg_dead_mask = 0;
+ for (u32 i = 0; i < desc->nargs && i < 64u; ++i)
+ if (api_temp_dead(g, desc->args[i])) desc->arg_dead_mask |= (u64)1u << i;
g->target->call(g->target, desc);
/* Push the single result (if any) onto the stack. */
diff --git a/src/cg/cgir.h b/src/cg/cgir.h
@@ -404,6 +404,13 @@ typedef struct CGCallDesc {
* call+ret (ALLOWED), or diagnose (MUST). */
u8 pad;
KitCgInlinePolicy inline_policy;
+ /* Bit i set => args[i] is a transient that is provably dead after this call
+ * (no live value-stack reference). The -O0 NativeDirectTarget may then source
+ * such an arg from its live register and drop it without writing back,
+ * instead of the spill-to-home + reload round-trip; a live (clear-bit) arg is
+ * spilled like any other live-across value. Bits >= 64 are 0 (treated as
+ * live: correct, just unoptimized). Other backends ignore it. */
+ u64 arg_dead_mask;
} CGCallDesc;
typedef u32 Label;
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -676,6 +676,32 @@ static void nd_drop_all(NativeDirectTarget* d) {
nd_invalidate_local(d, (CGLocal)(d->cache_head + 1));
}
+/* A "kept" argument is one nd_call leaves register-resident across the call: it
+ * is named in desc->args and flagged dead in arg_dead_mask (so dropping it after
+ * the call loses nothing). A live or shared arg is not kept and is spilled. */
+static int nd_call_arg_kept(const CGCallDesc* desc, CGLocal local) {
+ for (u32 i = 0; i < desc->nargs; ++i)
+ if (desc->args[i] == local)
+ return i < 64u && ((desc->arg_dead_mask >> i) & 1u);
+ return 0;
+}
+
+/* Spill every cached local EXCEPT the kept (dead) argument locals to its home
+ * and empty those entries. Kept args stay cached so the marshalling can source
+ * them from their live registers; everything else is potentially live across
+ * the call and, on the -O0 caller-saved-only cache, sits in a call-clobbered
+ * register, so it must reach memory now. */
+static void nd_flush_all_except_kept_args(NativeDirectTarget* d,
+ const CGCallDesc* desc) {
+ i32 idx = d->cache_head;
+ while (idx >= 0) {
+ i32 next = d->locals[idx].cache_next; /* capture before flush unlinks idx */
+ CGLocal local = (CGLocal)(idx + 1);
+ if (!nd_call_arg_kept(desc, local)) nd_flush_local(d, local);
+ idx = next;
+ }
+}
+
static NativeAddr nd_addr_materialize(NativeDirectTarget* d, NativeAddr in,
NdAddrTemps* temps, MemAccess mem) {
NativeAddr out = in;
@@ -1723,17 +1749,37 @@ static void nd_call(CgTarget* t, const CGCallDesc* desc) {
NativeLoc* results;
NativeLoc callee_tmp;
int release_callee_tmp = 0;
- nd_flush_all(d);
- nd_barrier(d, NATIVE_DIRECT_BARRIER_CALL | NATIVE_DIRECT_BARRIER_MEMORY);
+ u32 nresults = desc->result != CG_LOCAL_NONE ? 1u : 0u;
memset(&plan, 0, sizeof plan);
memset(&nd, 0, sizeof nd);
memset(&callee_tmp, 0, sizeof callee_tmp);
- u32 nresults = desc->result != CG_LOCAL_NONE ? 1u : 0u;
args = nd_loc_buf(d, d->argbuf, ND_ARG_BUF, desc->nargs);
results = nd_loc_buf(d, d->retbuf, ND_RET_BUF, nresults);
- for (u32 i = 0; i < desc->nargs; ++i)
- args[i] = nd_loc_frame(d, desc->args[i], 0);
+ /* Source each argument cache-aware BEFORE spilling. A cached arg that is dead
+ * after the call (arg_dead_mask) flows from its live register straight into
+ * the ABI arg register — the backend's parallel-copy scheduler
+ * (native_arg_shuffle) resolves reg<->arg-reg conflicts — avoiding the
+ * spill-to-home + reload round-trip the old flush-all forced. Pin its register
+ * so the callee materialize below cannot evict it as a scratch victim. A live
+ * (or uncached) arg keeps the home source and is spilled by the flush below. */
+ for (u32 i = 0; i < desc->nargs; ++i) {
+ NativeDirectLocal* l = nd_local(d, desc->args[i]);
+ int dead = i < 64u && ((desc->arg_dead_mask >> i) & 1u);
+ if (dead && l->reg != REG_NONE) {
+ args[i] = native_loc_reg(l->type, (NativeAllocClass)l->cls, l->reg);
+ d->scratch_used[l->cls] |= 1u << l->reg;
+ } else {
+ args[i] = nd_loc_frame(d, desc->args[i], 0);
+ }
+ }
if (nresults) results[0] = nd_loc_frame(d, desc->result, 0);
+ /* An indirect callee is read from its home; make it authoritative before the
+ * selective flush (the cg only ever produces an OPK_LOCAL or OPK_GLOBAL
+ * callee). */
+ if (desc->callee.kind == OPK_LOCAL) nd_flush_local(d, desc->callee.v.local);
+ /* Spill the live-across set: everything cached except the kept (dead) args. */
+ nd_flush_all_except_kept_args(d, desc);
+ nd_barrier(d, NATIVE_DIRECT_BARRIER_CALL | NATIVE_DIRECT_BARRIER_MEMORY);
nd.fn_type = desc->fn_type;
nd.callee = nd_loc_operand(d, desc->callee);
if (nd.callee.kind == NATIVE_LOC_FRAME) {
@@ -1766,6 +1812,14 @@ static void nd_call(CgTarget* t, const CGCallDesc* desc) {
ND_REQUIRE_NATIVE(d, emit_call, "target does not emit calls");
d->native->emit_call(d->native, &plan);
}
+ /* The call clobbered the caller-saved registers, so the (dead-after) args'
+ * cached values are stale: unpin and drop their entries without writing back.
+ * args[i] is a reg only for args that were sourced from the cache. */
+ for (u32 i = 0; i < desc->nargs; ++i) {
+ if (args[i].kind == NATIVE_LOC_REG)
+ d->scratch_used[args[i].cls] &= ~(1u << args[i].v.reg);
+ nd_invalidate_local(d, desc->args[i]);
+ }
for (u32 i = 0; i < plan.nrets; ++i)
nd_write_loc(d, plan.rets[i].dst, plan.rets[i].src, plan.rets[i].mem);
if (release_callee_tmp)
