commit 3dbbfcf1bb2cc2e23eee8a8f147c107b5d3d84bc
parent 551c873fe9683e912226ead7bd5b14465accfa7e
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 16:20:04 -0700
perf(cg): -O0 spill reduction via eager dead-operand drop + branch reorder (PERF-O0-CODESIZE §6, Lever 2)
Implements doc/plan/PERF-O0-CODESIZE.md §6 (Lever 2: reduce -O0 frame
spilling). sqlite3.c .text on arm64-macOS: 398,433 -> 373,852 insns
(-24,581, -6.17%); 1.16x -> 1.091x tcc (excess +55,698 -> +31,117, ~44%
of the remaining excess removed). Deterministic; not byte-identical (it
removes spills/reloads, so allocation shifts).
Three sub-changes, all on the shared NativeDirectTarget (-O0):
2a Eager dead-operand drop. The NDT cache holds dirty compute results
and flushes the whole live set at every barrier (call/branch/...), so
a transient whose value dies before the next barrier is still
flush-stored there. New api_op_kill_if_dead (cg layer) sets the
existing OPK_FLAG_KILL on any operand api_temp_dead confirms is a dead
transient (distinct from the op's destination); new
nd_drop_killed_operand (NDT) drops such an operand's cache entry
WITHOUT a write-back after the op consumes it. Wired into binop, cmp,
unop, store-value (cg: arith.c/fold.c/memory.c; NDT consumers). This
generalizes the copy/convert OPK_FLAG_KILL pattern that already
existed; call-args were already handled via arg_dead_mask. Safe-set
alone: -2,533 insns (-1,927 stur, -419 str).
Branch barriers (the &&/||/?:/if case §6.2 names as the main excess):
nd_cmp_branch flushed FIRST, then materialized the compare operands —
spilling a cached operand and immediately reloading it for the
compare. Reorder to materialize-then-drop-killed-then-flush: a
cache-resident operand is read from its register (no reload), a dead
one is dropped (no store), and the flush still spills everything live
so the post-branch cache is empty (merge correctness unchanged). The
flush's stores precede the compare, so they cannot clobber its flags;
rv64's fused compare-branch is equally fine. control.c flags the
SV_CMP and truthiness operands dead. Reorder: -11,908 (mostly ldur);
branch kill: -10,079 (-7,095 stur, -2,125 str).
2c Skip the value-cache flush at pure memory barriers (volatile
load/store, atomic load/store/rmw/cas, fence). The cache holds only
non-escaped (non-address-taken, non-memory-required, hence
non-volatile) locals, which no pointer/other-agent can alias, so these
barriers can neither observe a cached value's stale home nor require
its write-back. The arch barrier hook / fence instruction still fires.
~0 on sqlite (few such ops) but sound and removes wasted work.
Gate (§5/§6.6 codesize-track, all green): determinism (self-cmp x2);
toy aa64 1392/0; toy cross-arch exec on x64+rv64 753/0; parse-ok (D/R/E/J)
/parse-err; dwarf; debug; cg-api; opt; smoke-x64/rv64; libc-glibc
(atomics/volatile, for 2c); aa64-inline; isa; link; ecosystem sqlite O0+O1
golden + vs-clang (real queries run); and clang-differential probes at
O0+O1 (dead operands across call/branch barriers, a*a / f(t,t)
doubly-used temps, transients live across nested calls, store-then-call,
volatile+atomic interleaved with cached locals).
Diffstat:
7 files changed, 113 insertions(+), 27 deletions(-)
diff --git a/src/cg/arith.c b/src/cg/arith.c
@@ -100,6 +100,10 @@ void api_cg_binop(KitCg* g, BinOp iop, u32 flags) {
rr = api_alloc_temp_local(g, ty);
dst = api_op_local(rr, ty);
+ /* Flag dead-transient operands so the -O0 backend drops them after the op
+ * instead of spilling them at the next barrier (eager dead-operand drop). */
+ ra = api_op_kill_if_dead(g, ra, dst);
+ rb = api_op_kill_if_dead(g, rb, dst);
T->binop(T, iop, dst, ra, rb);
api_release(g, &a);
api_release(g, &b);
@@ -132,6 +136,7 @@ void api_cg_unop(KitCg* g, UnOp iop, u32 flags) {
ra = api_force_local(g, &a, ty);
rr = api_alloc_temp_local(g, ty);
dst = api_op_local(rr, ty);
+ ra = api_op_kill_if_dead(g, ra, dst);
T->unop(T, iop, dst, ra);
api_release(g, &a);
api_push(g, api_make_sv(dst, ty));
@@ -171,6 +176,7 @@ void api_cg_unop(KitCg* g, UnOp iop, u32 flags) {
}
rr = api_alloc_temp_local(g, ty);
dst = api_op_local(rr, ty);
+ ra = api_op_kill_if_dead(g, ra, dst);
T->unop(T, iop, dst, ra);
api_release(g, &a);
api_push(g, api_make_sv(dst, ty));
diff --git a/src/cg/control.c b/src/cg/control.c
@@ -33,7 +33,13 @@ void api_branch_if(KitCg* g, ApiSValue* v, int branch_when_true, Label label) {
if (v->kind == SV_CMP) {
CmpOp op = branch_when_true ? v->delayed->cmp.op
: api_invert_cmp(v->delayed->cmp.op);
- T->cmp_branch(T, op, v->delayed->cmp.a, v->delayed->cmp.b, label);
+ /* Flag dead-transient operands so the -O0 backend drops them at the branch
+ * instead of spilling them (eager dead-operand drop). The compare has no
+ * local destination, so pass an immediate as the dst sentinel. */
+ Operand none = api_op_imm(0, builtin_id(KIT_CG_BUILTIN_I32));
+ Operand ca = api_op_kill_if_dead(g, v->delayed->cmp.a, none);
+ Operand cb = api_op_kill_if_dead(g, v->delayed->cmp.b, none);
+ T->cmp_branch(T, op, ca, cb, label);
api_release(g, v);
return;
}
@@ -64,6 +70,9 @@ void api_branch_if(KitCg* g, ApiSValue* v, int branch_when_true, Label label) {
{
Operand a = api_force_local(g, v, ty);
Operand zero = api_op_imm(0, ty);
+ /* The truthiness test is `v`'s last use; drop it at the branch if it is a
+ * dead transient (zero is the non-local dst sentinel). */
+ a = api_op_kill_if_dead(g, a, zero);
T->cmp_branch(T, branch_when_true ? CMP_NE : CMP_EQ, a, zero, label);
api_release(g, v);
}
diff --git a/src/cg/fold.c b/src/cg/fold.c
@@ -209,7 +209,11 @@ void api_release_cmp(KitCg* g, ApiSValue* sv) {
void api_materialize_cmp_to(KitCg* g, ApiSValue* sv, Operand dst) {
ApiDelayed* d = sv->delayed;
- g->target->cmp(g->target, d->cmp.op, dst, d->cmp.a, d->cmp.b);
+ /* Flag dead-transient operands so the -O0 backend drops them after the compare
+ * instead of spilling them at the next barrier (eager dead-operand drop). */
+ Operand a = api_op_kill_if_dead(g, d->cmp.a, dst);
+ Operand b = api_op_kill_if_dead(g, d->cmp.b, dst);
+ g->target->cmp(g->target, d->cmp.op, dst, a, b);
api_delayed_free(g, d);
sv->delayed = NULL;
sv->kind = SV_OPERAND;
@@ -269,9 +273,14 @@ void api_release_arith(KitCg* g, ApiSValue* sv) {
void api_materialize_arith_to(KitCg* g, ApiSValue* sv, Operand dst) {
ApiDelayed* d = sv->delayed;
if (d->arith.kind == API_DELAYED_UNOP) {
- g->target->unop(g->target, d->arith.un_op, dst, d->arith.a);
+ Operand a = api_op_kill_if_dead(g, d->arith.a, dst);
+ g->target->unop(g->target, d->arith.un_op, dst, a);
} else {
- g->target->binop(g->target, d->arith.bin_op, dst, d->arith.a, d->arith.b);
+ /* Flag dead-transient operands so the -O0 backend drops them after the op
+ * instead of spilling them at the next barrier (eager dead-operand drop). */
+ Operand a = api_op_kill_if_dead(g, d->arith.a, dst);
+ Operand b = api_op_kill_if_dead(g, d->arith.b, dst);
+ g->target->binop(g->target, d->arith.bin_op, dst, a, b);
}
api_delayed_free(g, d);
sv->delayed = NULL;
diff --git a/src/cg/internal.h b/src/cg/internal.h
@@ -535,6 +535,15 @@ void api_release(KitCg* g, ApiSValue* sv);
* its references are re-accounted. */
int api_coalesce_on(KitCg* g);
int api_temp_dead(KitCg* g, CGLocal local);
+/* Flag OP for the -O0 backend's eager dead-operand drop: when coalescing is on and
+ * OP names a transient that api_temp_dead confirms is dead after the consuming op,
+ * AND OP is distinct from that op's destination DST (so it is not the result being
+ * written), return OP with OPK_FLAG_KILL set; otherwise return OP unchanged. The
+ * NativeDirectTarget then drops OP's live cache register after the op (no spill, no
+ * reload) instead of leaving it resident to be flush-stored at the next barrier.
+ * Pass a non-OPK_LOCAL operand for DST (e.g. the store's memory place) when the op
+ * has no local destination — the self-guard is then a no-op. */
+Operand api_op_kill_if_dead(KitCg* g, Operand op, Operand dst);
void api_reseat_begin(KitCg* g, const ApiSValue* sv);
void api_reseat_end(KitCg* g, const ApiSValue* sv);
diff --git a/src/cg/memory.c b/src/cg/memory.c
@@ -564,6 +564,12 @@ void kit_cg_store(KitCg* g, KitCgMemAccess access) {
api_local_const_memory_boundary(g);
}
+ /* Flag a dead-transient store value so the -O0 backend drops it after the store
+ * instead of spilling it at the next barrier (eager dead-operand drop). mem_op
+ * is the memory place (OPK_INDIRECT/GLOBAL, never a scalar local — that is the
+ * scalar_local_place fast path above), so the dst self-guard is a no-op and the
+ * store has already read the address before the drop. */
+ src = api_op_kill_if_dead(g, src, mem_op);
if (is_bitfield) {
/* A bit-field store rides the generic `store` with a bit-field MemAccess;
* the CgTarget impl does the read-modify-write insert. */
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -1100,6 +1100,22 @@ static Reg nd_rename_killed_to_dst(NativeDirectTarget* d, Operand dst,
return r;
}
+/* After an op has materialized and consumed OP, drop OP's cache entry WITHOUT a
+ * write-back when it carries OPK_FLAG_KILL — a transient the cg layer proved dead
+ * after this op (api_temp_dead). The op already read OP's value, so storing it at
+ * the next barrier would be wasted and holding its register inflates pressure;
+ * dropping it now both removes that dead spill and frees the register immediately
+ * (fewer eviction spills + reloads downstream). Sound for the same reason
+ * nd_rename_killed_to_dst is: OPK_FLAG_KILL is set only from api_temp_dead, which
+ * confirms no live value-stack reference to OP remains. A non-local or non-cached
+ * operand is a no-op (nd_invalidate_local early-returns on REG_NONE). The caller
+ * must have already unpinned OP's materialized register (nd_release_materialized);
+ * the cg layer never flags an operand that is also the op's destination. */
+static void nd_drop_killed_operand(NativeDirectTarget* d, Operand op) {
+ if ((op.flags & OPK_FLAG_KILL) && op.kind == OPK_LOCAL)
+ nd_invalidate_local(d, op.v.local);
+}
+
static void nd_store_operand_from_reg(NativeDirectTarget* d, Operand dst,
NativeLoc src) {
if (dst.kind != OPK_LOCAL) nd_panic(d, "destination is not a semantic local");
@@ -1295,9 +1311,20 @@ static void nd_cmp_branch(CgTarget* t, CmpOp op, Operand a, Operand b,
Label label) {
NativeDirectTarget* d = nd_of(t);
NativeLoc ar, br;
- nd_flush_all(d);
+ /* Materialize the compare operands BEFORE spilling the live-across set. A
+ * cache-resident operand is then read straight from its register instead of
+ * being spilled by the flush and immediately reloaded for the compare; a dead
+ * one (OPK_FLAG_KILL) is dropped without a write-back so the flush skips its
+ * store entirely. The operands stay pinned across the flush — their values
+ * survive in their registers (a spill store does not clobber them) — and the
+ * flush's stores are emitted before the compare, so they cannot clobber its
+ * condition flags. The post-branch cache state is still empty (the flush spills
+ * everything live), so control-flow-merge correctness is unchanged. */
ar = nd_materialize_operand(d, a);
br = nd_rhs_imm_or_reg(d, NATIVE_IMM_CMP, (u32)op, b);
+ nd_drop_killed_operand(d, a);
+ nd_drop_killed_operand(d, b);
+ nd_flush_all(d);
ND_REQUIRE_NATIVE(d, cmp_branch, "target does not emit compare branches");
d->native->cmp_branch(d->native, op, ar, br, nd_mc_label(d, label));
nd_release_materialized(d, br);
@@ -1569,14 +1596,12 @@ static void nd_load(CgTarget* t, Operand dst, Operand addr, MemAccess mem) {
return;
}
size = mem.size ? mem.size : (mem.type ? cg_type_size(t->c, mem.type) : 0);
- /* No value-cache flush: only escaped (address-taken / memory-required) locals
- * can be aliased through a pointer, and those are never cached. A volatile
- * access may observe memory and needs the cache made authoritative first. */
- if (mem.flags & MF_VOLATILE) {
- nd_flush_all(d);
- nd_barrier(d,
- NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_VOLATILE);
- }
+ /* No value-cache flush (2c): the cache holds only non-escaped locals, which no
+ * pointer can alias, so a volatile access — foreign memory — can neither observe
+ * a cached value's stale home nor overwrite it. The arch barrier hook still
+ * fires so an instrumentation seam can order the access. */
+ if (mem.flags & MF_VOLATILE)
+ nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_VOLATILE);
NativeAddr naddr =
nd_addr_materialize(d, nd_addr_storage(d, addr), &temps, mem);
if (size > (u64)t->c->target.ptr_size) {
@@ -1625,15 +1650,12 @@ static void nd_store(CgTarget* t, Operand addr, Operand src, MemAccess mem) {
return;
}
size = mem.size ? mem.size : (mem.type ? cg_type_size(t->c, mem.type) : 0);
- /* No value-cache flush (see nd_load): a store through a pointer cannot alias
- * a cached non-escaped local. The store target is foreign memory, so there is
- * no dst local entry to invalidate; SRC is read via nd_materialize_operand.
- */
- if (mem.flags & MF_VOLATILE) {
- nd_flush_all(d);
- nd_barrier(d,
- NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_VOLATILE);
- }
+ /* No value-cache flush (2c, see nd_load): a store through a pointer cannot
+ * alias a cached non-escaped local. The store target is foreign memory, so
+ * there is no dst local entry to invalidate; SRC is read via
+ * nd_materialize_operand. The arch barrier hook still fires. */
+ if (mem.flags & MF_VOLATILE)
+ nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_VOLATILE);
NativeAddr naddr =
nd_addr_materialize(d, nd_addr_storage(d, addr), &temps, mem);
if (size > (u64)t->c->target.ptr_size) {
@@ -1656,6 +1678,9 @@ static void nd_store(CgTarget* t, Operand addr, Operand src, MemAccess mem) {
d->native->store(d->native, naddr, val, mem);
nd_release_materialized(d, val);
nd_addr_temps_release(d, &temps);
+ /* The store consumed SRC; if it was a dead transient, drop it (no write-back)
+ * so it is not flush-stored at the next barrier (eager dead-operand drop). */
+ nd_drop_killed_operand(d, src);
}
static void nd_addr_of(CgTarget* t, Operand dst, Operand lv) {
@@ -1826,6 +1851,11 @@ static void nd_binop(CgTarget* t, BinOp op, Operand dst, Operand a, Operand b) {
nd_dst_writeback(d, dst, dr);
nd_release_materialized(d, br);
nd_release_materialized(d, ar);
+ /* Eager dead-operand drop: a/b flagged dead by the cg layer (OPK_FLAG_KILL) are
+ * consumed now — drop their cache entries without a write-back so they are not
+ * flush-stored at the next barrier and their registers free immediately. */
+ nd_drop_killed_operand(d, b);
+ nd_drop_killed_operand(d, a);
}
static void nd_unop(CgTarget* t, UnOp op, Operand dst, Operand a) {
@@ -1850,6 +1880,7 @@ static void nd_unop(CgTarget* t, UnOp op, Operand dst, Operand a) {
d->native->unop(d->native, op, dr, ar);
nd_dst_writeback(d, dst, dr);
nd_release_materialized(d, ar);
+ nd_drop_killed_operand(d, a);
}
static void nd_cmp(CgTarget* t, CmpOp op, Operand dst, Operand a, Operand b) {
@@ -1883,6 +1914,8 @@ static void nd_cmp(CgTarget* t, CmpOp op, Operand dst, Operand a, Operand b) {
nd_dst_writeback(d, dst, dr);
nd_release_materialized(d, br);
nd_release_materialized(d, ar);
+ nd_drop_killed_operand(d, b);
+ nd_drop_killed_operand(d, a);
}
static void nd_convert(CgTarget* t, ConvKind op, Operand dst, Operand src) {
@@ -2202,7 +2235,8 @@ static void nd_atomic_load(CgTarget* t, Operand dst, Operand addr,
MemAccess mem, KitCgMemOrder order) {
NativeDirectTarget* d = nd_of(t);
NdAddrTemps temps;
- nd_flush_all(d);
+ /* No value-cache flush (2c): the atomic accesses foreign memory, which cannot
+ * alias a cached non-escaped local; the arch barrier hook still orders it. */
nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_ATOMIC);
NativeAddr naddr =
nd_addr_materialize(d, nd_addr_pointer(d, addr), &temps, mem);
@@ -2218,7 +2252,8 @@ static void nd_atomic_store(CgTarget* t, Operand addr, Operand src,
MemAccess mem, KitCgMemOrder order) {
NativeDirectTarget* d = nd_of(t);
NdAddrTemps temps;
- nd_flush_all(d);
+ /* No value-cache flush (2c): foreign atomic memory cannot alias a cached
+ * non-escaped local. */
nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_ATOMIC);
NativeAddr naddr =
nd_addr_materialize(d, nd_addr_pointer(d, addr), &temps, mem);
@@ -2234,7 +2269,8 @@ static void nd_atomic_rmw(CgTarget* t, KitCgAtomicOp op, Operand dst,
KitCgMemOrder order) {
NativeDirectTarget* d = nd_of(t);
NdAddrTemps temps;
- nd_flush_all(d);
+ /* No value-cache flush (2c): foreign atomic memory cannot alias a cached
+ * non-escaped local. */
nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_ATOMIC);
NativeAddr naddr =
nd_addr_materialize(d, nd_addr_pointer(d, addr), &temps, mem);
@@ -2253,7 +2289,8 @@ static void nd_atomic_cas(CgTarget* t, Operand prior, Operand ok, Operand addr,
KitCgMemOrder success, KitCgMemOrder failure) {
NativeDirectTarget* d = nd_of(t);
NdAddrTemps temps;
- nd_flush_all(d);
+ /* No value-cache flush (2c): foreign atomic memory cannot alias a cached
+ * non-escaped local. */
nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_ATOMIC);
NativeAddr naddr =
nd_addr_materialize(d, nd_addr_pointer(d, addr), &temps, mem);
@@ -2276,7 +2313,9 @@ static void nd_atomic_cas(CgTarget* t, Operand prior, Operand ok, Operand addr,
static void nd_fence(CgTarget* t, KitCgMemOrder order) {
NativeDirectTarget* d = nd_of(t);
- nd_flush_all(d);
+ /* No value-cache flush (2c): a fence orders memory visible to other agents; a
+ * cached local is non-escaped, so no other agent can observe its deferred home
+ * write. The arch fence instruction is still emitted. */
ND_REQUIRE_NATIVE(d, fence, "target does not emit fences");
d->native->fence(d->native, order);
}
diff --git a/src/cg/value.c b/src/cg/value.c
@@ -572,6 +572,14 @@ int api_temp_dead(KitCg* g, CGLocal local) {
return api_temp_scan_refs(g, local) == 0; /* confirm before acting */
}
+Operand api_op_kill_if_dead(KitCg* g, Operand op, Operand dst) {
+ if (api_coalesce_on(g) && op.kind == OPK_LOCAL &&
+ (dst.kind != OPK_LOCAL || op.v.local != dst.v.local) &&
+ api_temp_dead(g, op.v.local))
+ op.flags |= OPK_FLAG_KILL;
+ return op;
+}
+
/* ---- BinOp / UnOp / CmpOp mapping ---- */
BinOp api_map_int_binop(KitCgIntBinOp op) {
