commit 2b1b03c21d1b949107cf692febb5a709969608f0
parent 93aafda346d271d17ce2ea95abb2083e4ff896b2
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 06:49:38 -0700
perf(cg): -O0 assignment coalescing — RHS flows straight into the local
Two coordinated changes remove the temp->local routing `mov` for the common
`x = <expr>` / `int x = <expr>`:
1. frontend (cg_adapter): pcg_store always dup'd the RHS to preserve the
assignment's expression value, but 34 of 36 store sites immediately drop
that value (`pcg_store; pcg_drop` — statements and initializers). Add
pcg_store_void (keep_result=0) that consumes [place, value] without the
dup, and convert those 34 sites. The un-dup'd RHS now reaches kit_cg_store
still delayed (the 2 result-keeping sites — `a=b=c`, `if((x=f()))` — keep
pcg_store).
2. cg (kit_cg_store): when a still-delayed arith/cmp is stored into a scalar
local, materialize it DIRECTLY into the local (api_materialize_{arith,cmp}_to
with dst = the local) instead of into a temp that is then copied in. The
producer targets the consumer's storage — no temp, no copy.
chain(): add w8,w9,#1; mov w11,w8; lsl w8,w11,#1; mov w12,w8; sub w8,w12,#3;
mov w13,w8 -> add w8,w9,#1; lsl w11,w8,#1; sub w12,w11,#3
sqlite3.c -c -O0 (arm64-macOS): compile instrs 2,150.9M -> 2,145.0M (-5.9M);
emitted insns 520,854 -> 518,757; object 2,477,912 -> 2,469,528 B.
Byte-deterministic (compile x2 identical).
Green: test-toy 1392/0, test-parse-ok 3920/0, test-parse-err 129/0,
test-smoke-x64 3/0, test-smoke-rv64 3/0; sqlite end-to-end 84|2.
Diffstat:
8 files changed, 96 insertions(+), 84 deletions(-)
diff --git a/lang/c/parse/cg_adapter.c b/lang/c/parse/cg_adapter.c
@@ -684,7 +684,13 @@ void pcg_branch_false(Parser* p, CGLabel l) {
/* Store [lv, rv] -> [rv]. The expression-value of an assignment is the
* assigned rvalue, so the store sequence must leave a copy of rv on TOS. */
-void pcg_store(Parser* p) {
+/* keep_result == 0: consume [place, value] and leave nothing — the assignment's
+ * value is discarded (the dominant `expr;` / initializer case). Skipping the
+ * dup lets a still-delayed RHS flow straight into the destination local
+ * (kit_cg_store's scalar-local fast path), killing the temp->local routing mov.
+ * keep_result == 1: leave the assignment's value on the stack for an enclosing
+ * expression (`a = b = c`, `if ((x = f()))`). */
+static void pcg_store_impl(Parser* p, int keep_result) {
const Type* lv_ty = pcg_top2_type(p);
const Type* rv_ty = pcg_top_type(p);
const Type* mem_ty = lv_ty;
@@ -703,11 +709,15 @@ void pcg_store(Parser* p) {
rv_ty && (rv_ty->kind == TY_INT128 || rv_ty->kind == TY_UINT128 ||
rv_ty->kind == TY_LDOUBLE);
if (pcg_lv_is_trivial_local(lv) && !wide) {
- /* The destination place is already on the CG stack. Keep a copy of rv as
- * the assignment's value. Stack: [place, value]. */
- kit_cg_dup(p->cg);
- kit_cg_rot3(p->cg);
- kit_cg_swap(p->cg);
+ /* The destination place is already on the CG stack: [place, value]. When
+ * the result is kept, dup rv first so a copy survives the store; when it
+ * is discarded, store straight from [place, value] so the RHS (possibly a
+ * still-delayed arith/cmp) lands directly in the local. */
+ if (keep_result) {
+ kit_cg_dup(p->cg);
+ kit_cg_rot3(p->cg);
+ kit_cg_swap(p->cg);
+ }
kit_cg_store(p->cg, access);
} else {
/* Stash rv into a temp so the destination place can be built from the
@@ -740,15 +750,24 @@ void pcg_store(Parser* p) {
kit_cg_push_local(p->cg, tmp);
kit_cg_load(p->cg, rv_access); /* [dst, value] */
kit_cg_store(p->cg, access); /* [] */
- kit_cg_push_local(p->cg, tmp);
- kit_cg_load(p->cg, rv_access); /* [value] */
+ if (keep_result) {
+ kit_cg_push_local(p->cg, tmp);
+ kit_cg_load(p->cg, rv_access); /* [value] */
+ }
}
}
pcg_drop_type(p);
pcg_drop_type(p);
- pcg_push_type(p, rv_ty);
+ if (keep_result) pcg_push_type(p, rv_ty);
}
+void pcg_store(Parser* p) { pcg_store_impl(p, 1); }
+
+/* Store whose assignment value is discarded (the `expr;` / initializer case).
+ * Replaces the `pcg_store(p); pcg_drop(p)` idiom; avoids materializing the RHS
+ * into a temp just to drop it. */
+void pcg_store_void(Parser* p) { pcg_store_impl(p, 0); }
+
void pcg_deref(Parser* p, const Type* pointee) {
const Type* ptr_ty = pcg_top_type(p);
if (pointee && pointee->kind == TY_FUNC) {
diff --git a/lang/c/parse/cg_adapter.h b/lang/c/parse/cg_adapter.h
@@ -300,6 +300,8 @@ void pcg_jump(Parser*, CGLabel);
void pcg_branch_true(Parser*, CGLabel);
void pcg_branch_false(Parser*, CGLabel);
void pcg_store(Parser*);
+/* Store whose assignment value is discarded (replaces `pcg_store; pcg_drop`). */
+void pcg_store_void(Parser*);
void pcg_deref(Parser*, const Type*);
/* ---- Lvalue auxiliary access ----
diff --git a/lang/c/parse/parse.c b/lang/c/parse/parse.c
@@ -568,8 +568,7 @@ static void store_top_to_size_slot(Parser* p, FrameSlot slot) {
pcg_push_local_typed(p, slot, ty_size_t(p));
pcg_swap(p);
coerce_top_to_lvalue(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
static void reset_vla_pending(Parser* p) {
@@ -947,8 +946,7 @@ static void parse_init_declarator(Parser* p, const DeclSpecs* specs) {
pcg_alloca(p);
pcg_push_local_typed(p, ptr_slot, ptr_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
sym_entry = scope_lookup(p, name);
if (sym_entry && sym_entry->kind == SEK_LOCAL) {
sym_entry->vla_byte_slot = byte_slot;
@@ -1007,8 +1005,7 @@ static void parse_init_declarator(Parser* p, const DeclSpecs* specs) {
perr(p, "%.*s", KIT_SLICE_ARG(kit_slice_cstr(chk.message)));
}
coerce_top_to_lvalue(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
} else if (p->auto_var_init == KIT_AUTOVAR_ZERO && var_ty &&
!(var_ty->kind == TY_ARRAY && var_ty->arr.incomplete)) {
diff --git a/lang/c/parse/parse_expr.c b/lang/c/parse/parse_expr.c
@@ -1440,8 +1440,7 @@ static int parse_builtin_overflow_call(Parser* p, Sym name, SrcLoc loc) {
ptr_slot = builtin_tmp_slot(p, ptr_ty);
pcg_push_local_typed(p, ptr_slot, ptr_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_set_loc(p, loc);
if (pcg_emit_enabled(p)) {
@@ -1455,15 +1454,13 @@ static int parse_builtin_overflow_call(Parser* p, Sym name, SrcLoc loc) {
ov_slot = builtin_tmp_slot(p, bool_ty);
pcg_push_local_typed(p, ov_slot, bool_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_push_local_typed(p, ptr_slot, ptr_ty);
pcg_load(p);
pcg_deref(p, out_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_push_local_typed(p, ov_slot, bool_ty);
pcg_load(p);
@@ -1554,13 +1551,11 @@ static int parse_builtin_fp_cmp_call(Parser* p, Sym name, SrcLoc loc) {
/* stack: [a, b] */
pcg_push_local_typed(p, slot_b, common);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
/* stack: [a] */
pcg_push_local_typed(p, slot_a, common);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
/* stack: [] */
pcg_push_local_typed(p, slot_a, common);
pcg_load(p);
@@ -1673,8 +1668,7 @@ static int parse_builtin_fabs_call(Parser* p, Sym name, SrcLoc loc) {
slot = builtin_tmp_slot(p, ty);
pcg_push_local_typed(p, slot, ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_set_loc(p, loc);
pcg_push_local_typed(p, slot, ty);
@@ -1687,8 +1681,7 @@ static int parse_builtin_fabs_call(Parser* p, Sym name, SrcLoc loc) {
pcg_push_local_typed(p, slot, ty);
pcg_load(p);
pcg_unop(p, UO_NEG);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_label_place(p, L_nonneg);
pcg_push_local_typed(p, slot, ty);
pcg_load(p);
@@ -1698,8 +1691,7 @@ static int parse_builtin_fabs_call(Parser* p, Sym name, SrcLoc loc) {
pcg_branch_false(p, L_nonzero);
pcg_push_local_typed(p, slot, ty);
pcg_push_float(p, 0.0, ty);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_label_place(p, L_nonzero);
pcg_push_local_typed(p, slot, ty);
pcg_load(p);
@@ -2029,8 +2021,7 @@ static int try_parse_builtin_call(Parser* p) {
FrameSlot eslot = pcg_local(p, &fsd);
pcg_push_local_typed(p, eslot, eptr_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_push_local_typed(p, eslot, eptr_ty);
pcg_load(p);
@@ -2063,8 +2054,7 @@ static int try_parse_builtin_call(Parser* p) {
FrameSlot okslot = pcg_local(p, &okd);
pcg_push_local_typed(p, okslot, ok_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
FrameSlotDesc pd;
memset(&pd, 0, sizeof pd);
@@ -2075,8 +2065,7 @@ static int try_parse_builtin_call(Parser* p) {
FrameSlot pslot = pcg_local(p, &pd);
pcg_push_local_typed(p, pslot, val_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_push_local_typed(p, okslot, ok_ty);
pcg_load(p);
@@ -2087,8 +2076,7 @@ static int try_parse_builtin_call(Parser* p) {
pcg_deref(p, val_ty);
pcg_push_local_typed(p, pslot, val_ty);
pcg_load(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_label_place(p, L_done);
pcg_push_local_typed(p, okslot, ok_ty);
@@ -3271,8 +3259,7 @@ static FrameSlot ll_tmp_slot(Parser* p, const Type* ty) {
static void ll_store_const(Parser* p, FrameSlot tmp, const Type* ty, i64 v) {
pcg_push_local_typed(p, tmp, ty);
pcg_push_int(p, v, ty);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
static void parse_land(Parser* p) {
@@ -3398,8 +3385,7 @@ static void parse_ternary(Parser* p) {
if (pcg_top_type(p) != then_store_ty) pcg_convert(p, then_store_ty);
pcg_push_local_typed(p, tmp, then_store_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_jump(p, L_end);
pcg_label_place(p, L_else);
expect_punct(p, ':', "':' in ternary");
@@ -3424,8 +3410,7 @@ static void parse_ternary(Parser* p) {
}
pcg_push_local_typed(p, tmp, result_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_label_place(p, L_end);
if (common && common != result_ty) {
FrameSlotDesc cfsd;
@@ -3442,8 +3427,7 @@ static void parse_ternary(Parser* p) {
pcg_convert(p, common);
pcg_push_local_typed(p, ctmp, common);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_push_local_typed(p, ctmp, common);
return;
}
diff --git a/lang/c/parse/parse_init.c b/lang/c/parse/parse_init.c
@@ -214,8 +214,7 @@ static void emit_copy_leaf(Parser* p, FrameSlot dst_slot,
pcg_deref(p, leaf_ty);
pcg_lv_member(p, (i64)src_off, leaf_ty, 0, 0, 0);
pcg_load(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
/* Walk a (possibly nested) aggregate, emitting a leaf load+store for each
@@ -276,8 +275,7 @@ void emit_struct_copy_into_slot(Parser* p, FrameSlot dst_slot,
src_ptr_slot = pcg_local(p, &fsd);
pcg_push_local_typed(p, src_ptr_slot, ptr_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
emit_walk_copy(p, dst_slot, dst_arr_ty, dst_off, src_ptr_slot, ptr_ty, 0, ty);
}
@@ -299,8 +297,7 @@ void zero_init_at(Parser* p, FrameSlot slot, const Type* arr_ty, u32 offset,
if (f->flags & FIELD_BITFIELD) {
push_record_field_lv(p, slot, arr_ty, offset, ty, i);
pcg_push_int(p, 0, f->type);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
continue;
}
{
@@ -322,8 +319,7 @@ void zero_init_at(Parser* p, FrameSlot slot, const Type* arr_ty, u32 offset,
}
push_subobject_lv(p, slot, arr_ty, offset, ty);
pcg_push_int(p, 0, ty);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
static void init_field_at(Parser* p, FrameSlot slot, const Type* arr_ty,
@@ -341,8 +337,7 @@ static void init_field_at(Parser* p, FrameSlot slot, const Type* arr_ty,
if (!chk.ok) perr(p, "%.*s", KIT_SLICE_ARG(kit_slice_cstr(chk.message)));
}
coerce_top_to_lvalue(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
return;
}
init_at(p, slot, arr_ty, rec_offset + L->fields[field_index].offset, f->type);
@@ -363,14 +358,12 @@ static void init_string_at(Parser* p, FrameSlot slot, const Type* arr_ty,
push_subobject_lv(p, slot, arr_ty, offset + (u32)i * elem_size, elem_ty);
pcg_push_int(p, (i64)decode_lit_unit_le(bytes + i * elem_size, elem_size),
elem_ty);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
for (; i < count; ++i) {
push_subobject_lv(p, slot, arr_ty, offset + (u32)i * elem_size, elem_ty);
pcg_push_int(p, 0, elem_ty);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
kit_compiler_context(p->c)->heap->free(kit_compiler_context(p->c)->heap,
bytes, 0);
@@ -528,8 +521,7 @@ static u32 init_struct_fields(Parser* p, FrameSlot slot, const Type* arr_ty,
if (!(zf->flags & FIELD_ZERO_WIDTH)) {
push_record_field_lv(p, slot, arr_ty, offset, ty, zero_lo);
pcg_push_int(p, 0, zf->type);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
} else {
u32 zoff = offset + L->fields[zero_lo].offset;
@@ -582,8 +574,7 @@ static u32 init_struct_fields(Parser* p, FrameSlot slot, const Type* arr_ty,
if (!(f->flags & FIELD_ZERO_WIDTH)) {
push_record_field_lv(p, slot, arr_ty, offset, ty, j);
pcg_push_int(p, 0, f->type);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
} else {
u32 foff = offset + L->fields[j].offset;
@@ -626,8 +617,7 @@ static void init_aggregate_remainder(Parser* p, FrameSlot slot,
if (!(f->flags & FIELD_ZERO_WIDTH)) {
push_record_field_lv(p, slot, arr_ty, offset, ty, i);
pcg_push_int(p, 0, f->type);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
} else {
const Type* fty = init_field_type_at(ty, i);
@@ -763,8 +753,7 @@ void init_at(Parser* p, FrameSlot slot, const Type* arr_ty, u32 offset,
if (!chk.ok) perr(p, "%.*s", KIT_SLICE_ARG(kit_slice_cstr(chk.message)));
}
coerce_top_to_lvalue(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
if (had_brace) {
accept_punct(p, ',');
expect_punct(p, '}', "'}' after scalar initializer");
diff --git a/lang/c/parse/parse_stmt.c b/lang/c/parse/parse_stmt.c
@@ -468,8 +468,7 @@ static void parse_switch_stmt(Parser* p) {
pcg_push_local_typed(p, ctx.value_slot, vty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
pcg_jump(p, L_dispatch);
@@ -702,8 +701,7 @@ static void parse_asm_stmt(Parser* p) {
slot = pcg_local(p, &fsd);
pcg_push_local_typed(p, slot, ptr_ty);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
lv.addr_slot = slot;
lv.ptr_ty = ptr_ty;
}
@@ -843,8 +841,7 @@ static void parse_asm_stmt(Parser* p) {
AsmOutLValue* lv = &out_lvs[i];
asm_out_lvalue_push(p, lv);
pcg_swap(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
}
}
}
diff --git a/lang/c/parse/parse_type.c b/lang/c/parse/parse_type.c
@@ -1462,8 +1462,7 @@ int parse_decl_suffix(Parser* p, DeclSuffix* out) {
pcg_push_local_typed(p, out->vla_count_slot, fsd.type);
pcg_swap(p);
coerce_top_to_lvalue(p);
- pcg_store(p);
- pcg_drop(p);
+ pcg_store_void(p);
p->vla_pending = 1;
++p->vla_mark;
p->vla_pending_count_slot = out->vla_count_slot;
diff --git a/src/cg/memory.c b/src/cg/memory.c
@@ -475,6 +475,37 @@ void kit_cg_store(KitCg* g, KitCgMemAccess access) {
rv = api_make_wide8_int_const(g, rv.op.v.imm, ty);
}
+ /* Does this store land a scalar value straight into a local's own storage
+ * (a plain `x = <expr>` / `int x = <expr>`, not bit-field/aggregate/indirect)?
+ */
+ int scalar_local_place =
+ !is_bitfield && base.source_local != KIT_CG_LOCAL_NONE &&
+ base.op.kind == OPK_LOCAL &&
+ !api_sv_local_storage_is_aggregate(g, &base) &&
+ !cg_type_is_aggregate(g->c, api_sv_type(&base)) &&
+ !cg_type_is_aggregate(g->c, ty) &&
+ api_unalias_type(g->c, api_sv_type(&base)) ==
+ api_unalias_type(g->c, ty);
+
+ /* A still-delayed arith/cmp value going into a scalar local: emit the op
+ * DIRECTLY into the local instead of materializing it into a temp and then
+ * copying that in. Kills the temp->local routing mov for the common
+ * `x = <expr>` — the producer targets the consumer's storage. (Equivalence:
+ * api_materialize_*_to with dst=base.op emits the same op the old temp got,
+ * just landing in base's storage; binops read both sources before writing the
+ * dst, so a self-referential RHS like `b = b + c` stays correct.) */
+ if (scalar_local_place && (rv.kind == SV_ARITH || rv.kind == SV_CMP)) {
+ Operand dst = base.op;
+ if (rv.kind == SV_ARITH)
+ api_materialize_arith_to(g, &rv, dst);
+ else
+ api_materialize_cmp_to(g, &rv, dst);
+ api_local_const_clear(api_local_from_handle(g, base.source_local));
+ api_release(g, &base);
+ api_release(g, &rv);
+ return;
+ }
+
/* General scalar / bit-field store. Compute the source operand first so its
* local lifetime doesn't overlap any addressing arith. */
api_ensure_local(g, &rv);
@@ -485,13 +516,7 @@ void kit_cg_store(KitCg* g, KitCgMemAccess access) {
}
/* Scalar local-resident place, plain store: copy into the local. */
- if (!is_bitfield && base.source_local != KIT_CG_LOCAL_NONE &&
- base.op.kind == OPK_LOCAL &&
- !api_sv_local_storage_is_aggregate(g, &base) &&
- !cg_type_is_aggregate(g->c, api_sv_type(&base)) &&
- !cg_type_is_aggregate(g->c, ty) &&
- api_unalias_type(g->c, api_sv_type(&base)) ==
- api_unalias_type(g->c, ty)) {
+ if (scalar_local_place) {
Operand dst = base.op;
if (src.kind == OPK_IMM) {
T->load_imm(T, dst, src.v.imm);
