commit 9fb2baf5d5280fc66fafcff63cf83bf9fca0eaec
parent cb91bcba2cc4a3ceedd6bf84e636bee09c9acf37
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 07:55:21 -0700
perf(cg): -O0 convert coalescing — rename a dead source's register in place
A convert (sext/zext/trunc/bitcast) at -O0 sourced its operand into a pinned
register and wrote a separate scratch destination, so a chain like
`add w8,w9,#1; sxtw x11,w8; mov x0,x11` spent a fresh register (and, for a
width-preserving zext, an extra mov) on every widening.
This adds the coalescing mechanism the value-stack-residency work unblocked:
- A new per-operand hint OPK_FLAG_KILL (Operand.flags, reusing a pad byte;
additive — only the -O0 NativeDirectTarget reads it, every other backend
ignores it). The cg sets it on a convert's source operand when that source is
a transient with no other live value-stack reference (api_temp_dead, sound —
the source value `v` was already popped, so a true reading means dead).
- nd_rename_killed_to_dst: transfers ownership of the dead source's live cache
register to the destination (drop src no-writeback, take the reg for dst,
mark dirty), gated on both being cacheable scalars in the same register
class. nd_convert then converts in place (dr == sr); nd_copy gets the same
hook for a same-type copy (dormant until a copy site sets the flag).
- aa64 aa_convert: the >=32-bit zext path (`mov w,w` zero-extends) now elides
the move when dst and src are the same hard register — sound because the
source arrived from a 32-bit-writing op that already cleared the upper half.
Exposed (and only reachable) by the in-place rename.
chain(): add w8,w9,#1; sxtw x11,w8; mov x0,x11
-> add w8,w9,#1; sxtw x8,w8; mov x0,x8
sqlite3.c -c -O0 (arm64-macOS): emitted insns 474,752 -> 462,967 (-11,785);
object 2,293,504 -> 2,246,368 B (-2.1%); compile instrs ~2,137.0M -> ~2,133.4M.
Byte-deterministic (compile x2 identical).
Green: test-toy 1392/0, test-parse 3920+129/0, test-smoke-x64/rv64 3/0,
test-cg-api 0 fail, test-opt 0 fail, test-libc musl 18 + glibc 9; sqlite e2e
84|2 at -O0 and -O1; clang-differential probe matches (value-live-after-convert,
self-referential, narrow+widen chains) at -O0/-O1/clang.
Diffstat:
4 files changed, 107 insertions(+), 2 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -2501,7 +2501,13 @@ static void aa_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
if (src_bits >= dst_bits) {
aa_move(t, dst, src);
} else if (src_bits >= 32u) {
- aa_emit32(t->mc, aa64_mov_reg(0, loc_reg(dst), loc_reg(src)));
+ /* w-reg mov zero-extends into the x-reg. When dst and src are the same
+ * hard register (the -O0 convert-coalescing rename targets the source's
+ * own register) it is a no-op: the value already arrived there from a
+ * 32-bit-writing op, which itself cleared the upper 32 bits. */
+ if (!(dst.kind == NATIVE_LOC_REG && src.kind == NATIVE_LOC_REG &&
+ dst.v.reg == src.v.reg))
+ aa_emit32(t->mc, aa64_mov_reg(0, loc_reg(dst), loc_reg(src)));
} else {
aa_emit32(t->mc,
aa_ubfm(sf, loc_reg(dst), loc_reg(src), 0, src_bits - 1u));
diff --git a/src/cg/arith.c b/src/cg/arith.c
@@ -329,6 +329,13 @@ void api_cg_convert_kind(KitCg* g, KitCgTypeId dst_type, ConvKind ck) {
src = api_force_local(g, &v, sty);
rr = api_alloc_temp_local(g, dty);
dst = api_op_local(rr, dty);
+ /* If the source operand is a transient with no other live value-stack
+ * reference (v, its last reference, was already popped), flag it dead so the
+ * -O0 backend can convert in place into the source's register instead of
+ * routing through a fresh one — and drop a redundant extend entirely. */
+ if (api_coalesce_on(g) && src.kind == OPK_LOCAL &&
+ api_temp_dead(g, src.v.local))
+ src.flags |= OPK_FLAG_KILL;
T->convert(T, ck, dst, src);
api_release(g, &v);
api_push(g, api_make_sv(dst, dty));
diff --git a/src/cg/cgir.h b/src/cg/cgir.h
@@ -202,6 +202,20 @@ typedef enum OpKind {
OPK_INDIRECT, /* [local + ofs], with optional indexed local */
} OpKind;
+/* Per-operand hint flags carried in Operand.flags. Additive: set by the cg layer
+ * at a specific op's call site on a transient copy of the operand (never on a
+ * value-stack entry), read only by the -O0 NativeDirectTarget; every other
+ * backend (opt recorder, c_target, wasm, check) ignores them. */
+typedef enum OpFlag {
+ OPK_FLAG_NONE = 0,
+ /* This OPK_LOCAL operand names a transient that is provably dead after the op
+ * consuming it (confirmed by api_temp_dead). The -O0 backend may then rename
+ * the local's live cache register to the op's destination (transfer ownership)
+ * instead of materializing the source and emitting a routing mov / in-place
+ * convert into a fresh register. Set only on a copy/convert source. */
+ OPK_FLAG_KILL = 1u << 0,
+} OpFlag;
+
typedef enum CGLocalFlag {
CG_LOCAL_FLAG_NONE = 0,
CG_LOCAL_ADDR_TAKEN = 1u << 0,
@@ -316,7 +330,8 @@ static inline BitFieldAccess bf_from_mem(MemAccess m) {
typedef struct Operand {
u8 kind;
- u8 pad[3];
+ u8 flags; /* OpFlag bitset; 0 except where a cg op sets a per-call-site hint */
+ u8 pad[2];
KitCgTypeId type;
union {
i64 imm;
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -969,6 +969,50 @@ static void nd_dst_writeback(NativeDirectTarget* d, Operand dst, NativeLoc dr) {
nd_release_materialized(d, dr);
}
+/* Coalescing rename: SRC (a transient flagged dead — OPK_FLAG_KILL) holds its
+ * live value in a cache register; transfer ownership of that register to DST so
+ * a copy needs no mov and a convert can run in place. Returns the now-DST-owned
+ * register (pinned for the consuming instruction; the caller passes it to
+ * nd_dst_writeback to unpin + mark dirty), or REG_NONE when a rename is not
+ * applicable and the caller must take its normal materialize path.
+ *
+ * Applicable only when: SRC is cached at its own (storage) width so the register
+ * truly holds SRC's value; SRC and DST are both cacheable scalars in the SAME
+ * register class (a width change within the class — e.g. a sxtb — is fine: the
+ * caller's convert rewrites the register in place; a class change GPR<->FPR is
+ * not, that needs a real cross-bank move). The register's bits become DST's, so
+ * DST is marked dirty (its home is now stale) and pinned; SRC is dropped without
+ * write-back (it is dead). A pre-existing DST cache entry is discarded first
+ * (the copy/convert supersedes it). Sound because OPK_FLAG_KILL is set only from
+ * api_temp_dead, which confirms no live value-stack reference to SRC remains. */
+static Reg nd_rename_killed_to_dst(NativeDirectTarget* d, Operand dst,
+ Operand src) {
+ NativeDirectLocal* sl;
+ NativeDirectLocal* dl;
+ Reg r;
+ if (!(src.flags & OPK_FLAG_KILL) || src.kind != OPK_LOCAL ||
+ dst.kind != OPK_LOCAL || src.v.local == dst.v.local)
+ return REG_NONE;
+ sl = nd_local(d, src.v.local);
+ dl = nd_local(d, dst.v.local);
+ if (sl->reg == REG_NONE || sl->type != src.type || !nd_local_cacheable(d, sl))
+ return REG_NONE;
+ if (sl->cls != dl->cls || !nd_local_cacheable(d, dl)) return REG_NONE;
+ r = sl->reg;
+ /* Detach SRC from R without writing it back (dead) and discard any stale DST
+ * cache entry (its old value is about to be overwritten). */
+ nd_invalidate_local(d, src.v.local);
+ if (dl->reg != REG_NONE) nd_invalidate_local(d, dst.v.local);
+ /* R is now DST's: take ownership, mark dirty (home stale), pin for the op. */
+ d->reg_owner[dl->cls][r] = dst.v.local;
+ dl->reg = r;
+ dl->dirty = 1;
+ nd_cache_link(d, dst.v.local);
+ d->scratch_used[dl->cls] |= 1u << r;
+ nd_touch_local(d, dl);
+ return r;
+}
+
static void nd_store_operand_from_reg(NativeDirectTarget* d, Operand dst,
NativeLoc src) {
if (dst.kind != OPK_LOCAL) nd_panic(d, "destination is not a semantic local");
@@ -1401,6 +1445,19 @@ static void nd_copy(CgTarget* t, Operand dst, Operand src) {
nd_addr_temps_release(d, &dt);
return;
}
+ /* Coalesce `x = t` when t is a dead transient resident in a register: rename
+ * its register to x (transfer ownership) instead of materialize + mov. Only
+ * for an exact same-type copy (the register bits become x's unchanged). */
+ if (dst.type == src.type) {
+ Reg r = nd_rename_killed_to_dst(d, dst, src);
+ if (r != REG_NONE) {
+ /* Register already holds the value as x's; just unpin + mark dirty. */
+ nd_dst_writeback(d, dst,
+ native_loc_reg(dst.type, nd_class_for_type(d, dst.type),
+ r));
+ return;
+ }
+ }
NativeLoc val = nd_materialize_operand(d, src);
NativeLoc dr = nd_dst_reg(d, dst);
nd_copy_to_reg(d, dr, val);
@@ -1733,6 +1790,26 @@ static void nd_convert(CgTarget* t, ConvKind op, Operand dst, Operand src) {
"soft-float double conversion reached the backend un-lowered "
"(should be a runtime call)");
}
+ /* Coalesce a convert whose source is a dead, register-resident transient of
+ * the same register class: rename the source's register to the destination
+ * and convert it in place (dr == sr). Saves the source materialize-into-a-
+ * pinned-reg + a separate destination scratch, and lets a width-preserving
+ * convert (e.g. a zext/sext/bitcast that is a no-op once dst==src reg)
+ * collapse to nothing in the arch move/convert path. A class-crossing convert
+ * (GPR<->FPR fmov) is excluded by nd_rename_killed_to_dst's same-class guard
+ * and falls through below. */
+ {
+ Reg r = nd_rename_killed_to_dst(d, dst, src);
+ if (r != REG_NONE) {
+ NativeAllocClass cls = nd_class_for_type(d, dst.type);
+ sr = native_loc_reg(src.type, cls, r);
+ dr = native_loc_reg(dst.type, cls, r);
+ ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
+ d->native->convert(d->native, op, dr, sr);
+ nd_dst_writeback(d, dst, dr);
+ return;
+ }
+ }
sr = nd_materialize_operand(d, src);
dr = nd_dst_reg(d, dst);
ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
