commit 7dfd54eaffaad486e695d118a5bf49690f72c50e
parent 8d7927db63f2281b5292479137dfb30a9345e263
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sun, 19 Jul 2026 07:56:07 -0700
perf(opt): recover O1 native emit performance
Diffstat:
8 files changed, 343 insertions(+), 106 deletions(-)
diff --git a/src/opt/mir_clone.c b/src/opt/mir_clone.c
@@ -3,16 +3,14 @@
#include "opt/opt_internal.h"
/*
- * MIR-owned containers form an independently mutable graph. The arena owns
- * both graphs for the lifetime of Func, while the temporary destination Func
- * view continues to share immutable function-wide context. No container that
- * an optimizer pass may rewrite is allowed to point into HIR. Leaf descriptors
+ * MIR-owned containers form an independently mutable graph. During lowering,
+ * the temporary destination Func borrows HIR instruction arrays read-only;
+ * rewrite_func deep-copies each instruction into its final MIR array before
+ * mutating it. Other mutable containers are cloned here. Leaf descriptors
* explicitly left shared below (types, ABI descriptions, symbol/string storage,
* and constant byte payloads) are immutable compiler-owned data.
*/
-static u32 clone_cap(u32 count, u32 cap) { return cap < count ? count : cap; }
-
static void clone_abi_value(Arena* arena, CGABIValue* dst,
const CGABIValue* src) {
*dst = *src;
@@ -233,7 +231,7 @@ static void* clone_inst_aux(Arena* arena, const Inst* src) {
#undef CLONE_PLAIN_AUX
}
-static void clone_inst(Arena* arena, Inst* dst, const Inst* src) {
+void opt_mir_clone_inst(Arena* arena, Inst* dst, const Inst* src) {
*dst = *src;
if (src->defs && src->ndefs) {
dst->defs = arena_array(arena, Val, src->ndefs);
@@ -247,17 +245,7 @@ static void clone_inst(Arena* arena, Inst* dst, const Inst* src) {
dst->extra.aux = clone_inst_aux(arena, src);
}
-static Inst* cloned_inst_by_id(Func* f, InstId id) {
- if (id == INST_ID_NONE) return NULL;
- for (u32 b = 0; b < f->nblocks; ++b) {
- Block* block = &f->blocks[b];
- for (u32 i = 0; i < block->ninsts; ++i)
- if (block->insts[i].id == id) return &block->insts[i];
- }
- return NULL;
-}
-
-void opt_mir_clone_func(Func* dst, const Func* src) {
+void opt_mir_prepare_rewrite(Func* dst, const Func* src) {
Arena* arena;
u32 blocks_cap;
u32 frame_slots_cap;
@@ -269,40 +257,47 @@ void opt_mir_clone_func(Func* dst, const Func* src) {
dst->mir = NULL;
dst->opt_rewritten = 0;
- blocks_cap = clone_cap(src->nblocks, src->blocks_cap);
+ /* The source graph's spare capacity reflects its recording history, not a
+ * useful property of MIR. Keep only live structural entries. */
+ blocks_cap = src->nblocks;
dst->blocks = arena_zarray(arena, Block, blocks_cap ? blocks_cap : 1u);
+ dst->blocks_cap = blocks_cap;
for (u32 b = 0; b < src->nblocks; ++b) {
const Block* src_block = &src->blocks[b];
Block* dst_block = &dst->blocks[b];
- u32 inst_cap;
*dst_block = *src_block;
+ /* rewrite_func only reads this array and replaces it with independently
+ * owned output before the structural view can escape opt_lower_to_mir. */
+ dst_block->insts = src_block->insts;
+ dst_block->cap = src_block->ninsts;
- inst_cap = clone_cap(src_block->ninsts, src_block->cap);
- dst_block->insts =
- arena_zarray(arena, Inst, inst_cap ? inst_cap : 1u);
- for (u32 i = 0; i < src_block->ninsts; ++i)
- clone_inst(arena, &dst_block->insts[i], &src_block->insts[i]);
-
+ dst_block->preds = NULL;
if (src_block->preds && src_block->npreds) {
dst_block->preds = arena_array(arena, u32, src_block->npreds);
memcpy(dst_block->preds, src_block->preds,
sizeof(dst_block->preds[0]) * src_block->npreds);
}
- if (src_block->succ && src_block->succ_cap) {
- dst_block->succ = arena_array(arena, u32, src_block->succ_cap);
+ dst_block->succ = NULL;
+ dst_block->succ_cap = src_block->nsucc;
+ if (src_block->succ && src_block->nsucc) {
+ dst_block->succ = arena_array(arena, u32, src_block->nsucc);
memcpy(dst_block->succ, src_block->succ,
- sizeof(dst_block->succ[0]) * src_block->succ_cap);
+ sizeof(dst_block->succ[0]) * src_block->nsucc);
}
}
- frame_slots_cap = clone_cap(src->nframe_slots, src->frame_slots_cap);
+ frame_slots_cap = src->nframe_slots;
+ dst->frame_slots = NULL;
+ dst->frame_slots_cap = frame_slots_cap;
if (src->frame_slots && frame_slots_cap) {
dst->frame_slots = arena_zarray(arena, IRFrameSlot, frame_slots_cap);
memcpy(dst->frame_slots, src->frame_slots,
sizeof(dst->frame_slots[0]) * src->nframe_slots);
}
- emit_order_cap = clone_cap(src->emit_order_n, src->emit_order_cap);
+ emit_order_cap = src->emit_order_n;
+ dst->emit_order = NULL;
+ dst->emit_order_cap = emit_order_cap;
if (src->emit_order && emit_order_cap) {
dst->emit_order = arena_array(arena, u32, emit_order_cap);
memcpy(dst->emit_order, src->emit_order,
@@ -318,13 +313,12 @@ void opt_mir_clone_func(Func* dst, const Func* src) {
sizeof(dst->preg_info[0]) * nregs);
}
- if (src->scope_aux_inst && src->scopes_cap) {
- dst->scope_aux_inst = arena_zarray(arena, Inst*, src->scopes_cap);
- for (u32 i = 0; i < src->nscopes; ++i) {
- const Inst* scope = src->scope_aux_inst[i];
- if (scope) dst->scope_aux_inst[i] = cloned_inst_by_id(dst, scope->id);
- }
- }
+ /* Structured scopes are fully lowered before this boundary. Their lookup
+ * table is a semantic-recording cache of HIR Inst pointers, so MIR neither
+ * needs it nor should spend an O(scopes * instructions) scan rebuilding it. */
+ dst->scope_aux_inst = NULL;
+ dst->nscopes = 0;
+ dst->scopes_cap = 0;
/* Def-use entries contain direct Operand pointers into HIR. They are an
* analysis cache, not part of the MIR graph, and must never cross this
diff --git a/src/opt/opt_internal.h b/src/opt/opt_internal.h
@@ -178,9 +178,12 @@ void opt_walk_abivalue(Func*, Inst*, CGABIValue*, int storage_def,
OptOperandWalkFn, void*);
void opt_walk_inst_operands(Func*, Inst*, OptOperandWalkFn, void*);
-/* Build the independently mutable post-allocation graph used by MIR lowering.
- * Mutable instruction, operand, aux, and CFG storage never aliases HIR. */
-void opt_mir_clone_func(Func* dst, const Func* src);
+/* Build the structural view used while rewriting HIR into MIR. Block/CFG and
+ * function-wide mutable storage is independent, but instruction arrays remain
+ * read-only HIR borrows until rewrite_func replaces every one. */
+void opt_mir_prepare_rewrite(Func* dst, const Func* src);
+/* Deep-copy one instruction into MIR-owned storage. */
+void opt_mir_clone_inst(Arena* arena, Inst* dst, const Inst* src);
/* Present the independently owned MIR graph through the ordinary Func pass
* interface. The view deliberately drops HIR pointer-based analyses; callers
diff --git a/src/opt/pass_combine.c b/src/opt/pass_combine.c
@@ -365,25 +365,89 @@ static void ctx_reset(CombineCtx* ctx) {
ctx->block_change_p = 0;
}
-static void ctx_record_kills(CombineCtx* ctx, const OptHardRegSet* kills,
- i32 i) {
- for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
- u32 mask = kills->cls[c];
- for (Reg r = 0; r < OPT_MAX_HARD_REGS; ++r)
- if (mask & (1u << r)) ctx->last_def[c][r] = i;
- }
+static void ctx_record_hard_def(CombineCtx* ctx, u8 cls, Reg reg, i32 i) {
+ if (cls < OPT_REG_CLASSES && reg < OPT_MAX_HARD_REGS)
+ ctx->last_def[cls][reg] = i;
+}
+
+static void ctx_record_operand_def(CombineCtx* ctx, const Operand* op, i32 i) {
+ if (op && op->kind == OPK_REG)
+ ctx_record_hard_def(ctx, op->cls, op->v.reg, i);
+}
+
+static void ctx_record_abi_defs(CombineCtx* ctx, const CGABIValue* value,
+ i32 i) {
+ if (!value) return;
+ ctx_record_operand_def(ctx, &value->storage, i);
+ for (u32 p = 0; p < value->nparts; ++p)
+ ctx_record_operand_def(ctx, &value->parts[p].op, i);
+}
+
+static void ctx_record_mask(CombineCtx* ctx, u8 cls, u32 mask, i32 i) {
+ if (cls >= OPT_REG_CLASSES) return;
+ for (Reg r = 0; r < OPT_MAX_HARD_REGS; ++r)
+ if (mask & (1u << r)) ctx->last_def[cls][r] = i;
}
/* Record the canonical explicit definitions and implicit clobbers after one
* instruction. Calls and asm no longer erase every producer: only the ABI or
- * target-declared registers are invalidated. */
+ * target-declared registers are invalidated. This hot path deliberately
+ * records only kills: the full register-effects analysis also counts every use,
+ * work that the combine context would immediately discard. */
static void ctx_record(CombineCtx* ctx, const Inst* in, i32 i) {
- OptRegEffects effects;
- OptHardRegSet kills;
+ const u32* machine_clobbers = NULL;
if (inst_writes_memory(in)) ctx->last_mem_def = i;
- opt_inst_reg_effects(ctx->f, in, &effects);
- opt_reg_effect_kills(&effects, &kills);
- ctx_record_kills(ctx, &kills, i);
+ for (u32 op = 0; op < in->nopnds; ++op)
+ if (opt_inst_operand_is_def(in, op))
+ ctx_record_operand_def(ctx, &in->opnds[op], i);
+
+ switch ((IROp)in->op) {
+ case IR_CALL: {
+ const IRCallAux* aux = (const IRCallAux*)in->extra.aux;
+ if (aux && aux->use_plan_replay) {
+ for (u32 a = 0; a < aux->plan.nargs; ++a)
+ if (aux->plan.args[a].dst_kind == CG_CALL_PLAN_REG)
+ ctx_record_hard_def(ctx, aux->plan.args[a].cls,
+ aux->plan.args[a].dst_reg, i);
+ for (u32 r = 0; r < aux->plan.nrets; ++r) {
+ ctx_record_hard_def(ctx, aux->plan.rets[r].cls,
+ aux->plan.rets[r].src_reg, i);
+ ctx_record_operand_def(ctx, &aux->plan.rets[r].dst, i);
+ }
+ } else if (aux) {
+ ctx_record_abi_defs(ctx, &aux->desc.ret, i);
+ }
+ for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls)
+ ctx_record_mask(ctx, (u8)cls,
+ opt_call_clobber_mask_for(ctx->f, in, (u8)cls), i);
+ break;
+ }
+ case IR_ASM_BLOCK: {
+ const IRAsmAux* aux = (const IRAsmAux*)in->extra.aux;
+ if (!aux) break;
+ for (u32 out = 0; out < aux->nout; ++out)
+ ctx_record_operand_def(ctx, &aux->out_ops[out], i);
+ for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls)
+ ctx_record_mask(ctx, (u8)cls, aux->clobber_mask[cls], i);
+ break;
+ }
+ case IR_INTRINSIC: {
+ const IRIntrinAux* aux = (const IRIntrinAux*)in->extra.aux;
+ if (!aux) break;
+ for (u32 dst = 0; dst < aux->ndst; ++dst)
+ ctx_record_operand_def(ctx, &aux->dsts[dst], i);
+ break;
+ }
+ default:
+ break;
+ }
+
+ if (ctx->f->inst_clobbers && in->id != INST_ID_NONE &&
+ in->id < ctx->f->inst_clobbers_cap)
+ machine_clobbers = ctx->f->inst_clobbers[in->id];
+ if (machine_clobbers)
+ for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls)
+ ctx_record_mask(ctx, (u8)cls, machine_clobbers[cls], i);
}
/* Lookup the producer of (cls, reg) in this BB, if any. Returns -1 if no
diff --git a/src/opt/pass_lower.c b/src/opt/pass_lower.c
@@ -1600,7 +1600,8 @@ static void rewrite_func(Func* f, const OptLiveInfo* live_info) {
for (u32 ri = bl->ninsts; ri > 0; --ri) {
u32 i = ri - 1u;
- Inst in = bl->insts[i];
+ Inst in;
+ opt_mir_clone_inst(f->arena, &in, &bl->insts[i]);
if ((IROp)in.op == IR_PARAM_DECL) {
out_prepend_inst(f, &out, &in);
continue;
@@ -1648,7 +1649,7 @@ static void rewrite_func(Func* f, const OptLiveInfo* live_info) {
void opt_lower_to_mir(Func* f, const OptLiveInfo* live_info) {
if (!f) return;
Func phys;
- opt_mir_clone_func(&phys, f);
+ opt_mir_prepare_rewrite(&phys, f);
rewrite_func(&phys, live_info);
diff --git a/src/opt/pass_mir.c b/src/opt/pass_mir.c
@@ -45,6 +45,8 @@ void opt_mir_commit(Func* f, const Func* view) {
f->next_inst_id = view->next_inst_id;
}
+#ifndef NDEBUG
+
typedef struct MirVerifyCtx {
const char* stage;
/* opt_walk_operand presents an indirect owner immediately followed by
@@ -507,7 +509,14 @@ static void mir_verify_operand(Func* f, Inst* in, Operand* op, int is_def,
mir_verify_scalar_location(f, in, op, is_def, ctx);
}
+#endif /* !NDEBUG */
+
void opt_mir_verify(Func* f, const char* stage) {
+#ifdef NDEBUG
+ (void)f;
+ (void)stage;
+ return;
+#else
Func v;
if (!opt_mir_view(f, &v)) return;
mir_verify_frame_table(&v, stage);
@@ -544,6 +553,7 @@ void opt_mir_verify(Func* f, const char* stage) {
ctx.indirect_part, in->id);
}
}
+#endif
}
void opt_mir_combine(Func* f, NativeTarget* target) {
diff --git a/src/opt/pass_native_emit.c b/src/opt/pass_native_emit.c
@@ -45,6 +45,7 @@ typedef struct NativeEmitCtx {
* bank is exhausted. */
OptHardRegSet* live_after_by_inst;
u32 live_after_cap;
+ u8 live_after_ready;
/* Clean frame-value forwarding cache, indexed by physical register. A
* nonzero slot means the register holds the exact value already stored in
* that private FS_SPILL slot. Memory is always authoritative. */
@@ -71,6 +72,7 @@ static NativeLoc frame_cache_take_slot(NativeEmitCtx* e, FrameSlot slot,
NativeAllocClass cls,
KitCgTypeId type, Reg avoid_a,
Reg avoid_b);
+static void compute_emit_live_after(NativeEmitCtx* e);
static _Noreturn void emit_panic(NativeEmitCtx* e, SrcLoc loc,
const char* msg) {
@@ -267,7 +269,9 @@ struct NativeEmitTempScope {
NativeEmitCtx* emit;
const Inst* inst;
u8 allow_asm_temps;
- u8 pad[3];
+ u8 effects_ready;
+ u8 live_after_applied;
+ u8 pad;
u32 leased[OPT_REG_CLASSES];
u32 unavailable[OPT_REG_CLASSES];
/* Definition registers that currently hold a cached stack source consumed
@@ -327,25 +331,16 @@ static int temp_loc_equal(NativeLoc a, NativeLoc b) {
return 0;
}
-static void temp_scope_begin(NativeEmitCtx* e, NativeEmitTempScope* scope,
- const Inst* in) {
+static void temp_scope_apply_effects(NativeEmitCtx* e,
+ NativeEmitTempScope* scope) {
OptRegEffects effects;
FrameCacheNeedCtx needed;
- u32 live_after[OPT_REG_CLASSES];
u32 invalidate[OPT_REG_CLASSES];
- memset(scope, 0, sizeof *scope);
+ if (!e || !scope || scope->effects_ready) return;
memset(&needed, 0, sizeof needed);
- memset(live_after, 0, sizeof live_after);
- scope->emit = e;
- scope->inst = in;
- if (in && in->id != INST_ID_NONE && in->id < e->live_after_cap)
- for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
- live_after[c] = e->live_after_by_inst[in->id].cls[c];
- scope->unavailable[c] = live_after[c];
- }
- opt_inst_reg_effects(e->f, in, &effects);
+ opt_inst_reg_effects(e->f, scope->inst, &effects);
needed.emit = e;
- frame_cache_collect_needed(in, &needed);
+ frame_cache_collect_needed(scope->inst, &needed);
for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
u32 clobbers = effects.clobbers.cls[c];
/* Call clobbers happen after argument/callee staging. A call-local temp is
@@ -354,7 +349,7 @@ static void temp_scope_begin(NativeEmitCtx* e, NativeEmitTempScope* scope,
* makes high-pressure calls impossible precisely when every callee-saved
* allocation register is live across them. Calls clear the block cache
* below, so this exception cannot preserve a stale value past the call. */
- if (in && (IROp)in->op == IR_CALL) clobbers = 0u;
+ if (scope->inst && (IROp)scope->inst->op == IR_CALL) clobbers = 0u;
scope->unavailable[c] |=
effects.uses.cls[c] | effects.defs.cls[c] | clobbers;
scope->cache_def_reuse[c] =
@@ -362,11 +357,83 @@ static void temp_scope_begin(NativeEmitCtx* e, NativeEmitTempScope* scope,
~(effects.uses.cls[c] | effects.clobbers.cls[c]);
invalidate[c] = scope->unavailable[c] & ~scope->cache_def_reuse[c];
}
- if (in && ((IROp)in->op == IR_CALL || (IROp)in->op == IR_ASM_BLOCK))
+ if (scope->inst && ((IROp)scope->inst->op == IR_CALL ||
+ (IROp)scope->inst->op == IR_ASM_BLOCK))
frame_cache_clear(e);
else
frame_cache_invalidate_masks(e, invalidate);
+ scope->effects_ready = 1u;
+}
+
+static void temp_scope_begin(NativeEmitCtx* e, NativeEmitTempScope* scope,
+ const Inst* in) {
+ const u32* machine_clobbers = NULL;
+ for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
+ scope->leased[c] = 0u;
+ scope->unavailable[c] = 0u;
+ scope->cache_def_reuse[c] = 0u;
+ }
+ scope->emit = e;
+ scope->inst = in;
+ scope->allow_asm_temps = 0u;
+ scope->effects_ready = 0u;
+ scope->live_after_applied = 0u;
+ /* ncache is the sole validity boundary for the materialization entries.
+ * Clearing the 16-entry payload on every MIR instruction only burns memory
+ * bandwidth; each entry is fully assigned before ncache exposes it. */
+ scope->ncache = 0u;
e->temps = scope;
+
+ if (e->live_after_ready) {
+ if (in && in->id != INST_ID_NONE && in->id < e->live_after_cap)
+ for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
+ scope->unavailable[c] |= e->live_after_by_inst[in->id].cls[c];
+ scope->live_after_applied = 1u;
+ temp_scope_apply_effects(e, scope);
+ return;
+ }
+
+ /* Before the allocation-register scavenger is enabled, MIR's ownership
+ * invariant keeps every explicit use/def out of the reserved emission-temp
+ * bank. Only an exceptional backend-declared machine clobber can collide
+ * with those temps, so handle that table directly and defer the full
+ * use/def analysis alongside hard liveness. */
+ if (in && in->id != INST_ID_NONE && e->f->inst_clobbers &&
+ in->id < e->f->inst_clobbers_cap)
+ machine_clobbers = e->f->inst_clobbers[in->id];
+ if (in && ((IROp)in->op == IR_CALL || (IROp)in->op == IR_ASM_BLOCK)) {
+ frame_cache_clear(e);
+ } else if (machine_clobbers) {
+ for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
+ scope->unavailable[c] |= machine_clobbers[c];
+ frame_cache_invalidate_masks(e, machine_clobbers);
+ }
+}
+
+/* Physical live-after is needed only for the allocation-register scavenger.
+ * Most instructions (and most functions) fit entirely in the target's
+ * dedicated emission-temp bank, so defer the full hard-liveness dataflow and
+ * per-instruction table until a scope actually exhausts that bank. */
+static void temp_scope_require_live_after(NativeEmitCtx* e) {
+ u32 live[OPT_REG_CLASSES];
+ const Inst* in;
+ if (!e) return;
+ if (!e->live_after_ready) compute_emit_live_after(e);
+ if (!e->temps) return;
+ if (e->temps->live_after_applied) return;
+ in = e->temps->inst;
+ memset(live, 0, sizeof live);
+ if (in && in->id != INST_ID_NONE && in->id < e->live_after_cap)
+ for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
+ live[c] = e->live_after_by_inst[in->id].cls[c];
+ e->temps->unavailable[c] |= live[c];
+ }
+ e->temps->live_after_applied = 1u;
+ temp_scope_apply_effects(e, e->temps);
+ for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
+ live[c] &= ~e->temps->cache_def_reuse[c];
+ }
+ frame_cache_invalidate_masks(e, live);
}
static NativeEmitTempMark temp_scope_mark(NativeEmitCtx* e) {
@@ -493,11 +560,13 @@ static Reg temp_acquire(NativeEmitCtx* e, NativeAllocClass cls, Reg a, Reg b,
* we overwrite it. This is the pressure escape hatch for two-temp targets
* and for ARM32, whose backend-internal LR must not be leased. */
if (e->temps) {
+ temp_scope_require_live_after(e);
for (u32 i = 0; i < e->f->opt_hard_reg_count[c]; ++i) {
Reg r = e->f->opt_hard_regs[c][i];
if (r >= 32u || !(e->f->opt_caller_saved[c] & (1u << r))) continue;
if (!temp_reg_available(e, cls, r, a, b)) continue;
e->temps->leased[c] |= 1u << r;
+ metrics_count(e->c, "opt.native_emit.scavenges", 1);
return r;
}
}
@@ -583,6 +652,11 @@ static u32 frame_cache_retainable_mask(NativeEmitCtx* e,
u32 preserved_allocable;
if (!ci || c >= OPT_REG_CLASSES) return 0u;
mask = ci->emit_cache_mask;
+ /* Without hard live-after data, only target-reserved cache registers can be
+ * retained across instructions. An ordinary allocation register reaching a
+ * spill store may still carry a live MIR value; treating it as a cache entry
+ * would let a later reload lease and overwrite that value. */
+ if (!e->live_after_ready) return mask;
preserved_allocable = native_target_caller_saved_mask(e->target, cls);
preserved_allocable |= e->callee_saved_used[c];
/* An allocable register is a legal persistent cache only when acquisition
@@ -608,6 +682,7 @@ static Reg temp_acquire_asm(NativeEmitCtx* e, NativeAllocClass cls, Reg fixed,
u32 c = (u32)cls;
if (!e->temps || c >= OPT_REG_CLASSES)
emit_panic(e, loc, "asm register staging outside temporary scope");
+ temp_scope_require_live_after(e);
if (fixed != (Reg)REG_NONE) {
if (!asm_temp_reg_ok(e, cls, fixed, allowed_mask) ||
!temp_reg_available(e, cls, fixed, REG_NONE, REG_NONE))
@@ -679,6 +754,7 @@ static int temp_available(NativeEmitCtx* e, NativeAllocClass cls, Reg a,
if (temp_reg_available(e, cls, r, a, b)) return 1;
}
if (e->temps) {
+ temp_scope_require_live_after(e);
for (u32 i = 0; i < e->f->opt_hard_reg_count[c]; ++i) {
Reg r = e->f->opt_hard_regs[c][i];
if (r < 32u && (e->f->opt_caller_saved[c] & (1u << r)) &&
@@ -1196,7 +1272,6 @@ static NativeLoc materialize_operand(NativeEmitCtx* e, const OptOperand* op,
static void write_loc(NativeEmitCtx* e, NativeLoc dst, NativeLoc src,
MemAccess mem, SrcLoc loc) {
NativeAddr addr;
- NativeLoc tmp;
NativeEmitTempMark mark = temp_scope_mark(e);
if (dst.kind == NATIVE_LOC_NONE) goto done;
if (loc_same_frame(dst, src)) goto done;
@@ -1213,11 +1288,23 @@ static void write_loc(NativeEmitCtx* e, NativeLoc dst, NativeLoc src,
e->target->load_imm(e->target, dst, src.v.imm);
goto done;
}
- tmp = materialize(e, src, (NativeAllocClass)dst.cls, dst.type, dst.v.reg,
- REG_NONE, loc);
- if (tmp.v.reg != dst.v.reg || tmp.cls != dst.cls)
- e->target->move(e->target, dst, tmp);
- goto done;
+ /* A register destination is itself the best materialization register.
+ * Loading through an emitter temp first adds a move to every spill reload
+ * and ABI/frame copy. Native loads consume their address before defining
+ * the destination, so even an address based on dst is safe here. */
+ if (src.kind == NATIVE_LOC_GLOBAL ||
+ src.kind == NATIVE_LOC_FRAME_ADDR) {
+ addr = addr_from_loc(e, src, loc);
+ e->target->load_addr(e->target, dst, addr);
+ goto done;
+ }
+ if (src.kind == NATIVE_LOC_FRAME || src.kind == NATIVE_LOC_STACK ||
+ src.kind == NATIVE_LOC_ADDR) {
+ addr = addr_from_loc(e, src, loc);
+ e->target->load(e->target, dst, addr, mem);
+ goto done;
+ }
+ emit_panic(e, loc, "cannot write location to register");
}
addr = addr_from_loc(e, dst, loc);
if (src.kind != NATIVE_LOC_REG)
@@ -1247,9 +1334,19 @@ static void write_operand(NativeEmitCtx* e, const OptOperand* dst,
* spill home are common producers and otherwise lose their forwarding fact
* at this abstraction boundary. */
if (dst && dst->kind == OPK_STACK && src.kind != NATIVE_LOC_REG &&
- !loc_same_frame(real_dst, src))
- src = materialize(e, src, (NativeAllocClass)real_dst.cls, real_dst.type,
- REG_NONE, REG_NONE, loc);
+ !loc_same_frame(real_dst, src)) {
+ /* Keep integer zero in the target's architectural zero register when it
+ * can be stored directly. Materializing it into an emitter temp costs an
+ * otherwise redundant movz/li on every zero-initialized spill. */
+ if (src.kind == NATIVE_LOC_IMM && src.v.imm == 0 &&
+ e->target->has_store_zero_reg &&
+ class_for_type(e, real_dst.type) == NATIVE_REG_INT)
+ src = loc_reg(real_dst.type, NATIVE_REG_INT,
+ e->target->store_zero_reg);
+ else
+ src = materialize(e, src, (NativeAllocClass)real_dst.cls, real_dst.type,
+ REG_NONE, REG_NONE, loc);
+ }
write_loc(e, real_dst, src, mem, loc);
frame_cache_note_store(e, dst, src, mem);
}
@@ -2441,6 +2538,7 @@ static u32 compute_callee_saved_used(NativeEmitCtx* e, u32* used, u32 cap) {
static void compute_emit_live_after(NativeEmitCtx* e) {
OptHardBlockLive* blocks = opt_maybe_build_hard_live(e->f);
+ e->live_after_ready = 1u;
e->live_after_cap = e->f->next_inst_id;
if (!e->live_after_cap) return;
e->live_after_by_inst =
@@ -2791,7 +2889,6 @@ void opt_emit_native(Compiler* c, Func* f, NativeTarget* target) {
arena_zarray(e.f->arena, u8, e.f->nblocks ? e.f->nblocks : 1u);
for (u32 i = 0; i < e.f->nblocks; ++i) e.labels[i] = MC_LABEL_NONE;
fd = semantic_func_desc(&e);
- compute_emit_live_after(&e);
plan_asm_staging(&e);
metrics_scope_end(c, "opt.native_emit.setup");
diff --git a/test/opt/location_mir_test.c b/test/opt/location_mir_test.c
@@ -113,9 +113,11 @@ static void run_machinize(void* arg) {
opt_machinize_native(call->f, call->target);
}
+#ifndef NDEBUG
static void run_mir_verify(void* arg) {
opt_mir_verify((Func*)arg, "negative-temp-escape");
}
+#endif
static void machinize_rejects_cross_owner_registers(KitUnit* u) {
KitCompiler* kc = NULL;
@@ -188,6 +190,9 @@ static void machinize_rejects_cross_owner_registers(KitUnit* u) {
}
static void mir_verify_rejects_private_temp_escape(KitUnit* u) {
+#ifdef NDEBUG
+ (void)u;
+#else
KitCompiler* kc = NULL;
Compiler* c;
KitCgTypeId i64_type;
@@ -252,6 +257,7 @@ static void mir_verify_rejects_private_temp_escape(KitUnit* u) {
}
kit_compiler_free(kc);
+#endif
}
static void mir_commit_propagates_instruction_namespace(KitUnit* u) {
@@ -432,11 +438,16 @@ static void spilled_values_remain_locations(KitUnit* u) {
CU_EXPECT(u, f->blocks == hir_blocks && f->blocks != f->mir->blocks,
"lowering did not preserve a distinct HIR block graph");
CU_EXPECT(u,
- f->mir->blocks_cap == f->blocks_cap &&
+ f->mir->blocks_cap == f->mir->nblocks &&
+ f->mir->emit_order_cap == f->mir->emit_order_n &&
+ f->mir->blocks[entry].cap ==
+ f->mir->blocks[entry].ninsts &&
+ f->mir->blocks[entry].succ_cap ==
+ f->mir->blocks[entry].nsucc &&
opt_mir_view(f, &view) && view.blocks == f->mir->blocks &&
view.blocks_cap == f->mir->blocks_cap && view.mir == NULL &&
view.scope_aux_inst == NULL && view.nscopes == 0,
- "canonical MIR view retained stale HIR graph metadata");
+ "MIR retained recording slack or stale HIR graph metadata");
CU_EXPECT(u,
f->blocks[entry].insts == hir_insts &&
f->mir->blocks[entry].insts != hir_insts,
diff --git a/test/opt/native_emit_frame_dst_test.c b/test/opt/native_emit_frame_dst_test.c
@@ -22,7 +22,9 @@ typedef struct MockNative {
NativeCallPlanRet ret_parts[2];
u32 callback_mask;
u32 writeback_slots;
+ u32 load_imms;
u32 loads;
+ u32 moves;
u32 load_addrs;
u32 frame_loads;
u32 reg_address_loads;
@@ -104,6 +106,7 @@ static void mock_load_imm(NativeTarget *target, NativeLoc dst, i64 imm) {
MockNative *mock = mock_of(target);
expect_reg(mock, dst, "load_imm destination");
if (dst.v.reg < 32u) mock->reg_values[dst.v.reg] = (u32)imm;
+ ++mock->load_imms;
}
static void mock_move(NativeTarget *target, NativeLoc dst, NativeLoc src) {
@@ -112,6 +115,7 @@ static void mock_move(NativeTarget *target, NativeLoc dst, NativeLoc src) {
expect_reg(mock, src, "move source");
if (dst.v.reg < 32u && src.v.reg < 32u)
mock->reg_values[dst.v.reg] = mock->reg_values[src.v.reg];
+ ++mock->moves;
}
static void mock_load_const(NativeTarget *target, NativeLoc dst,
@@ -1378,7 +1382,7 @@ static void atomic_store_collapses_address_for_value_temp(KitUnit *unit) {
kit_compiler_free(kit_c);
}
-static void allocator_cache_obeys_machine_clobbers(KitUnit *unit) {
+static void allocator_cache_waits_for_hard_liveness(KitUnit *unit) {
KitCompiler *kit_c = NULL;
Compiler *c;
KitCgTypeId i64_type, ptr_type;
@@ -1426,9 +1430,9 @@ static void allocator_cache_obeys_machine_clobbers(KitUnit *unit) {
preserved_slot = add_slot(f, i64_type, 8u, 8u);
record_slot = add_slot(f, i64_type, 8u, 8u);
- /* r10 is dead after this writeback. Its clean copy is therefore eligible
- * for allocator-register forwarding until a declared machine effect says
- * the backend encoding destroys r10. */
+ /* Even though r10 is dead after this writeback, lazy native emission has not
+ * needed hard-liveness yet. It must leave the authoritative frame copy alone
+ * rather than retain and later lease an ordinary allocation register. */
in = emit_with_ops(f, block, IR_COPY, 2u);
in->opnds[0] = stack_op(clobbered_slot, i64_type);
in->opnds[1] = reg_op(10u, i64_type);
@@ -1451,9 +1455,9 @@ static void allocator_cache_obeys_machine_clobbers(KitUnit *unit) {
in->opnds[0] = reg_op(12u, i64_type);
in->opnds[1] = stack_op(clobbered_slot, i64_type);
- /* Establish a new, independently dead r10 value for the preservation half
- * of the contract. The label hook reports no exceptional clobber, so the
- * following read should forward without touching the frame. */
+ /* Establish a new, independently dead r10 value. The ordinary label hook
+ * preserves it, but without hard-liveness the emitter must still reload the
+ * frame rather than turn r10 into a persistent cache register. */
in = emit_with_ops(f, block, IR_LOAD_IMM, 1u);
in->opnds[0] = reg_op(10u, i64_type);
in->extra.imm = 99;
@@ -1472,10 +1476,9 @@ static void allocator_cache_obeys_machine_clobbers(KitUnit *unit) {
mock_init(&mock, unit, c);
mock.report_bitfield_store_clobber = 1u;
- /* The O0 direct-target value bank is deliberately empty. Cross-instruction
- * O1 forwarding must derive allocator-register eligibility from the
- * machinized `f->opt_hard_regs` table above, never from an unrelated O0
- * cache policy. */
+ /* The O0 direct-target value bank is deliberately empty. In particular, it
+ * must not make the machinized allocation registers cache-eligible before
+ * the lazy hard-liveness table has actually been built. */
mock.classes[NATIVE_REG_INT].caller_saved_mask = 1u << 10u;
memset(&mop, 0, sizeof mop);
@@ -1502,9 +1505,9 @@ static void allocator_cache_obeys_machine_clobbers(KitUnit *unit) {
(SAW_BITFIELD_STORE | SAW_LABEL_ADDR)) ==
(SAW_BITFIELD_STORE | SAW_LABEL_ADDR),
"clobbering and ordinary preservation hooks must both execute");
- CU_EXPECT(unit, mock.frame_loads == 1u && mock.loads == 1u,
- "declared clobber must force one authoritative reload while the "
- "ordinary hook preserves forwarding (loads=%u frame-loads=%u)",
+ CU_EXPECT(unit, mock.frame_loads == 2u && mock.loads == 2u,
+ "allocation registers must not be retained before lazy "
+ "hard-liveness is available (loads=%u frame-loads=%u)",
mock.loads, mock.frame_loads);
kit_compiler_free(kit_c);
@@ -1640,13 +1643,66 @@ static void return_parts_keep_prior_abi_destinations_live(KitUnit *unit) {
"(loads=%u)",
mock.nframe_load_dsts);
if (mock.nframe_load_dsts == 2u) {
- CU_EXPECT(unit, mock.frame_load_dsts[0] != 8u,
- "first return-part load must not overwrite its r8 destination");
- CU_EXPECT(unit, mock.frame_load_dsts[1] != 8u,
- "second return-part load reused completed ABI destination r8");
- CU_EXPECT(unit, mock.frame_load_dsts[1] != 9u,
- "second return-part load must not overwrite its r9 destination");
+ CU_EXPECT(unit, mock.frame_load_dsts[0] == 8u,
+ "first return part should load directly into r8");
+ CU_EXPECT(unit, mock.frame_load_dsts[1] == 9u,
+ "second return part should load directly into r9");
}
+ CU_EXPECT(unit, mock.moves == 0u,
+ "frame return parts should not route through emitter temps");
+
+ kit_compiler_free(kit_c);
+}
+
+static void zero_spill_uses_architectural_zero_register(KitUnit *unit) {
+ KitCompiler *kit_c = NULL;
+ Compiler *c;
+ KitCgTypeId i64_type;
+ OptCGFuncDesc desc;
+ Func *f;
+ MockNative mock;
+ FrameSlot value_slot;
+ Inst *load;
+ u32 block;
+
+ CU_EXPECT(unit,
+ kit_unit_compiler_new(
+ unit,
+ kit_unit_target(KIT_ARCH_ARM_64, KIT_OS_LINUX, KIT_OBJ_ELF),
+ &kit_c) == KIT_OK &&
+ kit_c != NULL,
+ "compiler allocation failed for zero-spill test");
+ if (!kit_c)
+ return;
+ c = (Compiler *)kit_c;
+ i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64);
+
+ memset(&desc, 0, sizeof desc);
+ desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID);
+ f = ir_func_new(c, &desc);
+ block = ir_block_new(f);
+ f->entry = block;
+ ir_note_emit(f, block);
+ f->opt_rewritten = 1;
+ f->emit_temp_regs[RC_INT][0] = 8u;
+ f->emit_temp_reg_count[RC_INT] = 1u;
+ value_slot = add_slot(f, i64_type, 8u, 8u);
+
+ load = emit_with_ops(f, block, IR_LOAD_IMM, 1u);
+ load->opnds[0] = stack_op(value_slot, i64_type);
+ load->extra.imm = 0;
+
+ mock_init(&mock, unit, c);
+ mock.base.has_store_zero_reg = 1u;
+ mock.base.store_zero_reg = 31u;
+ opt_emit_native(c, f, &mock.base);
+
+ CU_EXPECT(unit, mock.load_imms == 0u,
+ "zero spill should not materialize an emitter temporary");
+ CU_EXPECT(unit, mock.moves == 0u,
+ "zero spill should not move through an emitter temporary");
+ CU_EXPECT(unit, (mock.writeback_slots & (1u << value_slot)) != 0u,
+ "zero spill was not stored to its authoritative frame home");
kit_compiler_free(kit_c);
}
@@ -1737,9 +1793,10 @@ int main(void) {
in_place_collapse_drops_overwritten_cache_fact(&unit);
frame_atomic_load_reuses_scoped_address_base(&unit);
atomic_store_collapses_address_for_value_temp(&unit);
- allocator_cache_obeys_machine_clobbers(&unit);
+ allocator_cache_waits_for_hard_liveness(&unit);
call_ends_pre_call_temp_phase(&unit);
return_parts_keep_prior_abi_destinations_live(&unit);
+ zero_spill_uses_architectural_zero_register(&unit);
call_setup_owns_callee_and_completed_arguments(&unit);
if (unit.fails) {
fprintf(stderr, "native-emit-frame-dst: %d/%d failed\n", unit.fails,
