commit bb82ba8985a793406af848a2ba1b3171a509a525
parent 7dfd54eaffaad486e695d118a5bf49690f72c50e
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sun, 19 Jul 2026 09:05:07 -0700
perf(opt): canonicalize register effects and emit planning
Diffstat:
14 files changed, 603 insertions(+), 319 deletions(-)
diff --git a/include/kit/profile.h b/include/kit/profile.h
@@ -195,6 +195,9 @@ typedef enum KitProfileCounter {
KIT_PROFILE_COUNTER_JIT_NSEGMENTS,
KIT_PROFILE_COUNTER_JIT_SEGMENT_BYTES,
+ /* Appended to preserve the numeric ids of existing public counters. */
+ KIT_PROFILE_COUNTER_OPT_NATIVE_EMIT_SCAVENGES,
+
/* Kit-owned counter ids are [1, KIT_PROFILE_COUNTER_KIT_LAST]. */
KIT_PROFILE_COUNTER_KIT_LAST = 0x3fff,
diff --git a/src/core/metrics.h b/src/core/metrics.h
@@ -205,6 +205,8 @@ static inline KitProfileCounter metrics_counter_from_name(const char* name) {
KIT_PROFILE_COUNTER_OPT_TINY_INLINE_REFUSE_REWRITE);
METRICS_COUNTER("opt.tiny_inline.inlined",
KIT_PROFILE_COUNTER_OPT_TINY_INLINE_INLINED);
+ METRICS_COUNTER("opt.native_emit.scavenges",
+ KIT_PROFILE_COUNTER_OPT_NATIVE_EMIT_SCAVENGES);
METRICS_COUNTER("link.inputs", KIT_PROFILE_COUNTER_LINK_INPUTS);
METRICS_COUNTER("link.sections", KIT_PROFILE_COUNTER_LINK_SECTIONS);
METRICS_COUNTER("link.segments", KIT_PROFILE_COUNTER_LINK_SEGMENTS);
diff --git a/src/core/profile.c b/src/core/profile.c
@@ -293,6 +293,8 @@ const char* kit_profile_counter_name(KitProfileCounter counter) {
return "jit.nsegments";
case KIT_PROFILE_COUNTER_JIT_SEGMENT_BYTES:
return "jit.segment_bytes";
+ case KIT_PROFILE_COUNTER_OPT_NATIVE_EMIT_SCAVENGES:
+ return "opt.native_emit.scavenges";
default:
return 0;
}
diff --git a/src/opt/opt_internal.h b/src/opt/opt_internal.h
@@ -9,6 +9,15 @@ typedef struct OptHardRegSet {
u32 cls[OPT_REG_CLASSES];
} OptHardRegSet;
+/* Mask-only physical-register effects for consumers that do not need use
+ * multiplicity. Keep this separate from OptRegEffects so hot liveness and
+ * native-emission paths never clear or update the detailed 3x32 count table. */
+typedef struct OptRegEffectMasks {
+ OptHardRegSet uses;
+ OptHardRegSet defs;
+ OptHardRegSet clobbers;
+} OptRegEffectMasks;
+
/* Canonical physical-register effects of one instruction. `uses` and `defs`
* are the registers named by the instruction/ABI operands; `clobbers` are
* implicit kills (call ABI, inline asm, and target machine constraints).
@@ -16,9 +25,14 @@ typedef struct OptHardRegSet {
* Keeping defs separate from clobbers matters to DCE: an implicit kill is not
* a produced value. */
typedef struct OptRegEffects {
- OptHardRegSet uses;
- OptHardRegSet defs;
- OptHardRegSet clobbers;
+ union {
+ OptRegEffectMasks masks;
+ struct {
+ OptHardRegSet uses;
+ OptHardRegSet defs;
+ OptHardRegSet clobbers;
+ };
+ };
/* Per-register explicit-use multiplicity, saturated at two. This keeps
* combine's single-use decisions in the same canonical operand model (and
* preserves the double-reference case `[r + r*scale]`). */
@@ -200,6 +214,7 @@ Inst* opt_block_insert_at(Func*, Block* bl, u32 at, u32 k);
int opt_mem_observable(const MemAccess*);
u32 opt_call_clobber_mask_for(Func*, const Inst*, u8 cls);
+const u32* opt_inst_machine_clobber_masks(Func*, const Inst*);
int opt_inst_has_side_effect(Func*, const Inst*);
@@ -218,6 +233,9 @@ void opt_hard_live_step(OptHardRegSet* live, const OptHardRegSet* use,
const OptHardRegSet* def);
void opt_inst_reg_effects(Func*, const Inst*, OptRegEffects*);
void opt_reg_effect_kills(const OptRegEffects*, OptHardRegSet*);
+void opt_inst_reg_effect_masks(Func*, const Inst*, OptRegEffectMasks*);
+void opt_reg_effect_mask_kills(const OptRegEffectMasks*, OptHardRegSet*);
+void opt_inst_reg_kills(Func*, const Inst*, OptHardRegSet*);
OptHardBlockLive* opt_maybe_build_hard_live(Func*);
OptHardRegSet opt_hard_live_out_for_block(const OptHardBlockLive*);
int opt_block_live_out_has_phys_reg(Func*, const OptHardBlockLive*, u32 block,
diff --git a/src/opt/pass_combine.c b/src/opt/pass_combine.c
@@ -130,13 +130,26 @@ static int operand_uses_phys_reg(const Operand* op, const Operand* r) {
return 0;
}
-static int inst_uses_phys_reg(Func* f, const Inst* in, const Operand* reg) {
+typedef struct CombinePhysRegEffect {
+ u8 uses;
+ u8 defines;
+ u8 clobbers;
+} CombinePhysRegEffect;
+
+static CombinePhysRegEffect inst_phys_reg_effect(Func* f, const Inst* in,
+ const Operand* reg) {
OptRegEffects effects;
+ CombinePhysRegEffect result = {0};
if (!reg || reg->kind != OPK_REG || reg->cls >= OPT_REG_CLASSES ||
reg->v.reg >= OPT_MAX_HARD_REGS)
- return 0;
+ return result;
opt_inst_reg_effects(f, in, &effects);
- return effects.use_count[reg->cls][reg->v.reg];
+ result.uses = effects.use_count[reg->cls][reg->v.reg];
+ result.defines =
+ (u8)((effects.defs.cls[reg->cls] & (1u << reg->v.reg)) != 0u);
+ result.clobbers =
+ (u8)((effects.clobbers.cls[reg->cls] & (1u << reg->v.reg)) != 0u);
+ return result;
}
static int hard_set_has_reg(const OptHardRegSet* set, const Operand* reg) {
@@ -146,24 +159,9 @@ static int hard_set_has_reg(const OptHardRegSet* set, const Operand* reg) {
return (set->cls[reg->cls] & (1u << reg->v.reg)) != 0;
}
-static int inst_defines_phys_reg(Func* f, const Inst* in, const Operand* reg) {
- OptRegEffects effects;
- opt_inst_reg_effects(f, in, &effects);
- return hard_set_has_reg(&effects.defs, reg);
-}
-
-static int inst_clobbers_phys_reg(Func* f, const Inst* in,
- const Operand* reg) {
- OptRegEffects effects;
- opt_inst_reg_effects(f, in, &effects);
- return hard_set_has_reg(&effects.clobbers, reg);
-}
-
static int inst_kills_phys_reg(Func* f, const Inst* in, const Operand* reg) {
- OptRegEffects effects;
OptHardRegSet kills;
- opt_inst_reg_effects(f, in, &effects);
- opt_reg_effect_kills(&effects, &kills);
+ opt_inst_reg_kills(f, in, &kills);
return hard_set_has_reg(&kills, reg);
}
@@ -365,89 +363,24 @@ static void ctx_reset(CombineCtx* ctx) {
ctx->block_change_p = 0;
}
-static void ctx_record_hard_def(CombineCtx* ctx, u8 cls, Reg reg, i32 i) {
- if (cls < OPT_REG_CLASSES && reg < OPT_MAX_HARD_REGS)
- ctx->last_def[cls][reg] = i;
-}
-
-static void ctx_record_operand_def(CombineCtx* ctx, const Operand* op, i32 i) {
- if (op && op->kind == OPK_REG)
- ctx_record_hard_def(ctx, op->cls, op->v.reg, i);
-}
-
-static void ctx_record_abi_defs(CombineCtx* ctx, const CGABIValue* value,
- i32 i) {
- if (!value) return;
- ctx_record_operand_def(ctx, &value->storage, i);
- for (u32 p = 0; p < value->nparts; ++p)
- ctx_record_operand_def(ctx, &value->parts[p].op, i);
-}
-
static void ctx_record_mask(CombineCtx* ctx, u8 cls, u32 mask, i32 i) {
if (cls >= OPT_REG_CLASSES) return;
- for (Reg r = 0; r < OPT_MAX_HARD_REGS; ++r)
- if (mask & (1u << r)) ctx->last_def[cls][r] = i;
+ while (mask) {
+ Reg r = (Reg)__builtin_ctz(mask);
+ ctx->last_def[cls][r] = i;
+ mask &= mask - 1u;
+ }
}
/* Record the canonical explicit definitions and implicit clobbers after one
* instruction. Calls and asm no longer erase every producer: only the ABI or
- * target-declared registers are invalidated. This hot path deliberately
- * records only kills: the full register-effects analysis also counts every use,
- * work that the combine context would immediately discard. */
-static void ctx_record(CombineCtx* ctx, const Inst* in, i32 i) {
- const u32* machine_clobbers = NULL;
+ * target-declared registers are invalidated. The driver shares this instruction
+ * effect with constant tracking, so the hot forward walk scans operands once. */
+static void ctx_record(CombineCtx* ctx, const Inst* in,
+ const OptHardRegSet* kills, i32 i) {
if (inst_writes_memory(in)) ctx->last_mem_def = i;
- for (u32 op = 0; op < in->nopnds; ++op)
- if (opt_inst_operand_is_def(in, op))
- ctx_record_operand_def(ctx, &in->opnds[op], i);
-
- switch ((IROp)in->op) {
- case IR_CALL: {
- const IRCallAux* aux = (const IRCallAux*)in->extra.aux;
- if (aux && aux->use_plan_replay) {
- for (u32 a = 0; a < aux->plan.nargs; ++a)
- if (aux->plan.args[a].dst_kind == CG_CALL_PLAN_REG)
- ctx_record_hard_def(ctx, aux->plan.args[a].cls,
- aux->plan.args[a].dst_reg, i);
- for (u32 r = 0; r < aux->plan.nrets; ++r) {
- ctx_record_hard_def(ctx, aux->plan.rets[r].cls,
- aux->plan.rets[r].src_reg, i);
- ctx_record_operand_def(ctx, &aux->plan.rets[r].dst, i);
- }
- } else if (aux) {
- ctx_record_abi_defs(ctx, &aux->desc.ret, i);
- }
- for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls)
- ctx_record_mask(ctx, (u8)cls,
- opt_call_clobber_mask_for(ctx->f, in, (u8)cls), i);
- break;
- }
- case IR_ASM_BLOCK: {
- const IRAsmAux* aux = (const IRAsmAux*)in->extra.aux;
- if (!aux) break;
- for (u32 out = 0; out < aux->nout; ++out)
- ctx_record_operand_def(ctx, &aux->out_ops[out], i);
- for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls)
- ctx_record_mask(ctx, (u8)cls, aux->clobber_mask[cls], i);
- break;
- }
- case IR_INTRINSIC: {
- const IRIntrinAux* aux = (const IRIntrinAux*)in->extra.aux;
- if (!aux) break;
- for (u32 dst = 0; dst < aux->ndst; ++dst)
- ctx_record_operand_def(ctx, &aux->dsts[dst], i);
- break;
- }
- default:
- break;
- }
-
- if (ctx->f->inst_clobbers && in->id != INST_ID_NONE &&
- in->id < ctx->f->inst_clobbers_cap)
- machine_clobbers = ctx->f->inst_clobbers[in->id];
- if (machine_clobbers)
- for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls)
- ctx_record_mask(ctx, (u8)cls, machine_clobbers[cls], i);
+ for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls)
+ ctx_record_mask(ctx, (u8)cls, kills->cls[cls], i);
}
/* Lookup the producer of (cls, reg) in this BB, if any. Returns -1 if no
@@ -506,8 +439,9 @@ static int count_uses_in_live_range(Func* f, const Block* bl, i32 prod_idx,
int killed = 0;
for (i32 i = prod_idx + 1; i < (i32)bl->ninsts; ++i) {
const Inst* in = &bl->insts[i];
- n += inst_uses_phys_reg(f, in, def);
- if (inst_kills_phys_reg(f, in, def)) {
+ CombinePhysRegEffect effect = inst_phys_reg_effect(f, in, def);
+ n += effect.uses;
+ if (effect.defines || effect.clobbers) {
killed = 1;
break;
}
@@ -521,9 +455,10 @@ static int use_after_clobber_before_redef(Func* f, const Block* bl,
int saw_clobber = 0;
for (i32 i = prod_idx + 1; i < (i32)bl->ninsts; ++i) {
const Inst* in = &bl->insts[i];
- if (saw_clobber && inst_uses_phys_reg(f, in, def)) return 1;
- if (inst_defines_phys_reg(f, in, def)) return 0;
- if (inst_clobbers_phys_reg(f, in, def)) saw_clobber = 1;
+ CombinePhysRegEffect effect = inst_phys_reg_effect(f, in, def);
+ if (saw_clobber && effect.uses) return 1;
+ if (effect.defines) return 0;
+ if (effect.clobbers) saw_clobber = 1;
}
return 0;
}
@@ -1278,8 +1213,8 @@ static int try_sink(CombineCtx* ctx, Inst* in, i32 i) {
* copy that originally wrote it. */
for (i32 j = prod_idx + 1; j < i; ++j) {
Inst* mid = &ctx->bl->insts[j];
- if (inst_uses_phys_reg(ctx->f, mid, &dst) ||
- inst_kills_phys_reg(ctx->f, mid, &dst))
+ CombinePhysRegEffect effect = inst_phys_reg_effect(ctx->f, mid, &dst);
+ if (effect.uses || effect.defines || effect.clobbers)
return 0;
}
@@ -2119,14 +2054,11 @@ static int try_fold_const_convert(CombineCtx* ctx, Inst* in, i32 i) {
/* Record / invalidate the per-register known-constant state after visiting the
* inst at index `i`. Must run for EVERY inst (called from the forward driver),
* so the tracking stays sound across barriers and redefinitions. */
-static void ctx_track_const(CombineCtx* ctx, const Inst* in) {
- OptRegEffects effects;
- OptHardRegSet kills;
- opt_inst_reg_effects(ctx->f, in, &effects);
- opt_reg_effect_kills(&effects, &kills);
+static void ctx_track_const(CombineCtx* ctx, const Inst* in,
+ const OptHardRegSet* kills) {
/* Any register this inst defines or clobbers no longer holds its old known
* constant. (RC_INT only — cmp immediates are integer.) */
- ctx->const_valid &= ~kills.cls[RC_INT];
+ ctx->const_valid &= ~kills->cls[RC_INT];
/* A load_imm of an integer hard register records the new constant. */
if ((IROp)in->op == IR_LOAD_IMM && in->nopnds >= 1 &&
in->opnds[0].kind == OPK_REG && in->opnds[0].cls == RC_INT &&
@@ -2198,10 +2130,8 @@ static int opt_combine_fold_block(Func* f, Block* bl,
/* The producer's destination changed. Remove every old effect no longer
* present, then record its canonical post-rewrite effects. */
Inst* prev = &bl->insts[i - 1];
- OptRegEffects effects;
OptHardRegSet kills;
- opt_inst_reg_effects(f, prev, &effects);
- opt_reg_effect_kills(&effects, &kills);
+ opt_inst_reg_kills(f, prev, &kills);
for (u8 c = 0; c < OPT_REG_CLASSES; ++c) {
for (Reg r = 0; r < OPT_MAX_HARD_REGS; ++r) {
if (ctx.last_def[c][r] == i - 1 &&
@@ -2215,7 +2145,7 @@ static int opt_combine_fold_block(Func* f, Block* bl,
}
}
}
- ctx_record(&ctx, prev, i - 1);
+ ctx_record(&ctx, prev, &kills, i - 1);
}
/* Skip NOPs left by prior sink rewrites. */
@@ -2266,10 +2196,14 @@ static int opt_combine_fold_block(Func* f, Block* bl,
addr_cse_record(&ctx, in, i);
load_cse_record(&ctx, in, i);
compute_cse_record(&ctx, in, i);
- ctx_record(&ctx, in, i);
- /* W6: update the per-register known-constant tracker for this inst (after
- * rewrites, so a load_imm produced/rewritten here is recorded). */
- ctx_track_const(&ctx, in);
+ {
+ OptHardRegSet kills;
+ opt_inst_reg_kills(f, in, &kills);
+ ctx_record(&ctx, in, &kills, i);
+ /* W6: update the per-register known-constant tracker for this inst
+ * (after rewrites, so a load_imm produced/rewritten here is recorded). */
+ ctx_track_const(&ctx, in, &kills);
+ }
}
return ctx.block_change_p;
}
diff --git a/src/opt/pass_hard_live.c b/src/opt/pass_hard_live.c
@@ -45,10 +45,10 @@ static void hard_live_blocks(Func* f, OptHardBlockLive* live) {
memset(&seen_def, 0, sizeof seen_def);
memset(&live[b], 0, sizeof live[b]);
for (u32 i = 0; i < bl->ninsts; ++i) {
- OptRegEffects effects;
+ OptRegEffectMasks effects;
OptHardRegSet kills;
- opt_inst_reg_effects(f, &bl->insts[i], &effects);
- opt_reg_effect_kills(&effects, &kills);
+ opt_inst_reg_effect_masks(f, &bl->insts[i], &effects);
+ opt_reg_effect_mask_kills(&effects, &kills);
for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
live[b].live_use.cls[c] |=
effects.uses.cls[c] & ~seen_def.cls[c];
diff --git a/src/opt/pass_lower.c b/src/opt/pass_lower.c
@@ -234,11 +234,10 @@ static void apply_asm_register_constraints(Func* f, Inst* in, u64* use,
* the backend stages them into/out of the fixed registers itself. */
static void apply_machine_reg_clobbers(Func* f, Inst* in, u64* def,
u64* live_after) {
- if (!f->preg_info || !f->inst_clobbers || in->id == INST_ID_NONE ||
- in->id >= f->inst_clobbers_cap)
- return;
+ const u32* clobbers = opt_inst_machine_clobber_masks(f, in);
+ if (!f->preg_info || !clobbers) return;
for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls) {
- u32 mask = f->inst_clobbers[in->id][cls];
+ u32 mask = clobbers[cls];
if (!mask) continue;
for (Reg r = 0; r < OPT_MAX_HARD_REGS; ++r) {
if ((mask & (1u << r)) == 0) continue;
diff --git a/src/opt/pass_native_emit.c b/src/opt/pass_native_emit.c
@@ -333,12 +333,12 @@ static int temp_loc_equal(NativeLoc a, NativeLoc b) {
static void temp_scope_apply_effects(NativeEmitCtx* e,
NativeEmitTempScope* scope) {
- OptRegEffects effects;
+ OptRegEffectMasks effects;
FrameCacheNeedCtx needed;
u32 invalidate[OPT_REG_CLASSES];
if (!e || !scope || scope->effects_ready) return;
memset(&needed, 0, sizeof needed);
- opt_inst_reg_effects(e->f, scope->inst, &effects);
+ opt_inst_reg_effect_masks(e->f, scope->inst, &effects);
needed.emit = e;
frame_cache_collect_needed(scope->inst, &needed);
for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
@@ -398,9 +398,7 @@ static void temp_scope_begin(NativeEmitCtx* e, NativeEmitTempScope* scope,
* bank. Only an exceptional backend-declared machine clobber can collide
* with those temps, so handle that table directly and defer the full
* use/def analysis alongside hard liveness. */
- if (in && in->id != INST_ID_NONE && e->f->inst_clobbers &&
- in->id < e->f->inst_clobbers_cap)
- machine_clobbers = e->f->inst_clobbers[in->id];
+ machine_clobbers = opt_inst_machine_clobber_masks(e->f, in);
if (in && ((IROp)in->op == IR_CALL || (IROp)in->op == IR_ASM_BLOCK)) {
frame_cache_clear(e);
} else if (machine_clobbers) {
@@ -2508,34 +2506,6 @@ static void collect_used_reg(Func* f, Inst* in, OptOperand* op, int is_def,
used[op->cls] |= 1u << op->v.reg;
}
-/* After register allocation the MIR names hard registers directly, so we scan
- * it for the callee-saved registers the allocator assigned. Fills `used[cls]`
- * (one bitmask per alloc class, masked to each class's callee-saved set) and
- * returns the class count. The masks feed NativeKnownFrameDesc so the backend
- * reserves the save slots as part of the up-front frame. */
-static u32 compute_callee_saved_used(NativeEmitCtx* e, u32* used, u32 cap) {
- NativeTarget* t = e->target;
- const NativeRegInfo* ri = t->regs;
- u32 nclasses;
- for (u32 i = 0; i < cap; ++i) used[i] = 0;
- if (!ri) return 0;
- for (u32 b = 0; b < e->f->nblocks; ++b) {
- Block* bl = &e->f->blocks[b];
- for (u32 i = 0; i < bl->ninsts; ++i)
- opt_walk_inst_operands(e->f, &bl->insts[i], collect_used_reg, used);
- }
- nclasses = ri->nclasses < cap ? ri->nclasses : cap;
- for (u32 i = 0; i < ri->nclasses; ++i) {
- const NativeAllocClassInfo* ci = &ri->classes[i];
- if (ci->cls < cap) {
- used[ci->cls] &=
- native_target_callee_saved_mask(t, (NativeAllocClass)ci->cls);
- used[ci->cls] |= e->asm_stage_callee_used[ci->cls];
- }
- }
- return nclasses;
-}
-
static void compute_emit_live_after(NativeEmitCtx* e) {
OptHardBlockLive* blocks = opt_maybe_build_hard_live(e->f);
e->live_after_ready = 1u;
@@ -2549,72 +2519,76 @@ static void compute_emit_live_after(NativeEmitCtx* e) {
opt_hard_live_out_for_block(blocks ? &blocks[b] : NULL);
for (u32 ri = bl->ninsts; ri > 0; --ri) {
Inst* in = &bl->insts[ri - 1u];
- OptRegEffects effects;
+ OptRegEffectMasks effects;
OptHardRegSet kills;
if (in->id != INST_ID_NONE && in->id < e->live_after_cap)
e->live_after_by_inst[in->id] = live;
- opt_inst_reg_effects(e->f, in, &effects);
- opt_reg_effect_kills(&effects, &kills);
+ opt_inst_reg_effect_masks(e->f, in, &effects);
+ opt_reg_effect_mask_kills(&effects, &kills);
opt_hard_live_step(&live, &effects.uses, &kills);
}
}
}
-static int asm_mir_reg_satisfies(NativeEmitCtx* e, const OptOperand* op,
- NativeAllocClass cls, Reg fixed,
- u32 allowed) {
- NativeLoc loc;
- if (!op || op->kind != OPK_REG) return 0;
- loc = loc_reg(op->type, (NativeAllocClass)op->cls, op->v.reg);
- return asm_reg_loc_satisfies(e, loc, cls, fixed, allowed);
+static NativeLoc asm_planning_loc(const OptOperand* op) {
+ NativeLoc loc = loc_none();
+ if (!op) return loc;
+ loc.type = op->type;
+ if (op->kind == OPK_REG)
+ loc = loc_reg(op->type, (NativeAllocClass)op->cls, op->v.reg);
+ return loc;
}
-/* Dry-run the instruction-local asm allocator before frame finalization. The
- * same candidate order and unavailable masks are used by final emission, so
- * this pass need only retain the ABI-relevant result: which callee-saved
- * registers the staging leases will overwrite. */
-static void plan_asm_staging(NativeEmitCtx* e) {
- memset(e->asm_stage_callee_used, 0, sizeof e->asm_stage_callee_used);
- for (u32 b = 0; b < e->f->nblocks; ++b) {
- Block* bl = &e->f->blocks[b];
- for (u32 i = 0; i < bl->ninsts; ++i) {
- Inst* in = &bl->insts[i];
- IRAsmAux* aux;
- NativeEmitTempScope scope;
- if ((IROp)in->op != IR_ASM_BLOCK) continue;
- aux = (IRAsmAux*)in->extra.aux;
- if (!aux) continue;
- temp_scope_begin(e, &scope, in);
- scope.allow_asm_temps = (u8)!aux->has_memory_constraint;
- for (u32 k = 0; k < aux->nout; ++k) {
- NativeAllocClass cls;
- Reg fixed;
- u32 allowed;
- if (!asm_reg_requirement(
- e, aux->out_reg_reqs ? &aux->out_reg_reqs[k] : NULL, &cls,
- &fixed, &allowed, in->loc) ||
- asm_mir_reg_satisfies(e, &aux->out_ops[k], cls, fixed, allowed))
- continue;
- (void)temp_acquire_asm(e, cls, fixed, allowed, in->loc);
- }
- for (u32 k = 0; k < aux->nin; ++k) {
- NativeAllocClass cls;
- Reg fixed;
- u32 allowed;
- if (native_asm_match_index(aux->ins[k].str) >= 0) continue;
- if (!asm_reg_requirement(
- e, aux->in_reg_reqs ? &aux->in_reg_reqs[k] : NULL, &cls,
- &fixed, &allowed, in->loc) ||
- asm_mir_reg_satisfies(e, &aux->in_ops[k], cls, fixed, allowed))
- continue;
- (void)temp_acquire_asm(e, cls, fixed, allowed, in->loc);
- }
- for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
- e->asm_stage_callee_used[c] |=
- scope.leased[c] & e->f->opt_callee_saved[c];
- temp_scope_end(e, &scope);
- }
+/* Dry-run one instruction's asm allocator before frame finalization. The same
+ * candidate order and unavailable masks are used by final emission, so retain
+ * only the ABI-relevant result: callee-saved staging registers overwritten. */
+static void plan_one_asm_staging(NativeEmitCtx* e, Inst* in, IRAsmAux* aux) {
+ NativeEmitTempScope scope;
+ if (!aux) return;
+ temp_scope_begin(e, &scope, in);
+ scope.allow_asm_temps = (u8)!aux->has_memory_constraint;
+ for (u32 k = 0; k < aux->nout; ++k) {
+ (void)asm_stage_reg_operand(
+ e, &aux->outs[k],
+ aux->out_reg_reqs ? &aux->out_reg_reqs[k] : NULL,
+ asm_planning_loc(&aux->out_ops[k]), 0, NULL, in->loc);
+ }
+ for (u32 k = 0; k < aux->nin; ++k) {
+ if (native_asm_match_index(aux->ins[k].str) >= 0) continue;
+ (void)asm_stage_reg_operand(
+ e, &aux->ins[k],
+ aux->in_reg_reqs ? &aux->in_reg_reqs[k] : NULL,
+ asm_planning_loc(&aux->in_ops[k]), 0, NULL, in->loc);
+ }
+ for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
+ e->asm_stage_callee_used[c] |=
+ scope.leased[c] & e->f->opt_callee_saved[c];
+ temp_scope_end(e, &scope);
+}
+
+static NativeLoc* plan_call_args_reserve(NativeEmitCtx* e, NativeLoc* args,
+ u32* cap, u32 want) {
+ u32 grown;
+ if (want <= *cap) return args;
+ grown = *cap ? *cap : want;
+ while (grown < want && grown <= 0x7fffffffu) grown *= 2u;
+ if (grown < want) grown = want;
+ *cap = grown;
+ return arena_zarray(e->f->arena, NativeLoc, grown);
+}
+
+static void plan_asm_clobber_push(NativeEmitCtx* e, Sym** clobbers, u32* count,
+ u32* cap, Sym clobber) {
+ if (*count == *cap) {
+ u32 grown = *cap ? *cap * 2u : 8u;
+ Sym* next;
+ if (grown < *cap) grown = *count + 1u;
+ next = arena_array(e->f->arena, Sym, grown);
+ if (*count) memcpy(next, *clobbers, sizeof *next * *count);
+ *clobbers = next;
+ *cap = grown;
}
+ (*clobbers)[(*count)++] = clobber;
}
/* Plan the complete call frame before any code is emitted, then hand it to the
@@ -2632,28 +2606,30 @@ static void plan_frame(NativeEmitCtx* e, const CGFuncDesc* fd) {
NativeFrameSlot* out_slots = NULL;
u32 used[EMIT_MAX_REG_CLASSES];
u32 nclasses;
- u32 max_args = 0, max_outgoing = 0;
+ NativeLoc* call_args = NULL;
+ u32 call_args_cap = 0;
+ u32 max_outgoing = 0;
u8 has_alloca = 0;
u8 needs_scratch_spill = 0;
u8 has_call = 0;
u8 has_asm = 0;
u8 reads_frame = 0;
u32 nasm_clob = 0;
+ u32 asm_clob_cap = 0;
u32 asm_clobber_abi_sets = 0;
Sym* asm_clobbers = NULL;
memset(&frame, 0, sizeof frame);
- nclasses = t->reserve_callee_saves
- ? compute_callee_saved_used(e, used, EMIT_MAX_REG_CLASSES)
- : 0u;
+ memset(used, 0, sizeof used);
+ memset(e->asm_stage_callee_used, 0, sizeof e->asm_stage_callee_used);
memset(e->callee_saved_used, 0, sizeof e->callee_saved_used);
- for (u32 c = 0; c < nclasses && c < OPT_REG_CLASSES; ++c)
- e->callee_saved_used[c] = used[c];
- /* Outgoing-arg area = max stack-arg bytes over all calls; also note alloca.
- */
+ /* One primary MIR scan collects allocated hard registers, asm staging, and
+ * all frame-shape facts. This used to be three independent full walks. */
for (u32 b = 0; b < e->f->nblocks; ++b) {
Block* bl = &e->f->blocks[b];
for (u32 i = 0; i < bl->ninsts; ++i) {
Inst* in = &bl->insts[i];
+ if (t->reserve_callee_saves)
+ opt_walk_inst_operands(e->f, in, collect_used_reg, used);
if ((IROp)in->op == IR_ALLOCA) {
has_alloca = 1;
} else if ((IROp)in->op == IR_ATOMIC_RMW) {
@@ -2663,22 +2639,44 @@ static void plan_frame(NativeEmitCtx* e, const CGFuncDesc* fd) {
/* Any call (regular or sibling/tail) means the function is not a leaf:
* it clobbers the return-address register and the stack below sp. */
has_call = 1;
- if (aux && aux->desc.nargs > max_args) max_args = aux->desc.nargs;
+ if (aux && t->call_stack_bytes) {
+ NativeCallDesc d;
+ u32 sb;
+ call_args = plan_call_args_reserve(
+ e, call_args, &call_args_cap, aux->desc.nargs);
+ memset(&d, 0, sizeof d);
+ d.fn_type = aux->desc.fn_type;
+ d.flags = aux->desc.flags;
+ d.nargs = aux->desc.nargs;
+ for (u32 k = 0; k < aux->desc.nargs; ++k) {
+ memset(&call_args[k], 0, sizeof call_args[k]);
+ call_args[k].type = aux->desc.args[k].type;
+ }
+ d.args = call_args;
+ sb = t->call_stack_bytes(t, &d);
+ if (sb > max_outgoing) max_outgoing = sb;
+ }
} else if ((IROp)in->op == IR_ASM_BLOCK) {
/* Inline asm may clobber the return-address register or the red zone
* opaquely; disqualifies the frame-eliding tiers (see has_asm). Its
* callee-saved register clobbers and named-register operand
- * requirements are equally opaque to the operand scan below; count
- * them now so the backend can fold them into the saved set (collected
- * into a single Sym list in a second pass below). */
+ * requirements are equally opaque to the operand scan below; collect
+ * them now so the backend can fold them into the saved set. */
IRAsmAux* aux = (IRAsmAux*)in->extra.aux;
has_asm = 1;
if (aux) {
- nasm_clob += aux->nclob;
+ plan_one_asm_staging(e, in, aux);
+ for (u32 k = 0; k < aux->nclob; ++k)
+ plan_asm_clobber_push(e, &asm_clobbers, &nasm_clob,
+ &asm_clob_cap, aux->clobbers[k]);
for (u32 k = 0; k < aux->nout; ++k)
- if (aux->outs[k].reg) ++nasm_clob;
+ if (aux->outs[k].reg)
+ plan_asm_clobber_push(e, &asm_clobbers, &nasm_clob,
+ &asm_clob_cap, aux->outs[k].reg);
for (u32 k = 0; k < aux->nin; ++k)
- if (aux->ins[k].reg) ++nasm_clob;
+ if (aux->ins[k].reg)
+ plan_asm_clobber_push(e, &asm_clobbers, &nasm_clob,
+ &asm_clob_cap, aux->ins[k].reg);
asm_clobber_abi_sets |= aux->clobber_abi_sets;
}
} else if ((IROp)in->op == IR_INTRINSIC) {
@@ -2700,57 +2698,20 @@ static void plan_frame(NativeEmitCtx* e, const CGFuncDesc* fd) {
}
}
}
- /* Gather the union of every asm block's clobber names and named-register
- * operand requirements for known-frame callee-save reservation. Keep the raw
- * names: each backend's frame planner uses its clobber parser so diagnostics
- * and ABI exclusions match final emission. */
- if (nasm_clob) {
- u32 n = 0;
- asm_clobbers = arena_array(e->f->arena, Sym, nasm_clob);
- for (u32 b = 0; b < e->f->nblocks; ++b) {
- Block* bl = &e->f->blocks[b];
- for (u32 i = 0; i < bl->ninsts; ++i) {
- Inst* in = &bl->insts[i];
- IRAsmAux* aux;
- if ((IROp)in->op != IR_ASM_BLOCK) continue;
- aux = (IRAsmAux*)in->extra.aux;
- for (u32 k = 0; aux && k < aux->nclob; ++k)
- asm_clobbers[n++] = aux->clobbers[k];
- for (u32 k = 0; aux && k < aux->nout; ++k)
- if (aux->outs[k].reg) asm_clobbers[n++] = aux->outs[k].reg;
- for (u32 k = 0; aux && k < aux->nin; ++k)
- if (aux->ins[k].reg) asm_clobbers[n++] = aux->ins[k].reg;
- }
- }
- nasm_clob = n;
- }
- if (t->call_stack_bytes) {
- NativeLoc* args =
- max_args ? arena_zarray(e->f->arena, NativeLoc, max_args) : NULL;
- for (u32 b = 0; b < e->f->nblocks; ++b) {
- Block* bl = &e->f->blocks[b];
- for (u32 i = 0; i < bl->ninsts; ++i) {
- Inst* in = &bl->insts[i];
- IRCallAux* aux;
- NativeCallDesc d;
- u32 sb;
- if ((IROp)in->op != IR_CALL) continue;
- aux = (IRCallAux*)in->extra.aux;
- if (!aux) continue;
- memset(&d, 0, sizeof d);
- d.fn_type = aux->desc.fn_type;
- d.flags = aux->desc.flags;
- d.nargs = aux->desc.nargs;
- for (u32 k = 0; k < aux->desc.nargs; ++k) {
- memset(&args[k], 0, sizeof args[k]);
- args[k].type = aux->desc.args[k].type;
- }
- d.args = args;
- sb = t->call_stack_bytes(t, &d);
- if (sb > max_outgoing) max_outgoing = sb;
- }
- }
+ nclasses = t->reserve_callee_saves && t->regs
+ ? (t->regs->nclasses < EMIT_MAX_REG_CLASSES
+ ? t->regs->nclasses
+ : EMIT_MAX_REG_CLASSES)
+ : 0u;
+ for (u32 i = 0; t->regs && i < t->regs->nclasses; ++i) {
+ const NativeAllocClassInfo* ci = &t->regs->classes[i];
+ if (ci->cls >= EMIT_MAX_REG_CLASSES) continue;
+ used[ci->cls] &=
+ native_target_callee_saved_mask(t, (NativeAllocClass)ci->cls);
+ used[ci->cls] |= e->asm_stage_callee_used[ci->cls];
}
+ for (u32 c = 0; c < nclasses && c < OPT_REG_CLASSES; ++c)
+ e->callee_saved_used[c] = used[c];
e->slot_map =
arena_zarray(e->f->arena, NativeFrameSlot, e->f->nframe_slots + 1u);
if (e->f->nframe_slots) {
@@ -2889,7 +2850,6 @@ void opt_emit_native(Compiler* c, Func* f, NativeTarget* target) {
arena_zarray(e.f->arena, u8, e.f->nblocks ? e.f->nblocks : 1u);
for (u32 i = 0; i < e.f->nblocks; ++i) e.labels[i] = MC_LABEL_NONE;
fd = semantic_func_desc(&e);
- plan_asm_staging(&e);
metrics_scope_end(c, "opt.native_emit.setup");
metrics_scope_begin(c, "opt.native_emit.func_begin");
diff --git a/src/opt/reg_effects.c b/src/opt/reg_effects.c
@@ -2,44 +2,58 @@
#include "opt/opt_internal.h"
+#if defined(__GNUC__) || defined(__clang__)
+#define EFFECT_ALWAYS_INLINE static inline __attribute__((always_inline))
+#else
+#define EFFECT_ALWAYS_INLINE static inline
+#endif
+
static void effect_add(OptHardRegSet* set, u8 cls, Reg reg) {
if (!set || cls >= OPT_REG_CLASSES || reg >= OPT_MAX_HARD_REGS) return;
set->cls[cls] |= 1u << reg;
}
-static void effect_add_use(OptRegEffects* effects, u8 cls, Reg reg) {
+static void effect_add_use(OptRegEffectMasks* effects,
+ u8 (*use_count)[OPT_MAX_HARD_REGS], u8 cls,
+ Reg reg) {
if (!effects || cls >= OPT_REG_CLASSES || reg >= OPT_MAX_HARD_REGS) return;
effect_add(&effects->uses, cls, reg);
- if (effects->use_count[cls][reg] < 2u)
- ++effects->use_count[cls][reg];
+ if (use_count && use_count[cls][reg] < 2u)
+ ++use_count[cls][reg];
}
-static void effect_use_operand(OptRegEffects* effects, const Operand* op) {
+EFFECT_ALWAYS_INLINE void effect_use_operand(
+ OptRegEffectMasks* effects, u8 (*use_count)[OPT_MAX_HARD_REGS],
+ const Operand* op) {
if (!effects || !op) return;
if (op->kind == OPK_REG) {
- effect_add_use(effects, op->cls, op->v.reg);
+ effect_add_use(effects, use_count, op->cls, op->v.reg);
} else if (op->kind == OPK_INDIRECT) {
if (op->v.ind.base_kind == OPT_INDIRECT_REG)
- effect_add_use(effects, RC_INT, op->v.ind.base);
+ effect_add_use(effects, use_count, RC_INT, op->v.ind.base);
if (op->v.ind.index_kind == OPT_INDIRECT_REG &&
op->v.ind.index != (Reg)REG_NONE)
- effect_add_use(effects, RC_INT, op->v.ind.index);
+ effect_add_use(effects, use_count, RC_INT, op->v.ind.index);
}
}
-static void effect_def_operand(OptRegEffects* effects, const Operand* op) {
+static void effect_def_operand(OptRegEffectMasks* effects,
+ const Operand* op) {
if (effects && op && op->kind == OPK_REG)
effect_add(&effects->defs, op->cls, op->v.reg);
}
-static void effect_use_abivalue(OptRegEffects* effects, const CGABIValue* v) {
+EFFECT_ALWAYS_INLINE void effect_use_abivalue(
+ OptRegEffectMasks* effects, u8 (*use_count)[OPT_MAX_HARD_REGS],
+ const CGABIValue* v) {
if (!v) return;
- effect_use_operand(effects, &v->storage);
+ effect_use_operand(effects, use_count, &v->storage);
for (u32 i = 0; i < v->nparts; ++i)
- effect_use_operand(effects, &v->parts[i].op);
+ effect_use_operand(effects, use_count, &v->parts[i].op);
}
-static void effect_def_abivalue(OptRegEffects* effects, const CGABIValue* v) {
+static void effect_def_abivalue(OptRegEffectMasks* effects,
+ const CGABIValue* v) {
if (!v) return;
effect_def_operand(effects, &v->storage);
for (u32 i = 0; i < v->nparts; ++i)
@@ -60,13 +74,24 @@ u32 opt_call_clobber_mask_for(Func* f, const Inst* in, u8 cls) {
return f->opt_caller_saved[cls];
}
-static void effect_explicit_operands(Func* f, const Inst* in,
- OptRegEffects* effects) {
+const u32* opt_inst_machine_clobber_masks(Func* f, const Inst* in) {
+ if (!f || !in || !f->inst_clobbers || in->id == INST_ID_NONE ||
+ in->id >= f->inst_clobbers_cap)
+ return NULL;
+ return f->inst_clobbers[in->id];
+}
+
+/* This is the sole opcode/ABI authority for physical-register effects. Every
+ * detailed, masks-only, and kills-only view below is a projection of this
+ * walker; consumers must not reproduce these operand or clobber rules. */
+EFFECT_ALWAYS_INLINE void effect_explicit_operands(
+ Func* f, const Inst* in, OptRegEffectMasks* effects,
+ u8 (*use_count)[OPT_MAX_HARD_REGS], int collect_uses) {
for (u32 i = 0; i < in->nopnds; ++i) {
if (opt_inst_operand_is_def(in, i))
effect_def_operand(effects, &in->opnds[i]);
- else
- effect_use_operand(effects, &in->opnds[i]);
+ else if (collect_uses)
+ effect_use_operand(effects, use_count, &in->opnds[i]);
}
/* Direct operand roles come from opt_inst_operand_is_def above. Only
@@ -75,9 +100,11 @@ static void effect_explicit_operands(Func* f, const Inst* in,
case IR_CALL: {
const IRCallAux* aux = (const IRCallAux*)in->extra.aux;
if (aux && aux->use_plan_replay) {
- effect_use_operand(effects, &aux->plan.callee);
+ if (collect_uses)
+ effect_use_operand(effects, use_count, &aux->plan.callee);
for (u32 i = 0; i < aux->plan.nargs; ++i) {
- effect_use_operand(effects, &aux->plan.args[i].src);
+ if (collect_uses)
+ effect_use_operand(effects, use_count, &aux->plan.args[i].src);
if (aux->plan.args[i].dst_kind == CG_CALL_PLAN_REG)
effect_add(&effects->defs, aux->plan.args[i].cls,
aux->plan.args[i].dst_reg);
@@ -88,9 +115,11 @@ static void effect_explicit_operands(Func* f, const Inst* in,
effect_def_operand(effects, &aux->plan.rets[i].dst);
}
} else if (aux) {
- effect_use_operand(effects, &aux->desc.callee);
- for (u32 i = 0; i < aux->desc.nargs; ++i)
- effect_use_abivalue(effects, &aux->desc.args[i]);
+ if (collect_uses) {
+ effect_use_operand(effects, use_count, &aux->desc.callee);
+ for (u32 i = 0; i < aux->desc.nargs; ++i)
+ effect_use_abivalue(effects, use_count, &aux->desc.args[i]);
+ }
effect_def_abivalue(effects, &aux->desc.ret);
}
for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
@@ -100,14 +129,16 @@ static void effect_explicit_operands(Func* f, const Inst* in,
}
case IR_RET: {
const IRRetAux* aux = (const IRRetAux*)in->extra.aux;
- if (aux && aux->present) effect_use_abivalue(effects, &aux->val);
+ if (collect_uses && aux && aux->present)
+ effect_use_abivalue(effects, use_count, &aux->val);
break;
}
case IR_ASM_BLOCK: {
const IRAsmAux* aux = (const IRAsmAux*)in->extra.aux;
if (!aux) break;
- for (u32 i = 0; i < aux->nin; ++i)
- effect_use_operand(effects, &aux->in_ops[i]);
+ if (collect_uses)
+ for (u32 i = 0; i < aux->nin; ++i)
+ effect_use_operand(effects, use_count, &aux->in_ops[i]);
for (u32 i = 0; i < aux->nout; ++i)
effect_def_operand(effects, &aux->out_ops[i]);
effect_add_masks(&effects->clobbers, aux->clobber_mask);
@@ -116,8 +147,9 @@ static void effect_explicit_operands(Func* f, const Inst* in,
case IR_INTRINSIC: {
const IRIntrinAux* aux = (const IRIntrinAux*)in->extra.aux;
if (!aux) break;
- for (u32 i = 0; i < aux->narg; ++i)
- effect_use_operand(effects, &aux->args[i]);
+ if (collect_uses)
+ for (u32 i = 0; i < aux->narg; ++i)
+ effect_use_operand(effects, use_count, &aux->args[i]);
for (u32 i = 0; i < aux->ndst; ++i)
effect_def_operand(effects, &aux->dsts[i]);
break;
@@ -128,25 +160,51 @@ static void effect_explicit_operands(Func* f, const Inst* in,
}
static void effect_add_machine_clobbers(Func* f, const Inst* in,
- OptRegEffects* effects) {
- if (!f->inst_clobbers || in->id == INST_ID_NONE ||
- in->id >= f->inst_clobbers_cap)
- return;
- effect_add_masks(&effects->clobbers, f->inst_clobbers[in->id]);
+ OptRegEffectMasks* effects) {
+ effect_add_masks(&effects->clobbers,
+ opt_inst_machine_clobber_masks(f, in));
+}
+
+EFFECT_ALWAYS_INLINE void effect_build(
+ Func* f, const Inst* in, OptRegEffectMasks* effects,
+ u8 (*use_count)[OPT_MAX_HARD_REGS], int collect_uses) {
+ if (!f || !in) return;
+ effect_explicit_operands(f, in, effects, use_count, collect_uses);
+ effect_add_machine_clobbers(f, in, effects);
}
void opt_inst_reg_effects(Func* f, const Inst* in, OptRegEffects* effects) {
if (!effects) return;
memset(effects, 0, sizeof *effects);
- if (!f || !in) return;
- effect_explicit_operands(f, in, effects);
- effect_add_machine_clobbers(f, in, effects);
+ effect_build(f, in, &effects->masks, effects->use_count, 1);
+}
+
+void opt_inst_reg_effect_masks(Func* f, const Inst* in,
+ OptRegEffectMasks* masks) {
+ if (!masks) return;
+ memset(masks, 0, sizeof *masks);
+ effect_build(f, in, masks, NULL, 1);
+}
+
+void opt_inst_reg_kills(Func* f, const Inst* in, OptHardRegSet* kills) {
+ OptRegEffectMasks effects;
+ if (!kills) return;
+ memset(&effects, 0, sizeof effects);
+ effect_build(f, in, &effects, NULL, 0);
+ opt_reg_effect_mask_kills(&effects, kills);
}
void opt_reg_effect_kills(const OptRegEffects* effects, OptHardRegSet* kills) {
+ opt_reg_effect_mask_kills(effects ? &effects->masks : NULL, kills);
+}
+
+void opt_reg_effect_mask_kills(const OptRegEffectMasks* masks,
+ OptHardRegSet* kills) {
if (!kills) return;
memset(kills, 0, sizeof *kills);
- if (!effects) return;
+ if (!masks) return;
for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
- kills->cls[c] = effects->defs.cls[c] | effects->clobbers.cls[c];
+ kills->cls[c] = masks->defs.cls[c] | masks->clobbers.cls[c];
}
+
+#undef EFFECT_ALWAYS_INLINE
diff --git a/test/api/profile_test.c b/test/api/profile_test.c
@@ -16,6 +16,10 @@ static void check_names_and_ranges(void) {
EXPECT(strcmp(kit_profile_counter_name(KIT_PROFILE_COUNTER_COMPILE_INPUT_BYTES),
"compile.input_bytes") == 0,
"builtin counter name");
+ EXPECT(strcmp(kit_profile_counter_name(
+ KIT_PROFILE_COUNTER_OPT_NATIVE_EMIT_SCAVENGES),
+ "opt.native_emit.scavenges") == 0,
+ "native emit scavenger counter name");
EXPECT(kit_profile_scope_name(KIT_PROFILE_SCOPE_LANG_FIRST) == NULL,
"language scopes are open");
EXPECT(kit_profile_counter_name(KIT_PROFILE_COUNTER_EXTERNAL_FIRST) == NULL,
diff --git a/test/opt/frame_value_backend_test.c b/test/opt/frame_value_backend_test.c
@@ -26,6 +26,11 @@
#include "core/pool.h"
#include "lib/kit_unit.h"
#include "obj/obj.h"
+#include "opt/opt.h"
+
+/* Keep NativeTarget callbacks on the semantic descriptor while naming the
+ * optimizer-private descriptor explicitly in the forced-emitter test. */
+#undef CGFuncDesc
typedef NativeTarget *(*NativeCtor)(Compiler *, ObjBuilder *, MCEmitter *);
@@ -749,6 +754,147 @@ done:
kit_compiler_free(kit_c);
}
+static Operand forced_stack_op(FrameSlot slot, KitCgTypeId type) {
+ Operand op;
+ memset(&op, 0, sizeof op);
+ op.kind = OPK_STACK;
+ op.cls = RC_INT;
+ op.type = type;
+ op.v.frame_slot = slot;
+ return op;
+}
+
+static FrameSlot forced_slot_new(Func *f, KitCgTypeId type) {
+ FrameSlotDesc desc;
+ memset(&desc, 0, sizeof desc);
+ desc.type = type;
+ desc.size = 4u;
+ desc.align = 4u;
+ desc.kind = FS_SPILL;
+ return ir_frame_slot_new(f, &desc);
+}
+
+static const NativeAllocClassInfo *int_class_info(const NativeTarget *native) {
+ if (!native || !native->regs) return NULL;
+ for (u32 i = 0; i < native->regs->nclasses; ++i)
+ if (native->regs->classes[i].cls == NATIVE_REG_INT)
+ return &native->regs->classes[i];
+ return NULL;
+}
+
+static void check_forced_scavenger(KitUnit *unit, const BackendCase *tc) {
+ KitTargetSpec spec = kit_unit_target(tc->arch, KIT_OS_LINUX, KIT_OBJ_ELF);
+ KitCompiler *kit_c = NULL;
+ Compiler *c;
+ ObjBuilder *obj = NULL;
+ MCEmitter *mc = NULL;
+ NativeTarget *native;
+ const NativeAllocClassInfo *ci;
+ ObjSecId text;
+ ObjSymId sym;
+ KitCgTypeId void_type, i32_type, fn_type;
+ KitCgFuncResult result;
+ KitCgFuncSig sig;
+ OptCGFuncDesc desc;
+ Func *f;
+ FrameSlot dst_slot, lhs_slot, rhs_slot;
+ Inst *in;
+ u32 block;
+ Reg emit_temp, scavenge_temp;
+
+ CU_CHECK(unit, unit->ctx.profiler != NULL,
+ "%s: profiler allocation failed", tc->name);
+ if (!unit->ctx.profiler) return;
+ spec.ptr_size = tc->ptr_size;
+ spec.ptr_align = tc->ptr_size;
+ CU_CHECK(unit, kit_unit_compiler_new(unit, spec, &kit_c) == KIT_OK && kit_c,
+ "%s: forced-scavenger compiler allocation failed", tc->name);
+ if (!kit_c) return;
+ c = (Compiler *)kit_c;
+ obj = obj_new(c);
+ CU_CHECK(unit, obj != NULL, "%s: forced-scavenger object allocation failed",
+ tc->name);
+ if (!obj) goto done;
+ mc = mc_new(c, obj);
+ CU_CHECK(unit, mc != NULL, "%s: forced-scavenger MC allocation failed",
+ tc->name);
+ if (!mc) goto done;
+ native = tc->ctor(c, obj, mc);
+ CU_CHECK(unit, native != NULL,
+ "%s: forced-scavenger native target allocation failed", tc->name);
+ if (!native) goto done;
+ ci = int_class_info(native);
+ CU_CHECK(unit, ci != NULL && ci->nemit_temps >= 2u,
+ "%s: forced-scavenger test needs two target-legal integer temps",
+ tc->name);
+ if (!ci || ci->nemit_temps < 2u) goto done;
+ emit_temp = ci->emit_temps[0];
+ scavenge_temp = ci->emit_temps[1];
+
+ text = obj_section(obj, pool_intern_slice(c->global, SLICE_LIT(".text")),
+ SEC_TEXT, SF_EXEC | SF_ALLOC, 16u);
+ sym = obj_symbol(obj,
+ pool_intern_slice(c->global,
+ SLICE_LIT("forced_native_scavenger")),
+ SB_LOCAL, SK_FUNC, text, 0u, 0u);
+ void_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID);
+ i32_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I32);
+ memset(&result, 0, sizeof result);
+ result.type = void_type;
+ memset(&sig, 0, sizeof sig);
+ sig.result = result;
+ sig.call_conv = KIT_CG_CC_TARGET_C;
+ fn_type = kit_cg_type_func(kit_c, sig);
+
+ memset(&desc, 0, sizeof desc);
+ desc.sym = sym;
+ desc.text_section_id = text;
+ desc.fn_type = fn_type;
+ desc.result_type = void_type;
+ f = ir_func_new(c, &desc);
+ block = ir_block_new(f);
+ f->entry = block;
+ ir_note_emit(f, block);
+ f->opt_rewritten = 1u;
+
+ /* Narrow the instruction bank to one register, then lend a second
+ * target-legal caller-saved register through the allocator list. An all-stack
+ * binary op must hold lhs and rhs simultaneously, deterministically entering
+ * the liveness-guarded scavenger before calling the real backend hook. */
+ f->emit_temp_regs[RC_INT][0] = emit_temp;
+ f->emit_temp_reg_count[RC_INT] = 1u;
+ f->opt_reserved_regs[RC_INT] = 1u << emit_temp;
+ f->opt_hard_regs[RC_INT][0] = scavenge_temp;
+ f->opt_hard_reg_count[RC_INT] = 1u;
+ f->opt_caller_saved[RC_INT] = 1u << scavenge_temp;
+
+ dst_slot = forced_slot_new(f, i32_type);
+ lhs_slot = forced_slot_new(f, i32_type);
+ rhs_slot = forced_slot_new(f, i32_type);
+ in = ir_emit(f, block, IR_BINOP);
+ in->nopnds = 3u;
+ in->opnds = arena_zarray(f->arena, Operand, in->nopnds);
+ in->opnds[0] = forced_stack_op(dst_slot, i32_type);
+ in->opnds[1] = forced_stack_op(lhs_slot, i32_type);
+ in->opnds[2] = forced_stack_op(rhs_slot, i32_type);
+ in->extra.imm = BO_IADD;
+
+ kit_profiler_reset(unit->ctx.profiler);
+ opt_emit_native(c, f, native);
+ CU_EXPECT(unit,
+ unit->ctx.profiler
+ ->counters[KIT_PROFILE_COUNTER_OPT_NATIVE_EMIT_SCAVENGES] >
+ 0u,
+ "%s: forced pressure did not enter native scavenger", tc->name);
+ CU_EXPECT(unit, mc_pos(mc) != 0u,
+ "%s: forced-scavenger backend emitted no code", tc->name);
+
+done:
+ if (mc) mc_free(mc);
+ if (obj) obj_free(obj);
+ kit_compiler_free(kit_c);
+}
+
int main(void) {
static const BackendCase cases[] = {
{KIT_ARCH_X86_64, "x64", x64_native_target_new, 0u, 8u},
@@ -757,10 +903,20 @@ int main(void) {
{KIT_ARCH_ARM_32, "arm32", arm32_native_target_new, 0u, 4u},
};
KitUnit unit;
+ KitProfiler *profiler;
kit_unit_init(&unit);
+ profiler = (KitProfiler *)malloc(sizeof *profiler);
+ CU_CHECK(&unit, profiler != NULL, "profiler allocation failed");
+ if (profiler) {
+ kit_profiler_reset(profiler);
+ unit.ctx.profiler = profiler;
+ }
check_location_address_roles(&unit);
- for (u32 i = 0; i < (u32)(sizeof cases / sizeof cases[0]); ++i)
+ for (u32 i = 0; i < (u32)(sizeof cases / sizeof cases[0]); ++i) {
check_typed_frame_address(&unit, &cases[i]);
+ check_forced_scavenger(&unit, &cases[i]);
+ }
kit_unit_summary(&unit, "frame-value-backends");
+ free(profiler);
return kit_unit_status(&unit);
}
diff --git a/test/opt/native_emit_frame_dst_test.c b/test/opt/native_emit_frame_dst_test.c
@@ -39,6 +39,7 @@ typedef struct MockNative {
u32 phase_callee_value;
u32 phase_arg_values[2];
u32 calls;
+ Reg machine_clobber_reg;
u8 report_bitfield_store_clobber;
u8 plan_two_arg_indirect_call;
u8 plan_two_part_ret;
@@ -236,7 +237,8 @@ static int mock_machine_op_clobbers(
if (!mock->report_bitfield_store_clobber ||
op->kind != NATIVE_MOP_BITFIELD_STORE)
return 0;
- mask[NATIVE_REG_INT] = 1u << 10u;
+ if (mock->machine_clobber_reg >= 32u) return 0;
+ mask[NATIVE_REG_INT] = 1u << mock->machine_clobber_reg;
return 1;
}
@@ -511,6 +513,7 @@ static void mock_init(MockNative *mock, KitUnit *unit, Compiler *c) {
mock->base.c = c;
mock->int_temps[0] = 8u;
mock->int_temps[1] = 9u;
+ mock->machine_clobber_reg = 10u;
mock->classes[NATIVE_REG_INT].cls = NATIVE_REG_INT;
mock->classes[NATIVE_REG_INT].scratch = mock->int_temps;
mock->classes[NATIVE_REG_INT].nscratch = 2u;
@@ -1513,6 +1516,123 @@ static void allocator_cache_waits_for_hard_liveness(KitUnit *unit) {
kit_compiler_free(kit_c);
}
+static void reserved_cache_obeys_machine_clobbers(KitUnit *unit) {
+ KitCompiler *kit_c = NULL;
+ Compiler *c;
+ KitCgTypeId i64_type, ptr_type;
+ OptCGFuncDesc desc;
+ Func *f;
+ MockNative mock;
+ FrameSlot clobbered_slot, preserved_slot, record_slot;
+ Inst *in, *clobber;
+ NativeMachineOp mop;
+ u32 clobber_mask[NATIVE_CALL_PLAN_CLASSES];
+ u32 block;
+
+ CU_EXPECT(unit,
+ kit_unit_compiler_new(
+ unit,
+ kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF),
+ &kit_c) == KIT_OK &&
+ kit_c != NULL,
+ "compiler allocation failed for reserved-cache clobber test");
+ if (!kit_c) return;
+ c = (Compiler *)kit_c;
+ i64_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_I64);
+ ptr_type = kit_cg_type_ptr(
+ kit_c, kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID), 0);
+
+ memset(&desc, 0, sizeof desc);
+ desc.result_type = kit_cg_type_builtin(kit_c, KIT_CG_BUILTIN_VOID);
+ f = ir_func_new(c, &desc);
+ block = ir_block_new(f);
+ f->entry = block;
+ ir_note_emit(f, block);
+ f->opt_rewritten = 1;
+ f->emit_temp_regs[RC_INT][0] = 8u;
+ f->emit_temp_regs[RC_INT][1] = 9u;
+ f->emit_temp_reg_count[RC_INT] = 2u;
+ f->opt_reserved_regs[RC_INT] = (1u << 8u) | (1u << 9u);
+ clobbered_slot = add_slot(f, i64_type, 8u, 8u);
+ preserved_slot = add_slot(f, i64_type, 8u, 8u);
+ record_slot = add_slot(f, i64_type, 8u, 8u);
+
+ /* The stack destination is staged through r8, then retained because r8 is
+ * in the target's explicit emit_cache_mask. The following machine hook says
+ * it overwrites r8. This happens on the lazy fast path, before the scavenger
+ * has requested hard-liveness, and must still invalidate the cache fact. */
+ in = emit_with_ops(f, block, IR_LOAD_IMM, 1u);
+ in->opnds[0] = stack_op(clobbered_slot, i64_type);
+ in->extra.imm = 41;
+
+ clobber = emit_with_ops(f, block, IR_BITFIELD_STORE, 2u);
+ clobber->opnds[0] = frame_addr_op(record_slot, ptr_type);
+ clobber->opnds[1] = imm_op(7, i64_type);
+ {
+ IRBitFieldAux *aux = arena_znew(f->arena, IRBitFieldAux);
+ aux->access.field_type = i64_type;
+ aux->access.storage.type = i64_type;
+ aux->access.storage.size = 8u;
+ aux->access.storage.align = 8u;
+ aux->access.bit_offset = 3u;
+ aux->access.bit_width = 11u;
+ clobber->extra.aux = aux;
+ }
+
+ in = emit_with_ops(f, block, IR_COPY, 2u);
+ in->opnds[0] = reg_op(10u, i64_type);
+ in->opnds[1] = stack_op(clobbered_slot, i64_type);
+
+ /* Establish a second r8 cache fact and cross a hook with no r8 effect. The
+ * reload should forward from r8, proving the focused invalidation above did
+ * not degenerate into clearing every cache entry on every instruction. */
+ in = emit_with_ops(f, block, IR_LOAD_IMM, 1u);
+ in->opnds[0] = stack_op(preserved_slot, i64_type);
+ in->extra.imm = 99;
+
+ in = emit_with_ops(f, block, IR_LOAD_LABEL_ADDR, 1u);
+ in->opnds[0] = reg_op(11u, ptr_type);
+ in->extra.imm = block;
+
+ in = emit_with_ops(f, block, IR_COPY, 2u);
+ in->opnds[0] = reg_op(12u, i64_type);
+ in->opnds[1] = stack_op(preserved_slot, i64_type);
+
+ mock_init(&mock, unit, c);
+ mock.report_bitfield_store_clobber = 1u;
+ mock.machine_clobber_reg = 8u;
+ memset(&mop, 0, sizeof mop);
+ mop.kind = NATIVE_MOP_BITFIELD_STORE;
+ CU_EXPECT(unit,
+ mock.base.machine_op_clobbers(&mock.base, &mop, clobber_mask) &&
+ clobber_mask[NATIVE_REG_INT] == (1u << 8u),
+ "mock bitfield-store effect must report reserved cache r8");
+ f->inst_clobbers_cap = f->next_inst_id;
+ f->inst_clobbers =
+ arena_zarray(f->arena, OptInstClobberMask, f->inst_clobbers_cap);
+ CU_EXPECT(unit,
+ clobber->id != INST_ID_NONE &&
+ clobber->id < f->inst_clobbers_cap,
+ "bitfield store must have a valid clobber-table instruction id");
+ if (clobber->id != INST_ID_NONE && clobber->id < f->inst_clobbers_cap)
+ for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls)
+ f->inst_clobbers[clobber->id][cls] = clobber_mask[cls];
+
+ opt_emit_native(c, f, &mock.base);
+
+ CU_EXPECT(unit,
+ (mock.callback_mask &
+ (SAW_BITFIELD_STORE | SAW_LABEL_ADDR)) ==
+ (SAW_BITFIELD_STORE | SAW_LABEL_ADDR),
+ "clobbering and preserving hooks must both execute");
+ CU_EXPECT(unit, mock.frame_loads == 1u && mock.loads == 1u,
+ "reserved r8 clobber must reload only the affected frame value "
+ "(loads=%u frame-loads=%u)",
+ mock.loads, mock.frame_loads);
+
+ kit_compiler_free(kit_c);
+}
+
static void call_ends_pre_call_temp_phase(KitUnit *unit) {
KitCompiler *kit_c = NULL;
Compiler *c;
@@ -1794,6 +1914,7 @@ int main(void) {
frame_atomic_load_reuses_scoped_address_base(&unit);
atomic_store_collapses_address_for_value_temp(&unit);
allocator_cache_waits_for_hard_liveness(&unit);
+ reserved_cache_obeys_machine_clobbers(&unit);
call_ends_pre_call_temp_phase(&unit);
return_parts_keep_prior_abi_destinations_live(&unit);
zero_spill_uses_architectural_zero_register(&unit);
diff --git a/test/opt/reg_effects_test.c b/test/opt/reg_effects_test.c
@@ -33,6 +33,29 @@ static Operand indirect_op(Reg base, Reg index) {
static u32 reg_bit(Reg reg) { return 1u << reg; }
+static void expect_mask_effects_match(KitUnit* u, Func* f, Inst* in,
+ const char* what) {
+ OptRegEffects detailed;
+ OptRegEffectMasks masks;
+ OptHardRegSet detailed_kills;
+ OptHardRegSet mask_kills;
+ OptHardRegSet direct_kills;
+ opt_inst_reg_effects(f, in, &detailed);
+ opt_inst_reg_effect_masks(f, in, &masks);
+ opt_reg_effect_kills(&detailed, &detailed_kills);
+ opt_reg_effect_mask_kills(&masks, &mask_kills);
+ opt_inst_reg_kills(f, in, &direct_kills);
+ CU_EXPECT(u,
+ !memcmp(&detailed.uses, &masks.uses, sizeof masks.uses) &&
+ !memcmp(&detailed.defs, &masks.defs, sizeof masks.defs) &&
+ !memcmp(&detailed.clobbers, &masks.clobbers,
+ sizeof masks.clobbers) &&
+ !memcmp(&detailed_kills, &mask_kills, sizeof mask_kills) &&
+ !memcmp(&detailed_kills, &direct_kills,
+ sizeof direct_kills),
+ "%s masks-only effects diverged from detailed effects", what);
+}
+
typedef struct MachinizeRunCtx {
Func* f;
NativeTarget* target;
@@ -234,6 +257,7 @@ static void explicit_and_machine_effects(KitUnit* u) {
opnds[2] = indirect_op(8, 8);
machine_clobbers[in.id][RC_INT] = reg_bit(5);
+ expect_mask_effects_match(u, &f, &in, "explicit/machine");
opt_inst_reg_effects(&f, &in, &effects);
CU_EXPECT(u, effects.uses.cls[RC_INT] ==
(reg_bit(2) | reg_bit(8)),
@@ -307,6 +331,7 @@ static void call_and_asm_effects(KitUnit* u) {
arg.storage = reg_op(RC_INT, 11);
call_aux.desc.ret.storage = reg_op(RC_INT, 0);
+ expect_mask_effects_match(u, &f, &call, "call");
opt_inst_reg_effects(&f, &call, &effects);
CU_EXPECT(u, effects.uses.cls[RC_INT] ==
(reg_bit(10) | reg_bit(11)),
@@ -340,6 +365,7 @@ static void call_and_asm_effects(KitUnit* u) {
asm_aux.nout = 1;
asm_aux.clobber_mask[RC_INT] = reg_bit(4);
+ expect_mask_effects_match(u, &f, &block, "asm");
opt_inst_reg_effects(&f, &block, &effects);
CU_EXPECT(u, effects.uses.cls[RC_INT] == reg_bit(2),
"asm uses mismatch: %#x", effects.uses.cls[RC_INT]);
diff --git a/test/opt/whole_program_inline.sh b/test/opt/whole_program_inline.sh
@@ -92,7 +92,8 @@ static int add1(int x) { return x + 1; }
int main(void) { return add1(41) == 42 ? 0 : 1; }
EOF
printf '%s\n' "$RUN_SRC" > "$WORK/run.c"
-if ! "$KIT" run --time -O1 "$WORK/run.c" >"$WORK/run.out" 2>"$WORK/run.err"; then
+if ! KIT_METRICS=1 "$KIT" run --time -O1 "$WORK/run.c" \
+ >"$WORK/run.out" 2>"$WORK/run.err"; then
printf 'whole-program-inline FAILED: `kit run -O1` did not exit 0\n' >&2
sed 's/^/ | /' "$WORK/run.err" >&2
exit 1
