commit 599588c5956838f1644b0c511e60cd2d270091cd
parent 1e34b851ae4ce33c8520189942c885db7bef59dd
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Wed, 17 Jun 2026 00:41:56 -0700
arm32: close the residual -O0/-O1 edge reds
Four correct-by-construction fixes to the backend, retiring the toy reds the
ARM32.md remaining-work list enumerated (112/123/124/140/154):
- Atomic CAS/RMW register budget (124/-O1): the STREX status + RMW new-value
temps were fixed r4/r5, which the -O1 allocator can hand to an operand.
arm_atomic_borrow now picks callee-saved temps operand-aware (clear of the
base + all value operands), push/pop-balanced — the §2 'operand-aware scratch
budget'.
- Indirect call + large outgoing stack window (112/123/-O0): the callee held in
LR was clobbered by large-offset arg-stage address materialization (LR is the
address scratch). Spill the callee to a frame home before arg staging when the
call has stack args, reload into ip just before BLX. -O1 (optimizer-allocated
callee) keeps the register form; its frame is final, so no body-time slot.
- fp-at-pair prologue (154/-O1, + backtraces): the single combined PUSH left r7
pointing at the lowest callee-save, so @frame_address/@return_address walked a
broken chain whenever r8..r11 sat between r7 and lr. Split the push so callee-
saves ride ABOVE a tight {r7,lr} pair; r7 anchors the frame record. Total
block size is unchanged, so saved_block_bytes and every [r7,#-off] body slot
and the va_start GP-save offset are preserved; epilogue peels the pair then
the callee-saves.
- Scaled-index memory resolve: arm_resolve_mem ignored a NativeAddr index, so a
base+index<<scale load/store accessed [base]. Materialize the whole effective
address once into a reserved scratch (no LDRD indexed form; one base shared by
both wide lanes), the §1 single-base guarantee for the high lane.
Diffstat:
1 file changed, 186 insertions(+), 40 deletions(-)
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -78,6 +78,12 @@ typedef struct Arm32NativeTarget {
u8 has_sret;
u8 is_variadic;
NativeFrameSlot sret_ptr_slot;
+ /* Lazily-allocated 4-byte home for an indirect call target when the call has
+ * outgoing stack args. At -O0 the two emit scratch (IP, LR) are both consumed
+ * staging stack args (IP = arg value, LR = large-offset address), so the callee
+ * cannot survive in a scratch register across staging: it is spilled here and
+ * reloaded just before BLX. NATIVE_FRAME_SLOT_NONE until first needed. */
+ NativeFrameSlot callee_spill_slot;
u32 func_start;
u32 prologue_sub_pos; /* file offset of the deferred SUB sp (patched at end) */
MCLabel epilogue_label;
@@ -441,6 +447,39 @@ static void arm_emit_load_u32(Arm32NativeTarget* a, u32 rd, u32 v) {
static ArmMemAddr arm_resolve_mem(Arm32NativeTarget* a, const NativeAddr* addr,
u32 rt, u32 span, u32 pinned) {
ArmMemAddr r;
+ /* Scaled-index address (base + offset + index<<scale): materialize the whole
+ * effective address once into a reserved scratch and return (scratch, 0).
+ * arm32 has no LDRD/STRD indexed form, so a wide (8-byte) indexed access must
+ * share ONE computed base across both lanes; doing it here keeps every base
+ * kind (frame / reg / global) uniform and is the §1 single-base guarantee for
+ * the high lane. The scratch aliases neither the transfer register(s) — rt and
+ * `pinned` — nor the index register. */
+ if (addr->index_kind == NATIVE_ADDR_INDEX_REG) {
+ MCEmitter* mc = a->base.mc;
+ u32 idx = addr->index.reg & 0xfu;
+ u32 s = arm_pick_scratch(a, rt, pinned | (1u << idx));
+ switch (addr->base_kind) {
+ case NATIVE_ADDR_BASE_FRAME: {
+ NativeFrameSlotEntry* e =
+ native_frame_slot_at(&a->frame, addr->base.frame);
+ arm_emit_base_off(a, s, ARM_FP, -(i32)e->off + addr->offset);
+ break;
+ }
+ case NATIVE_ADDR_BASE_REG:
+ arm_emit_base_off(a, s, addr->base.reg & 0xfu, addr->offset);
+ break;
+ case NATIVE_ADDR_BASE_GLOBAL:
+ arm_emit_global_addr(a, s, addr->base.global.sym,
+ addr->base.global.addend + addr->offset);
+ break;
+ default:
+ arm_panic(a, "unsupported indexed addressing mode");
+ }
+ arm_emit_t32(mc, arm_add_reg_lsl(s, s, idx, addr->log2_scale));
+ r.base = s;
+ r.off = 0;
+ return r;
+ }
switch (addr->base_kind) {
case NATIVE_ADDR_BASE_FRAME: {
NativeFrameSlotEntry* s =
@@ -1000,6 +1039,23 @@ static void arm_reserve_sret_slot(Arm32NativeTarget* a) {
}
}
+/* Lazily reserve the per-function indirect-callee spill home (4 bytes, r7-
+ * anchored). Allocated on first need during body codegen — at -O0 the frame is
+ * deferred (settled in arm_func_end), so a body-time slot just grows cum_off;
+ * one slot is reused across every indirect call in the function. */
+static NativeFrameSlot arm_callee_spill_slot(Arm32NativeTarget* a) {
+ if (a->callee_spill_slot == NATIVE_FRAME_SLOT_NONE) {
+ NativeFrameSlotDesc sd;
+ memset(&sd, 0, sizeof sd);
+ sd.type = builtin_id(KIT_CG_BUILTIN_I32);
+ sd.size = 4;
+ sd.align = 4;
+ sd.kind = NATIVE_FRAME_SLOT_SAVE;
+ a->callee_spill_slot = arm_frame_slot(&a->base, &sd);
+ }
+ return a->callee_spill_slot;
+}
+
/* Spill the incoming sret pointer (r0) to its hidden home. The home is
* r7-anchored, so this runs correctly after the frame is set up. */
static void arm_emit_sret_store(Arm32NativeTarget* a) {
@@ -1028,6 +1084,7 @@ static void arm_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
+ a->callee_spill_slot = NATIVE_FRAME_SLOT_NONE;
a->known_frame = 0;
a->slim_prologue = 0;
a->saved_reglist = (u16)((1u << ARM_FP) | (1u << 14u)); /* {r7, lr} */
@@ -1092,6 +1149,7 @@ static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
+ a->callee_spill_slot = NATIVE_FRAME_SLOT_NONE;
a->known_frame = 1;
a->slim_prologue = 0;
a->n_alloca_patch = 0;
@@ -1153,14 +1211,23 @@ static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
return;
}
- /* Emit the final prologue. A variadic callee first spills the GP arg registers
- * r0..r3 into a save area just above the saved pair (so they are contiguous
- * below the named incoming stack args); va_start/va_arg walk forward across
- * both. The frame teardown reclaims this 16-byte area (arm_emit_frame_restore).
- * (The single-pass path does the same in arm_func_begin.) */
+ /* Emit the final prologue as a fp-at-pair split push: the callee-saves ride
+ * ABOVE a tight {r7, lr} frame-record pair so r7 anchors exactly that pair —
+ * [r7]=caller's r7, [r7+4]=return address — which @frame_address/@return_address
+ * and any frame-pointer backtrace walk. (A single combined PUSH would leave r7
+ * pointing at the lowest callee-save, with the return address an arbitrary
+ * distance above, breaking the chain whenever r8..r11 sit between r7 and lr.)
+ * Push order, high address to low: variadic GP-save (r0..r3) first, then the
+ * callee-saves, then the pair; the total block size is unchanged, so
+ * saved_block_bytes and every [r7, #-off] body slot keep their offsets, and
+ * va_start still reaches the GP-save at r7 + saved_block_bytes.
+ * A variadic callee's r0..r3 save area sits just above the saved block (below
+ * the named incoming stack args); va_start/va_arg walk forward across both, and
+ * the teardown reclaims it (arm_emit_frame_restore). */
if (a->is_variadic) arm_emit_t32(mc, arm_push_w(0xfu)); /* PUSH {r0-r3} */
- arm_emit_t32(mc, arm_push_w(reglist));
- arm_emit_t16(mc, arm_mov_hi(ARM_FP, 13u)); /* MOV r7, sp */
+ if (cs_mask) arm_emit_t32(mc, arm_push_w(cs_mask)); /* PUSH {callee-saves} */
+ arm_emit_t32(mc, arm_push_w((1u << ARM_FP) | (1u << 14u))); /* PUSH {r7, lr} */
+ arm_emit_t16(mc, arm_mov_hi(ARM_FP, 13u)); /* MOV r7, sp */
frame = arm_frame_size(a);
arm_emit_sub_sp(a, frame);
@@ -1168,23 +1235,28 @@ static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
native_frame_set_final(&a->frame);
}
-/* Emit the frame teardown. With to_pc, return — the saved lr pops straight into
- * pc, the size-independent common case (MOV sp,r7 drops any locals frame). Else
- * restore the save set with lr intact and fall through with lr = the caller's
- * return address, for a tail-call branch. Both reclaim a variadic callee's
- * r0..r3 GP-save area (16 bytes above the saved block) so sp returns to entry. */
+/* Emit the frame teardown, mirroring the fp-at-pair split prologue: r7 anchors
+ * the {r7, lr} pair at the bottom of the saved block, the callee-saves sit above
+ * it, and (for a variadic callee) the r0..r3 GP-save above those.
+ *
+ * The common case — no callee-saves, no GP-save, returning — peels the pair with
+ * a single `POP {r7, pc}` (the saved lr lands straight in pc). Otherwise restore
+ * the pair with `POP {r7, lr}`, then peel the callee-saves and reclaim the
+ * GP-save, leaving lr = the caller's return address — popped into pc with a final
+ * `BX lr` when returning, or kept for a tail-call branch when not. */
static void arm_emit_frame_restore(Arm32NativeTarget* a, int to_pc) {
MCEmitter* mc = a->base.mc;
u32 va = a->is_variadic ? ARM_VA_GP_SAVE_BYTES : 0u;
- arm_emit_t16(mc, arm_mov_hi(13u, ARM_FP)); /* MOV sp, r7 */
- if (to_pc && va == 0u) {
- u32 poplist = (a->saved_reglist & ~(1u << 14u)) | (1u << 15u);
- arm_emit_t32(mc, arm_pop_w(poplist)); /* POP {.., r7, pc} */
+ u32 cs_mask = a->saved_reglist & ~((1u << ARM_FP) | (1u << 14u));
+ arm_emit_t16(mc, arm_mov_hi(13u, ARM_FP)); /* MOV sp, r7 (sp -> {r7,lr} pair) */
+ if (to_pc && va == 0u && cs_mask == 0u) {
+ arm_emit_t32(mc, arm_pop_w((1u << ARM_FP) | (1u << 15u))); /* POP {r7, pc} */
return;
}
- arm_emit_t32(mc, arm_pop_w(a->saved_reglist)); /* POP {.., r7, lr} */
- if (va) arm_emit_add_const(a, 13u, (i64)va); /* reclaim r0..r3 save area */
- if (to_pc) arm_emit_t16(mc, arm_bx(14u)); /* BX lr (return) */
+ arm_emit_t32(mc, arm_pop_w((1u << ARM_FP) | (1u << 14u))); /* POP {r7, lr} */
+ if (cs_mask) arm_emit_t32(mc, arm_pop_w(cs_mask)); /* POP {callee-saves} */
+ if (va) arm_emit_add_const(a, 13u, (i64)va); /* reclaim r0..r3 save */
+ if (to_pc) arm_emit_t16(mc, arm_bx(14u)); /* BX lr (return) */
}
static void arm_func_end(NativeTarget* t) {
@@ -1624,16 +1696,37 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
* POP of r0..r11. A tail call holds it in IP (never popped) and stages args via
* LR; a normal call stages args via IP, so an IP- or arg-register-resident
* callee moves to LR (BLX lr is valid). A callee already in a preserved
- * register (r4..r11, or the chosen staging reg) needs no move. */
+ * register (r4..r11, or the chosen staging reg) needs no move.
+ *
+ * The exception is the -O0 (single-pass) call with outgoing stack args: staging
+ * those args consumes BOTH emit scratch (IP for the value, LR for a large-offset
+ * store address), so the callee cannot survive in LR. Spill it to its frame
+ * home now — before any arg move clobbers its register — and reload just before
+ * BLX (arm_emit_call). The known-frame (-O1) path keeps the register form: its
+ * optimizer-allocated callee already survives, and its frame is final (no body-
+ * time slot allocation). */
if (plan->callee.kind == NATIVE_LOC_REG &&
(NativeAllocClass)plan->callee.cls == NATIVE_REG_INT) {
u32 cr = plan->callee.v.reg & 0xfu;
- u32 dst = tail ? ARM_SCRATCH : ARM_TMP;
- int needs = tail ? 1 : (cr <= 3u || cr == arg_stage);
- if (needs && cr != dst) {
- NativeLoc scratch = native_loc_reg(plan->callee.type, NATIVE_REG_INT, dst);
- arm_move(t, scratch, plan->callee);
- plan->callee = scratch;
+ if (!tail && !a->known_frame && plan->stack_arg_size > 0u) {
+ NativeFrameSlot slot = arm_callee_spill_slot(a);
+ NativeAddr home;
+ memset(&home, 0, sizeof home);
+ home.base_kind = NATIVE_ADDR_BASE_FRAME;
+ home.base.frame = slot;
+ home.base_type = plan->callee.type;
+ arm_emit_mem(a, 0, plan->callee, home,
+ native_mem_for_type(t, plan->callee.type, 4));
+ plan->callee = native_loc_stack(plan->callee.type, slot, 0);
+ } else {
+ u32 dst = tail ? ARM_SCRATCH : ARM_TMP;
+ int needs = tail ? 1 : (cr <= 3u || cr == arg_stage);
+ if (needs && cr != dst) {
+ NativeLoc scratch =
+ native_loc_reg(plan->callee.type, NATIVE_REG_INT, dst);
+ arm_move(t, scratch, plan->callee);
+ plan->callee = scratch;
+ }
}
}
{
@@ -1771,6 +1864,20 @@ static void arm_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
arm_emit_t16(mc, arm_blx_reg(loc_reg(plan->callee)));
return;
}
+ if (plan->callee.kind == NATIVE_LOC_STACK) {
+ /* -O0 indirect call with stack args: the callee was spilled to a frame home
+ * (arm_plan_call) to free the emit scratch for arg staging. Arg staging is
+ * done now, so IP is free; reload the target and BLX it. */
+ NativeLoc tmp = native_loc_reg(plan->callee.type, NATIVE_REG_INT, ARM_SCRATCH);
+ NativeAddr home;
+ memset(&home, 0, sizeof home);
+ home.base_kind = NATIVE_ADDR_BASE_FRAME;
+ home.base.frame = plan->callee.v.stack.slot;
+ home.base_type = plan->callee.type;
+ arm_emit_mem(a, 1, tmp, home, native_mem_for_type(t, plan->callee.type, 4));
+ arm_emit_t16(mc, arm_blx_reg(ARM_SCRATCH));
+ return;
+ }
arm_panic(a, "unsupported call target");
}
@@ -2174,9 +2281,35 @@ static void arm_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
* value operands in the r0..r3 pool OR in ip (ARM_SCRATCH, its materialization
* scratch), so the atomic sequences must NOT use ip as a temp — they would
* clobber an operand that happens to live there. STREX additionally needs its
- * status, value and base registers mutually distinct. The rmw/cas loops
- * therefore borrow callee-saved registers (r4/r5, free at -O0, push/pop-balanced
- * so safe at -O1) for the private temporaries (new value + STREX status). */
+ * status, value and base registers mutually distinct.
+ *
+ * The rmw/cas loops therefore borrow callee-saved registers for the private
+ * temporaries (new value + STREX status). At -O1 the optimizer freely allocates
+ * the operands (prior/expected/desired/ok, or rmw dst/val) and the address base
+ * across the whole callee-saved set, so a fixed r4/r5 would alias an operand. The
+ * borrow is therefore OPERAND-AWARE (arm_atomic_borrow): it picks free callee-
+ * saved registers clear of the live operand+base set, guaranteeing base + all
+ * operands + the STREX temporaries are mutually distinct (ARM32.md §2 "operand-
+ * aware scratch budget"). The picked registers are push/pop-balanced so the
+ * caller's callee-saved values are preserved (free at -O0, safe at -O1). */
+
+/* Pick `n` distinct callee-saved temporaries (r4,r5,r6,r8,r9,r10,r11 — fp/r7 is
+ * reserved) clear of the `used` register set, for the atomic LDREX/STREX private
+ * temporaries. Fills regs[0..n) and returns their push/pop mask in *mask. The
+ * pool has 7 registers; the largest live set (rmw: dst+val+base = 3, cas:
+ * 4 operands + base = 5) always leaves enough free for n (2 / 1 respectively). */
+static void arm_atomic_borrow(Arm32NativeTarget* a, u32 used, u32 n, u32* regs,
+ u32* mask) {
+ static const u32 pool[] = {4u, 5u, 6u, 8u, 9u, 10u, 11u};
+ u32 got = 0u, m = 0u, i;
+ for (i = 0; i < sizeof pool / sizeof pool[0] && got < n; ++i) {
+ if (used & (1u << pool[i])) continue;
+ regs[got++] = pool[i];
+ m |= 1u << pool[i];
+ }
+ if (got < n) arm_panic(a, "atomic: no free scratch register");
+ *mask = m;
+}
static int arm_order_acquire(KitCgMemOrder o) {
return o == KIT_CG_MO_CONSUME || o == KIT_CG_MO_ACQUIRE ||
@@ -2270,14 +2403,19 @@ static void arm_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
u32 rd = loc_reg(dst);
u32 rv = loc_reg(val);
- u32 base, newv = 4u, status = 5u; /* r4 = new value, r5 = STREX status */
+ u32 base, newv, status, temps[2], pushmask;
MCLabel retry = mc_label_new(mc);
if (sz > 4u) arm_panic(a, "8-byte atomic rmw not lowered (spinlock libcall)");
- /* Borrow r4 (new value) + r5 (STREX status) across a balanced push/pop so the
- * sequence touches no NDT-allocated register beyond lr=addr, rd=old value
- * (returned), rv=operand. ip is left alone (an operand may live there). */
- arm_emit_t32(mc, arm_push_w((1u << newv) | (1u << status)));
- base = arm_atomic_addr_reg(a, addr); /* lr */
+ /* Borrow a new-value temp + a STREX-status temp clear of {addr base, rd, rv}
+ * across a balanced push/pop, so the sequence touches no live operand. ip is
+ * left alone (an operand may live there) and the operand-aware pick avoids the
+ * base/operand registers the optimizer chose at -O1. */
+ base = arm_atomic_addr_reg(a, addr); /* lr (-O0) or the pointer's own reg */
+ arm_atomic_borrow(a, (1u << rd) | (1u << rv) | (1u << base), 2u, temps,
+ &pushmask);
+ newv = temps[0];
+ status = temps[1];
+ arm_emit_t32(mc, arm_push_w(pushmask));
if (arm_order_release(order)) arm_emit_t32(mc, arm_dmb(0xfu));
/* Retry loop: rd = *base (LDREX); r4 = rd OP val; STREX r5,r4,[base]; retry
* while r5 != 0. */
@@ -2314,7 +2452,7 @@ static void arm_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
arm_emit_t32(mc, arm_b_cond_w(ARM_CC_NE)); /* BNE retry */
mc_emit_label_ref(mc, retry, R_ARM_THM_JUMP19, 4, 0);
if (arm_order_acquire(order)) arm_emit_t32(mc, arm_dmb(0xfu));
- arm_emit_t32(mc, arm_pop_w((1u << newv) | (1u << status)));
+ arm_emit_t32(mc, arm_pop_w(pushmask));
}
static void arm_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
@@ -2328,7 +2466,7 @@ static void arm_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
u32 rexp = loc_reg(expected);
u32 rdes = loc_reg(desired);
u32 rok = loc_reg(ok);
- u32 base, status = 4u; /* r4 = STREX status (never ip — an operand may live there) */
+ u32 base, status, statusv[1], pushmask;
u32 enc0, enc1;
MCLabel retry = mc_label_new(mc);
MCLabel fail = mc_label_new(mc);
@@ -2337,8 +2475,16 @@ static void arm_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
if (sz > 4u) arm_panic(a, "8-byte atomic cas not lowered (spinlock libcall)");
thumb_expand_imm_encode(0u, &enc0);
thumb_expand_imm_encode(1u, &enc1);
- arm_emit_t32(mc, arm_push_w(1u << status));
- base = arm_atomic_addr_reg(a, addr); /* lr */
+ /* STREX status temp clear of {addr base, prior, expected, desired, ok} —
+ * never ip (an operand may live there), and operand-aware so the optimizer's
+ * -O1 register choices for the five live values never alias it. */
+ base = arm_atomic_addr_reg(a, addr); /* lr (-O0) or the pointer's own reg */
+ arm_atomic_borrow(a,
+ (1u << rprior) | (1u << rexp) | (1u << rdes) | (1u << rok) |
+ (1u << base),
+ 1u, statusv, &pushmask);
+ status = statusv[0];
+ arm_emit_t32(mc, arm_push_w(pushmask));
if (arm_order_release(success)) arm_emit_t32(mc, arm_dmb(0xfu));
/* retry: prior = *base (LDREX); if prior != expected goto fail; STREX
* r4,desired,[base]; retry on failure; ok = 1; goto done. fail: clear the
@@ -2360,7 +2506,7 @@ static void arm_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
arm_emit_t32(mc, arm_mov_imm(rok, enc0)); /* ok = 0 */
mc_label_place(mc, done);
if (arm_order_acquire(success)) arm_emit_t32(mc, arm_dmb(0xfu));
- arm_emit_t32(mc, arm_pop_w(1u << status));
+ arm_emit_t32(mc, arm_pop_w(pushmask));
}
static void arm_fence(NativeTarget* t, KitCgMemOrder order) {
