commit 07dcaf160e8114e93708a51433fca6e5157ee891
parent 034c337916199487593e4e99dfddfed77e66cc6d
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 17:41:19 -0700
arm32 Phase 2: -O1 known-frame prologue/epilogue + large-frame IP staging
Diffstat:
2 files changed, 277 insertions(+), 37 deletions(-)
diff --git a/src/arch/arm32/isa.h b/src/arch/arm32/isa.h
@@ -244,6 +244,8 @@ static inline u16 arm_bx(u32 rm) { return (u16)(0x4700u | (rm << 3)); }
static inline u16 arm_blx_reg(u32 rm) { return (u16)(0x4780u | (rm << 3)); }
static inline u16 arm_bkpt(u32 imm8) { return (u16)(0xbe00u | (imm8 & 0xffu)); }
static inline u16 arm_nop16(void) { return (u16)0xbf00u; }
+/* NOP.W (32-bit) — pads a multi-slot prologue placeholder region. */
+static inline u32 arm_nop32(void) { return arm_t32(0xf3afu, 0x8000u); }
/* --------- push / pop (32-bit, STMDB sp! / LDMIA sp!) --------- */
static inline u32 arm_push_w(u32 reglist) { return arm_t32(0xe92du, reglist & 0xdfffu); }
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -49,6 +49,10 @@ extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
* register between calls is free and cannot collide with an NDT operand in IP. */
#define ARM_TMP 14u
#define ARM_MAX_REG_ARG_MOVES 16u
+/* Worst-case T32 instruction slots the single-pass deferred-`SUB sp` region must
+ * hold: an IP-staged large frame is MOVW + MOVT + SUB sp,sp,ip = 3 words. A
+ * small frame patches the first slot with one SUBW and NOP-fills the rest. */
+#define ARM_NDT_SUB_WORDS 3u
/* ============================ state ============================ */
@@ -66,6 +70,15 @@ typedef struct Arm32NativeTarget {
u32 func_start;
u32 prologue_sub_pos; /* file offset of the deferred SUB sp (patched at end) */
MCLabel epilogue_label;
+ /* Known-frame (-O1) state. On the known-frame path the optimizer hands us the
+ * exact frame up front, so the prologue is emitted final (no deferred SUB to
+ * patch in arm_func_end). saved_reglist is the PUSH/POP register set (callee-
+ * saves + r7 + lr); saved_block_bytes is its size in bytes, which incoming
+ * stack args sit above (so bind_param adds it instead of the bare 8). */
+ u8 known_frame;
+ u8 slim_prologue; /* leaf, no frame: emit BX lr, no PUSH/anchor */
+ u16 saved_reglist; /* registers PUSHed by the prologue (POP restores via pc) */
+ u32 saved_block_bytes;
} Arm32NativeTarget;
static Arm32NativeTarget* arm_of(NativeTarget* t) {
@@ -327,6 +340,24 @@ static u32 arm_addr_base(Arm32NativeTarget* a, const NativeAddr* addr,
}
}
+/* Materialize the unsigned 32-bit constant `v` into `rd` (MOVW + optional MOVT).
+ * Used to stage large frame/memory offsets and SUB-sp amounts that exceed the
+ * inline immediate forms. */
+static void arm_emit_load_u32(Arm32NativeTarget* a, u32 rd, u32 v) {
+ MCEmitter* mc = a->base.mc;
+ u32 enc;
+ if (thumb_expand_imm_encode(v, &enc)) {
+ arm_emit_t32(mc, arm_mov_imm(rd, enc));
+ return;
+ }
+ if (thumb_expand_imm_encode(~v, &enc)) {
+ arm_emit_t32(mc, arm_mvn_imm(rd, enc));
+ return;
+ }
+ arm_emit_t32(mc, arm_movw(rd, v & 0xffffu));
+ if ((v >> 16) != 0u) arm_emit_t32(mc, arm_movt(rd, (v >> 16) & 0xffffu));
+}
+
/* Emit a load (is_load=1) or store of `reg` to [base, #off], dispatching the
* width from mem.size. Loads zero-extend (LDR/LDRB/LDRH); signed narrowing is a
* separate convert. */
@@ -357,7 +388,18 @@ static void arm_emit_mem(Arm32NativeTarget* a, int is_load, NativeLoc reg,
} else if (off >= 0 && off <= 255) {
arm_emit_t32(mc, arm_ldst_t4(t4hw1, rt, base, (u32)off, 1u));
} else {
- arm_panic(a, "frame/memory offset out of range (needs IP fallback)");
+ /* Large frame/memory offset (-O1 big frames): stage base+off into ARM_TMP
+ * (lr) and access [lr]. lr is reserved (never an NDT operand) and dead in
+ * the body, so it can never alias the transfer reg rt — including the
+ * rt==IP stack-arg path. base is a frame/general register here, never
+ * ARM_TMP: a GLOBAL base (the only ARM_TMP producer) folds its full byte
+ * offset into the MOVW/MOVT reloc addend, so it always reaches arm_emit_mem
+ * with off in range and never takes this branch. */
+ if (base == ARM_TMP) arm_panic(a, "large offset on lr-staged base");
+ arm_emit_load_u32(a, ARM_TMP, (u32)(off < 0 ? -off : off));
+ arm_emit_t32(mc, off < 0 ? arm_sub_reg(ARM_TMP, base, ARM_TMP)
+ : arm_add_reg(ARM_TMP, base, ARM_TMP));
+ arm_emit_t32(mc, arm_t32(t3hw1 | ARM_TMP, (rt << 12) | 0u));
}
}
@@ -424,7 +466,12 @@ static void arm_emit_base_off(Arm32NativeTarget* a, u32 rd, u32 base, i32 off) {
} else if (off < 0 && thumb_expand_imm_encode((u32)(-off), &enc)) {
arm_emit_t32(mc, arm_dp_imm(13u, 0u, rd, base, enc)); /* SUB rd, base, #-off */
} else {
- arm_panic(a, "load_addr offset out of range");
+ /* Large frame/array offset (-O1 big frames): stage |off| in ARM_TMP (lr) so
+ * the base register survives even when rd aliases it, then add/sub into rd.
+ * lr is reserved and dead in the body. */
+ arm_emit_load_u32(a, ARM_TMP, (u32)(off < 0 ? -off : off));
+ arm_emit_t32(mc, off < 0 ? arm_sub_reg(rd, base, ARM_TMP)
+ : arm_add_reg(rd, base, ARM_TMP));
}
}
@@ -679,6 +726,9 @@ static void arm_cmp_branch(NativeTarget* t, CmpOp op, NativeLoc a_loc,
/* ============================ frame lifecycle ============================ */
+static u32 arm_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
+ int* variadic, u32* nparams);
+
static NativeFrameSlot arm_frame_slot(NativeTarget* t,
const NativeFrameSlotDesc* d) {
return native_frame_slot_alloc(&arm_of(t)->frame, d);
@@ -709,6 +759,89 @@ static void arm_reserve_callee_saves(NativeTarget* t, const u32* used_by_class,
NULL, 0, 0);
}
+/* AAPCS callee-saved integer registers this backend lets the allocator use:
+ * r4..r11 except r7 (the frame pointer, preserved by the saved-pair head, not as
+ * an ordinary callee-save). Matches arm_int_phys's ARM_PHYS_CALLEE set. */
+static int arm_reg_is_callee_int(Reg r) {
+ return r >= 4u && r <= 11u && r != ARM_FP;
+}
+
+/* Build the callee-saved set the prologue must preserve, from the allocator's
+ * per-class masks. Only the integer class exists in this soft-float backend, so
+ * the FP/vec classes are ignored; inline asm is unsupported (asm_block is not
+ * installed), so there are no opaque asm clobbers to fold in. Restricts to the
+ * register file's actual callee-saved set as a guard. */
+static u32 arm_known_callee_saves(const NativeKnownFrameDesc* frame) {
+ u32 mask = 0;
+ if (frame && frame->callee_saved_used && frame->ncallee_classes > NATIVE_REG_INT)
+ mask = frame->callee_saved_used[NATIVE_REG_INT];
+ for (Reg r = 0; r < 16u; ++r)
+ if ((mask & (1u << r)) && !arm_reg_is_callee_int(r)) mask &= ~(1u << r);
+ return mask;
+}
+
+/* Build the `SUB sp, sp, #n` instruction sequence (n a multiple of 8) into
+ * words[], returning the count (<= ARM_NDT_SUB_WORDS). Small amounts use one
+ * inline SUBW; large amounts (n > 4095, big frames) stage n into IP via
+ * MOVW(+MOVT) and `SUB sp, sp, ip`. IP is the emit scratch, dead at the
+ * prologue. Shared by the inline known-frame emit and the single-pass patch. */
+static u32 arm_build_sub_sp(u32 n, u32* words) {
+ u32 enc, wi = 0;
+ if (n == 0) return 0;
+ if (n <= 0xfffu) {
+ words[wi++] = arm_sub_imm12(13u, 13u, n);
+ return wi;
+ }
+ if (thumb_expand_imm_encode(n, &enc)) {
+ words[wi++] = arm_dp_imm(13u, 0u, 13u, 13u, enc); /* SUB.W sp,sp,#modimm */
+ return wi;
+ }
+ words[wi++] = arm_movw(ARM_SCRATCH, n & 0xffffu);
+ if ((n >> 16) != 0u) words[wi++] = arm_movt(ARM_SCRATCH, (n >> 16) & 0xffffu);
+ words[wi++] = arm_sub_reg(13u, 13u, ARM_SCRATCH);
+ return wi;
+}
+
+/* Emit `SUB sp, sp, #n` inline (known-frame prologue, where the size is final). */
+static void arm_emit_sub_sp(Arm32NativeTarget* a, u32 n) {
+ u32 words[ARM_NDT_SUB_WORDS];
+ u32 nwords = arm_build_sub_sp(n, words), i;
+ for (i = 0; i < nwords; ++i) arm_emit_t32(a->base.mc, words[i]);
+}
+
+/* sret: reserve a hidden frame slot for the incoming destination pointer (r0)
+ * and advance the param cursor past it. Shared by the single-pass and
+ * known-frame paths (slot creation must happen at the same point in both). */
+static void arm_reserve_sret_slot(Arm32NativeTarget* a) {
+ NativeTarget* t = &a->base;
+ if (a->has_sret) {
+ NativeFrameSlotDesc sd;
+ memset(&sd, 0, sizeof sd);
+ sd.type = builtin_id(KIT_CG_BUILTIN_I32);
+ sd.size = 4;
+ sd.align = 4;
+ sd.kind = NATIVE_FRAME_SLOT_SAVE;
+ a->sret_ptr_slot = arm_frame_slot(t, &sd);
+ a->next_param_int = 1; /* r0 consumed by the sret pointer */
+ }
+}
+
+/* Spill the incoming sret pointer (r0) to its hidden home. The home is
+ * r7-anchored, so this runs correctly after the frame is set up. */
+static void arm_emit_sret_store(Arm32NativeTarget* a) {
+ NativeTarget* t = &a->base;
+ if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ NativeAddr addr;
+ memset(&addr, 0, sizeof addr);
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ addr.base.frame = a->sret_ptr_slot;
+ addr.base_type = i32t;
+ arm_emit_mem(a, 0, native_loc_reg(i32t, NATIVE_REG_INT, ARM_R0), addr,
+ native_mem_for_type(t, i32t, 4));
+ }
+}
+
static void arm_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
Arm32NativeTarget* a = arm_of(t);
MCEmitter* mc = t->mc;
@@ -721,6 +854,10 @@ static void arm_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
+ a->known_frame = 0;
+ a->slim_prologue = 0;
+ a->saved_reglist = (u16)((1u << ARM_FP) | (1u << 14u)); /* {r7, lr} */
+ a->saved_block_bytes = 8u; /* two words */
mc_set_section(mc, fd->text_section_id);
mc_emit_align(mc, 4, 0);
@@ -729,40 +866,118 @@ static void arm_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
mc_cfi_startproc(mc);
a->epilogue_label = mc_label_new(mc);
- /* Prologue: save fp+lr, set the frame anchor, reserve the (deferred) frame. */
+ /* Prologue: save fp+lr, set the frame anchor, reserve the (deferred) frame.
+ * The deferred `SUB sp` is patched in arm_func_end once cum_off/max_outgoing
+ * are final. Reserve ARM_NDT_SUB_WORDS T32 slots so a large frame (> 4095) can
+ * be patched as an IP-staged MOVW/MOVT/SUB sequence; a small frame patches the
+ * first slot with SUBW and fills the rest with NOPs. */
arm_emit_t32(mc, arm_push_w((1u << ARM_FP) | (1u << 14u))); /* PUSH {r7, lr} */
arm_emit_t16(mc, arm_mov_hi(ARM_FP, 13u)); /* MOV r7, sp */
a->prologue_sub_pos = mc_pos(mc);
- arm_emit_t32(mc, arm_sub_imm12(13u, 13u, 0u)); /* SUB sp, sp, #0 (patched) */
+ for (u32 i = 0; i < ARM_NDT_SUB_WORDS; ++i)
+ arm_emit_t32(mc, arm_nop32()); /* placeholder, patched in arm_func_end */
/* sret: spill the incoming destination pointer (r0) to a hidden home. */
- if (a->has_sret) {
- NativeFrameSlotDesc sd;
- NativeAddr addr;
- KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
- memset(&sd, 0, sizeof sd);
- sd.type = i32t;
- sd.size = 4;
- sd.align = 4;
- sd.kind = NATIVE_FRAME_SLOT_SAVE;
- a->sret_ptr_slot = arm_frame_slot(t, &sd);
- a->next_param_int = 1;
- memset(&addr, 0, sizeof addr);
- addr.base_kind = NATIVE_ADDR_BASE_FRAME;
- addr.base.frame = a->sret_ptr_slot;
- addr.base_type = i32t;
- arm_emit_mem(a, 0, native_loc_reg(i32t, NATIVE_REG_INT, ARM_R0), addr,
- native_mem_for_type(t, i32t, 4));
- }
-}
-
+ arm_reserve_sret_slot(a);
+ arm_emit_sret_store(a);
+}
+
+/* Optimizer (-O1) entry point: the optimizer has run register allocation and
+ * supplies the exact frame up front, so the prologue is emitted final the moment
+ * it is built — no deferred `SUB sp, #0` placeholder, no arm_func_end patch
+ * (arm_func_end skips patching when known_frame). The callee-saved registers the
+ * allocator used are folded directly into the PUSH/POP register list rather than
+ * given their own frame slots: they ride above the frame anchor r7 in the
+ * STMDB-saved block, which keeps slot offsets ([r7, #-off]) byte-identical to
+ * the single-pass layout and lets one LDMIA-via-pc restore-and-return.
+ *
+ * Slot creation order matches the single-pass path: static slots first, then the
+ * sret entry-save slot. (Callee-saves take no slots here, so order vs. them is
+ * moot.) */
static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
const NativeKnownFrameDesc* kf,
NativeFrameSlot* out_slots) {
- (void)fd;
- (void)kf;
- (void)out_slots;
- arm_panic(arm_of(t), "known-frame (-O1) path not implemented in Phase 1");
+ Arm32NativeTarget* a = arm_of(t);
+ MCEmitter* mc = t->mc;
+ const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
+ u32 cs_mask = arm_known_callee_saves(kf);
+ u32 reglist, nsaved, frame, i;
+
+ a->func = fd;
+ a->loc = fd->loc;
+ native_frame_reset(&a->frame);
+ a->next_param_int = 0;
+ a->next_param_stack = 0;
+ a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
+ a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
+ a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
+ a->known_frame = 1;
+ a->slim_prologue = 0;
+ a->frame.known_frame = 1;
+ a->frame.has_alloca = kf ? kf->has_alloca : 0u;
+
+ mc_set_section(mc, fd->text_section_id);
+ mc_emit_align(mc, 4, 0);
+ a->func_start = mc_pos(mc);
+ mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
+ mc_cfi_startproc(mc);
+ a->epilogue_label = mc_label_new(mc);
+
+ /* Record the callee-save set (for parity / debug) and build the body's slots
+ * and outgoing-area reservation, then settle the exact frame. */
+ if (cs_mask) {
+ u32 used[NATIVE_CALL_PLAN_CLASSES] = {0, 0, 0};
+ used[NATIVE_REG_INT] = cs_mask;
+ arm_reserve_callee_saves(t, used, NATIVE_CALL_PLAN_CLASSES);
+ }
+ if (kf) {
+ for (i = 0; i < kf->nslots; ++i) {
+ NativeFrameSlot slot = arm_frame_slot(t, &kf->slots[i]);
+ if (out_slots) out_slots[i] = slot;
+ }
+ arm_reserve_sret_slot(a);
+ native_frame_note_outgoing(&a->frame, kf->max_outgoing);
+ } else {
+ arm_reserve_sret_slot(a);
+ }
+
+ /* The PUSH/POP register list: callee-saves (low to high) + r7 + lr. STMDB
+ * stores in ascending register order regardless of the bit order, so r7 lands
+ * in the block correctly and `MOV r7, sp` anchors just below the whole block.
+ * The popped lr is restored into pc, returning. */
+ reglist = cs_mask | (1u << ARM_FP) | (1u << 14u);
+ nsaved = 0;
+ for (i = 0; i < 16u; ++i)
+ if (reglist & (1u << i)) nsaved++;
+ a->saved_reglist = (u16)reglist;
+ a->saved_block_bytes = nsaved * 4u;
+
+ /* Leaf no-frame tier (aa64 slim_prologue / rv frameless equivalent): a leaf
+ * with no callee-saves, no body slots, no outgoing args, no sret/variadic and
+ * only register params never touches the stack or clobbers lr, so emit no
+ * prologue at all and let arm_func_end emit a bare `BX lr`. Inline asm cannot
+ * occur (asm_block is not installed), so no opaque-lr-clobber guard is needed.
+ */
+ a->slim_prologue = kf && kf->is_leaf && cs_mask == 0 &&
+ !a->frame.has_alloca && a->frame.cum_off == 0 &&
+ a->frame.max_outgoing == 0 && !a->has_sret &&
+ !a->is_variadic &&
+ arm_signature_stack_bytes(t, fd->fn_type, NULL, NULL) == 0;
+ if (a->slim_prologue) {
+ a->saved_reglist = 0;
+ a->saved_block_bytes = 0;
+ native_frame_set_final(&a->frame);
+ return;
+ }
+
+ /* Emit the final prologue: PUSH the save set, anchor r7, drop the frame. */
+ arm_emit_t32(mc, arm_push_w(reglist));
+ arm_emit_t16(mc, arm_mov_hi(ARM_FP, 13u)); /* MOV r7, sp */
+ frame = arm_frame_size(a);
+ arm_emit_sub_sp(a, frame);
+
+ arm_emit_sret_store(a);
+ native_frame_set_final(&a->frame);
}
static void arm_func_end(NativeTarget* t) {
@@ -771,14 +986,35 @@ static void arm_func_end(NativeTarget* t) {
u32 frame;
/* Place the epilogue and emit the teardown. */
mc_label_place(mc, a->epilogue_label);
- arm_emit_t16(mc, arm_mov_hi(13u, ARM_FP)); /* MOV sp, r7 */
- arm_emit_t32(mc, arm_pop_w((1u << ARM_FP) | (1u << 15u))); /* POP {r7, pc} */
+ if (a->slim_prologue) {
+ /* Frameless leaf: nothing saved, sp untouched — return through lr directly. */
+ arm_emit_t16(mc, arm_bx(14u)); /* BX lr */
+ } else {
+ /* MOV sp, r7 drops the locals frame back to the saved-register block; POP
+ * the whole save set, restoring lr into pc to return. On the known-frame
+ * path the save set is {callee-saves, r7, lr}; on the single-pass path it is
+ * the bare {r7, lr}. Both pop lr → pc. */
+ u32 poplist = (a->saved_reglist & ~(1u << 14u)) | (1u << 15u);
+ arm_emit_t16(mc, arm_mov_hi(13u, ARM_FP)); /* MOV sp, r7 */
+ arm_emit_t32(mc, arm_pop_w(poplist)); /* POP {.., r7, pc} */
+ }
- native_frame_set_final(&a->frame);
- frame = arm_frame_size(a);
- if (frame > 0xfffu)
- arm_panic(a, "frame too large for Phase 1 (needs IP-staged SUB)");
- arm_patch_t32(a, a->prologue_sub_pos, arm_sub_imm12(13u, 13u, frame));
+ /* Single-pass path: settle the frame and patch the deferred `SUB sp` region.
+ * The known-frame path emitted its prologue final in
+ * arm_func_begin_known_frame (frame already settled there), so it never
+ * patches. The reserved region is ARM_NDT_SUB_WORDS NOP.W slots; patch the
+ * leading slots with the built SUB sequence (one SUBW for a small frame, an
+ * IP-staged MOVW/MOVT/SUB for a large one) and leave the trailing NOP.W slots,
+ * which execute harmlessly. */
+ if (!a->known_frame) {
+ u32 words[ARM_NDT_SUB_WORDS];
+ u32 nwords, i;
+ native_frame_set_final(&a->frame);
+ frame = arm_frame_size(a);
+ nwords = arm_build_sub_sp(frame, words);
+ for (i = 0; i < nwords; ++i)
+ arm_patch_t32(a, a->prologue_sub_pos + i * 4u, words[i]);
+ }
/* Publish the function symbol. ARM marks Thumb STT_FUNC symbols with the
* low bit set (the Thumb bit), so &fn and indirect BLX reach Thumb state;
@@ -817,14 +1053,16 @@ static void arm_bind_native_param(NativeTarget* t, const CGParamDesc* p,
if (a->next_param_int < 4u) {
src = native_loc_reg(p->type, NATIVE_REG_INT, (Reg)(a->next_param_int++));
} else {
- /* Incoming stack arg: above the saved {r7,lr} pair = [r7 + 8 + k]. */
+ /* Incoming stack arg: above the saved register block = [r7 + N + k],
+ * where N = saved_block_bytes (8 for the bare {r7,lr} -O0 prologue; larger
+ * when the -O1 known-frame prologue also PUSHes callee-saves). */
NativeAddr sa;
src = native_loc_reg(p->type, NATIVE_REG_INT, ARM_SCRATCH);
memset(&sa, 0, sizeof sa);
sa.base_kind = NATIVE_ADDR_BASE_REG;
sa.base.reg = ARM_FP;
sa.base_type = p->type;
- sa.offset = (i32)(8u + a->next_param_stack);
+ sa.offset = (i32)(a->saved_block_bytes + a->next_param_stack);
arm_emit_mem(a, 1, src, sa, native_mem_for_type(t, p->type, part->size));
a->next_param_stack += 4u;
}
