commit 21d150509f42a4aba98cb8211e89220be143b242
parent c404424a4ab14b13cca7869d5c7b7c22a73c0c00
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 22:01:02 -0700
arm32 §2: tail/sibling calls (AAPCS) + variadic epilogue SP fix
Implement true sibling calls, replacing the call-plus-return panic:
- emit_call CG_CALL_TAIL: tear the frame down with lr restored to OUR caller's
return address (not popped into pc), then B.W (direct) / BX (indirect) the
callee — it runs on the caller's frame and returns straight to our caller. The
arm32 epilogue is frame-size-independent (MOV sp,r7), so the single-pass path
needs no patch (unlike aa64).
- Outgoing stack args of a tail call are written into the caller's incoming-arg
window in place ([r7 + arg_window_base + off]) — the exact address the callee
reads after the epilogue restores sp to entry. plan_call threads the tail flag
through store_outgoing; max_outgoing is not grown for tail calls.
- sret tail: forward OUR spilled incoming sret pointer to r0 so the callee writes
the result into our caller's buffer.
- Indirect callee staged into IP (survives the epilogue's POP of r0..r11); tail
stack args stage through LR instead so they don't clobber the IP-held callee.
- arm_no_tail: realizable iff the callee's stack args fit incoming_stack_size
(set per function from its signature); register-only callees always fit.
- Shared arm_emit_frame_restore unifies the return + tail epilogues and, en
route, FIXES a latent bug: a variadic callee never reclaimed its 16-byte
r0..r3 GP-save area, returning with sp 16 bytes low. Now reclaimed.
Covers musttail/tail × direct/indirect/void/sret/variadic/stack-args/permute/
chain at O0+O1 (16 corpus cases). arm32 corpus 305 pass (was 285).
Diffstat:
1 file changed, 97 insertions(+), 30 deletions(-)
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -90,6 +90,11 @@ typedef struct Arm32NativeTarget {
u8 slim_prologue; /* leaf, no frame: emit BX lr, no PUSH/anchor */
u16 saved_reglist; /* registers PUSHed by the prologue (POP restores via pc) */
u32 saved_block_bytes;
+ /* Bytes of incoming stack-argument window this function owns (its own params
+ * beyond r0..r3, 8-byte rounded). A tail/sibling call is realizable only if the
+ * callee's outgoing stack args fit here — its outgoing stack args reuse this
+ * window in place. */
+ u32 incoming_stack_size;
/* alloca: each dynamic allocation moves sp down and returns sp + max_outgoing
* (the block sits just above the call outgoing-arg area at the bottom of the
* frame). max_outgoing is only final at func_end, so the `ADD dst, sp, #off`
@@ -1022,6 +1027,7 @@ static void arm_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
a->saved_reglist = (u16)((1u << ARM_FP) | (1u << 14u)); /* {r7, lr} */
a->saved_block_bytes = 8u; /* two words */
a->n_alloca_patch = 0;
+ a->incoming_stack_size = arm_signature_stack_bytes(t, fd->fn_type, NULL, NULL);
mc_set_section(mc, fd->text_section_id);
mc_emit_align(mc, 4, 0);
@@ -1083,6 +1089,7 @@ static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
a->known_frame = 1;
a->slim_prologue = 0;
a->n_alloca_patch = 0;
+ a->incoming_stack_size = arm_signature_stack_bytes(t, fd->fn_type, NULL, NULL);
a->frame.known_frame = 1;
a->frame.has_alloca = kf ? kf->has_alloca : 0u;
@@ -1150,24 +1157,36 @@ static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
native_frame_set_final(&a->frame);
}
+/* Emit the frame teardown. With to_pc, return — the saved lr pops straight into
+ * pc, the size-independent common case (MOV sp,r7 drops any locals frame). Else
+ * restore the save set with lr intact and fall through with lr = the caller's
+ * return address, for a tail-call branch. Both reclaim a variadic callee's
+ * r0..r3 GP-save area (16 bytes above the saved block) so sp returns to entry. */
+static void arm_emit_frame_restore(Arm32NativeTarget* a, int to_pc) {
+ MCEmitter* mc = a->base.mc;
+ u32 va = a->is_variadic ? ARM_VA_GP_SAVE_BYTES : 0u;
+ arm_emit_t16(mc, arm_mov_hi(13u, ARM_FP)); /* MOV sp, r7 */
+ if (to_pc && va == 0u) {
+ u32 poplist = (a->saved_reglist & ~(1u << 14u)) | (1u << 15u);
+ arm_emit_t32(mc, arm_pop_w(poplist)); /* POP {.., r7, pc} */
+ return;
+ }
+ arm_emit_t32(mc, arm_pop_w(a->saved_reglist)); /* POP {.., r7, lr} */
+ if (va) arm_emit_add_const(a, 13u, (i64)va); /* reclaim r0..r3 save area */
+ if (to_pc) arm_emit_t16(mc, arm_bx(14u)); /* BX lr (return) */
+}
+
static void arm_func_end(NativeTarget* t) {
Arm32NativeTarget* a = arm_of(t);
MCEmitter* mc = t->mc;
u32 frame;
/* Place the epilogue and emit the teardown. */
mc_label_place(mc, a->epilogue_label);
- if (a->slim_prologue) {
+ if (a->slim_prologue)
/* Frameless leaf: nothing saved, sp untouched — return through lr directly. */
arm_emit_t16(mc, arm_bx(14u)); /* BX lr */
- } else {
- /* MOV sp, r7 drops the locals frame back to the saved-register block; POP
- * the whole save set, restoring lr into pc to return. On the known-frame
- * path the save set is {callee-saves, r7, lr}; on the single-pass path it is
- * the bare {r7, lr}. Both pop lr → pc. */
- u32 poplist = (a->saved_reglist & ~(1u << 14u)) | (1u << 15u);
- arm_emit_t16(mc, arm_mov_hi(13u, ARM_FP)); /* MOV sp, r7 */
- arm_emit_t32(mc, arm_pop_w(poplist)); /* POP {.., r7, pc} */
- }
+ else
+ arm_emit_frame_restore(a, /*to_pc=*/1);
/* Single-pass path: settle the frame and patch the deferred `SUB sp` region.
* The known-frame path emitted its prologue final in
@@ -1503,19 +1522,25 @@ static void arm_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
}
}
+/* Store an outgoing stack argument. A normal call writes the sp-anchored
+ * outgoing-arg area ([sp, #stack_off]). A tail/sibling call instead writes the
+ * caller's incoming-arg window ([r7 + arg_window_base + stack_off]) — the exact
+ * address the tail-callee will read after the epilogue restores sp to the
+ * caller's entry sp and branches. */
static void arm_store_outgoing(NativeTarget* t, u32 stack_off, NativeLoc src,
- u32 size) {
+ u32 size, int tail) {
+ Arm32NativeTarget* a = arm_of(t);
NativeAddr addr;
MemAccess mem;
memset(&addr, 0, sizeof addr);
addr.base_kind = NATIVE_ADDR_BASE_REG;
- addr.base.reg = 13u; /* sp */
- addr.offset = (i32)stack_off;
+ addr.base.reg = tail ? ARM_FP : 13u;
+ addr.offset = (i32)(tail ? arm_arg_window_base(a) + stack_off : stack_off);
addr.base_type = src.type;
memset(&mem, 0, sizeof mem);
mem.type = src.type;
mem.size = size;
- arm_emit_mem(arm_of(t), 0, src, addr, mem);
+ arm_emit_mem(a, 0, src, addr, mem);
}
/* Materialize the address of a frame/stack-resident NativeLoc into `dst` (used
@@ -1564,6 +1589,7 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
? abi->ret.nparts
: ((!abi && desc->nresults) ? 1u : 0u);
+ int tail = (desc->flags & CG_CALL_TAIL) != 0;
memset(plan, 0, sizeof *plan);
rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPlanRet, nrets_cap) : NULL;
plan->callee = desc->callee;
@@ -1572,19 +1598,28 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
plan->has_sret = abi && abi->has_sret;
plan->is_variadic = abi && abi->variadic;
plan->stack_arg_size = arm_call_stack_size(t, desc);
- if (plan->stack_arg_size > a->frame.max_outgoing)
+ /* A tail call's outgoing stack args reuse the incoming-arg window in place, so
+ * they do not enlarge this frame's outgoing-arg area. */
+ if (!tail && plan->stack_arg_size > a->frame.max_outgoing)
a->frame.max_outgoing = plan->stack_arg_size;
- /* Stage an indirect callee out of the arg registers (arg loads clobber them). */
+ /* Stage an indirect callee into IP so it survives both the arg-register setup
+ * and (for a tail call) the epilogue's POP of r0..r11. A normal call only needs
+ * to dodge the arg registers (r0..r3); a tail call must dodge every restored
+ * register, so any REG callee is staged. IP is never popped/restored. */
if (plan->callee.kind == NATIVE_LOC_REG &&
(NativeAllocClass)plan->callee.cls == NATIVE_REG_INT &&
- plan->callee.v.reg <= 3u) {
+ (plan->callee.v.reg <= 3u || tail)) {
NativeLoc scratch =
native_loc_reg(plan->callee.type, NATIVE_REG_INT, ARM_SCRATCH);
arm_move(t, scratch, plan->callee);
plan->callee = scratch;
}
{
+ /* Outgoing stack args stage through IP normally; a tail call holds the callee
+ * in IP, so it stages through LR instead (LR is restored by the epilogue
+ * afterward, and the in-window store address needs no scratch). */
+ u32 arg_stage = tail ? ARM_TMP : ARM_SCRATCH;
u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
u32 stack = 0, nmoves = 0, i, p;
NativeArgMove moves[ARM_MAX_REG_ARG_MOVES];
@@ -1606,10 +1641,10 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
m->size = 4u;
m->is_addr = 1;
} else {
- NativeLoc ptr = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
+ NativeLoc ptr = native_loc_reg(i32t, NATIVE_REG_INT, arg_stage);
arm_addr_of_loc(t, ptr, desc->args[i]);
stack = align_up_u32(stack, 4u);
- arm_store_outgoing(t, stack, ptr, 4u);
+ arm_store_outgoing(t, stack, ptr, 4u, tail);
stack += 4u;
}
continue;
@@ -1632,21 +1667,29 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
m->size = part->size;
m->is_addr = 0;
} else {
- NativeLoc tmp = native_loc_reg(lty, NATIVE_REG_INT, ARM_SCRATCH);
+ NativeLoc tmp = native_loc_reg(lty, NATIVE_REG_INT, arg_stage);
arm_load_part(t, tmp, desc->args[i], part->src_offset, part->size);
if (p == 0 && eight) stack = align_up_u32(stack, 8u);
else stack = align_up_u32(stack, 4u);
- arm_store_outgoing(t, stack, tmp, part->size);
+ arm_store_outgoing(t, stack, tmp, part->size, tail);
stack += 4u;
}
}
}
arm_emit_reg_arg_moves(t, moves, nmoves);
- if (abi && abi->has_sret && desc->nresults) {
+ if (abi && abi->has_sret) {
+ /* sret destination pointer in r0. A tail call forwards OUR own incoming
+ * sret pointer (spilled at entry) so the tail-callee writes the result
+ * straight into our caller's buffer; a normal call passes &result. */
NativeLoc r0 = native_loc_reg(i32t, NATIVE_REG_INT, ARM_R0);
- arm_load_addr(t, r0, (NativeAddr){.base_kind = NATIVE_ADDR_BASE_FRAME,
- .base.frame = desc->results[0].v.frame,
- .base_type = i32t});
+ if (tail && a->has_sret) {
+ NativeLoc saved = native_loc_stack(i32t, a->sret_ptr_slot, 0);
+ arm_load_part(t, r0, saved, 0, 4u);
+ } else if (desc->nresults) {
+ arm_load_addr(t, r0, (NativeAddr){.base_kind = NATIVE_ADDR_BASE_FRAME,
+ .base.frame = desc->results[0].v.frame,
+ .base_type = i32t});
+ }
}
}
if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
@@ -1677,8 +1720,28 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
static void arm_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
Arm32NativeTarget* a = arm_of(t);
MCEmitter* mc = t->mc;
- if (plan->flags & CG_CALL_TAIL)
- arm_panic(a, "tail calls are Phase 2");
+ if (plan->flags & CG_CALL_TAIL) {
+ /* Sibling call: the outgoing args are already placed (register args live;
+ * stack args written into the incoming-arg window). Tear the frame down with
+ * lr restored to OUR caller's return address, then BRANCH to the callee — it
+ * runs on our caller's frame and returns straight to our caller. The arm32
+ * epilogue is frame-size-independent (MOV sp,r7), so no patch is needed even
+ * on the single-pass path. The callee is direct (B.W) or held in IP (BX). */
+ if (plan->callee.kind != NATIVE_LOC_GLOBAL &&
+ plan->callee.kind != NATIVE_LOC_REG)
+ arm_panic(a, "unsupported tail target");
+ arm_emit_frame_restore(a, /*to_pc=*/0);
+ if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
+ u32 pos = mc_pos(mc);
+ arm_emit_t32(mc, arm_b_w());
+ mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_JUMP24,
+ plan->callee.v.global.sym, plan->callee.v.global.addend, 0,
+ 0);
+ } else {
+ arm_emit_t16(mc, arm_bx(loc_reg(plan->callee)));
+ }
+ return;
+ }
if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
u32 pos = mc_pos(mc);
arm_emit_t32(mc, arm_bl());
@@ -3131,10 +3194,14 @@ static void arm_bind_param(NativeDirectTarget* d, const CGParamDesc* p,
arm_bind_native_param(d->native, p, dst);
}
+/* A sibling call's outgoing stack args reuse the caller's incoming-arg window in
+ * place; it is realizable only if they fit. (Register-only callees always fit.) */
static const char* arm_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
- (void)d;
- (void)call;
- return "arm32 tail calls not implemented in Phase 1";
+ NativeCallDesc nd;
+ native_direct_project_tail_call_desc(d, call, &nd);
+ if (arm_call_stack_size(d->native, &nd) > arm_of(d->native)->incoming_stack_size)
+ return "arm32 tail call: callee stack args exceed the caller's window";
+ return NULL;
}
/* ---- Direct (-O0) varargs wrappers: resolve a va_list operand's address into
