commit 319f6e7440208c4e0f7a79ec13548c5e32d6dce8
parent 0e43fcea5cb98d166ee610ebda16d422d714b718
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 18:04:17 -0700
arm32 Phase 2: i64 + soft-double end-to-end (AAPCS pair-align, 8-byte load_const, call/return pairs)
Wide8 i64/double: AAPCS even-register pair alignment (arm_arg_advance), per-lane
i32 types for i64/double call args/returns, 8-byte load_const backstop, and an
arm_cmp operand-aliasing fix (emit CMP before MOV dst,#0; dst can alias a compare
operand, e.g. a soft-float compare's r0 result). Integrated keeping the already-
landed REL emission (Track REL) and O1's saved_block_bytes stack-arg offset
(combined with the per-lane type in bind_param).
Diffstat:
2 files changed, 169 insertions(+), 53 deletions(-)
diff --git a/src/abi/abi_aapcs32.c b/src/abi/abi_aapcs32.c
@@ -31,15 +31,20 @@ static void classify_scalar(TargetABI* a, KitCgTypeId t, ABIArgInfo* out) {
(ti.scalar_kind == ABI_SC_INT || ti.scalar_kind == ABI_SC_FLOAT)) {
ABIArgPart* parts = arena_array(a->c->tu, ABIArgPart, 2);
memset(parts, 0, sizeof(ABIArgPart) * 2);
+ /* Both lanes carry align=8: AAPCS requires an 8-byte scalar to land in an
+ * even/odd core-register pair (r0:r1 or r2:r3) and to 8-byte-align its stack
+ * slot. The native backend reads this part alignment (> the 4-byte slot) as
+ * the round-NCRN-to-even / 8-align-stack signal — distinguishing an i64/
+ * double pair from a 4-byte-aligned two-word aggregate (struct{int;int;}). */
parts[0].cls = ABI_CLASS_INT;
parts[0].loc = ABI_LOC_REG;
parts[0].size = ARM32_GPR_BYTES;
- parts[0].align = ARM32_GPR_BYTES;
+ parts[0].align = 2u * ARM32_GPR_BYTES;
parts[0].src_offset = 0;
parts[1].cls = ABI_CLASS_INT;
parts[1].loc = ABI_LOC_REG;
parts[1].size = ARM32_GPR_BYTES;
- parts[1].align = ARM32_GPR_BYTES;
+ parts[1].align = 2u * ARM32_GPR_BYTES;
parts[1].src_offset = ARM32_GPR_BYTES;
out->kind = ABI_ARG_DIRECT;
out->flags = ABI_AF_NONE;
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -638,7 +638,12 @@ static void arm_emit_cmp_operands(Arm32NativeTarget* a, NativeLoc x,
arm_emit_t32(mc, arm_cmp_reg(loc_reg(x), loc_reg(y)));
}
-/* Materialize the boolean (a OP b) into dst: MOV dst,#0; CMP; IT cc; MOV dst,#1. */
+/* Materialize the boolean (a OP b) into dst: CMP; MOV dst,#0; IT cc; MOV dst,#1.
+ * The CMP must precede the `MOV dst,#0`: dst can ALIAS a compare operand (e.g. a
+ * soft-float compare leaves its libcall result in r0 and the bool destination is
+ * also r0), and a leading `MOV dst,#0` would clobber that operand before CMP
+ * reads it. MOV.W #imm (T2, S=0) does not set flags, so emitting it after CMP
+ * leaves the compare flags intact for the IT-predicated `MOV dst,#1`. */
static void arm_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc a_loc,
NativeLoc b) {
Arm32NativeTarget* a = arm_of(t);
@@ -648,8 +653,8 @@ static void arm_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc a_loc,
u32 enc0, enc1;
thumb_expand_imm_encode(0u, &enc0);
thumb_expand_imm_encode(1u, &enc1);
- arm_emit_t32(mc, arm_mov_imm(rd, enc0)); /* dst = 0 */
- arm_emit_cmp_operands(a, a_loc, b); /* CMP a, b */
+ arm_emit_cmp_operands(a, a_loc, b); /* CMP a, b (sets flags) */
+ arm_emit_t32(mc, arm_mov_imm(rd, enc0)); /* dst = 0 (MOV.W, no flag set) */
arm_emit_t16(mc, (u16)(0xbf08u | (cc << 4))); /* IT cc (1 insn, mask 0b1000) */
arm_emit_t32(mc, arm_mov_imm(rd, enc1)); /* MOV<cc> dst, #1 */
}
@@ -1037,33 +1042,75 @@ static void arm_func_end(NativeTarget* t) {
/* ============================ param binding ============================ */
+#define ARM32_GPR_BYTES 4u
+
+/* Lane scalar type for an ABI part: an 8-byte i64/double DIRECT argument is two
+ * 4-byte INT lanes, an aggregate up to four word/sub-word lanes. The NDT sizes
+ * every load/store/move it emits for a plan-return / plan-call / param-bind move
+ * from the lane location's TYPE (nd_type_mem -> cg_type_size), so each lane must
+ * carry a type whose natural size equals the lane's byte width — NOT the full
+ * 8-byte scalar type (which would drive an 8-byte access the 32-bit backend has
+ * no single-instruction form for). Mirrors riscv's rv_part_scalar_type. */
+static KitCgTypeId arm_part_scalar_type(const ABIArgPart* part) {
+ switch (part->size) {
+ case 1u: return builtin_id(KIT_CG_BUILTIN_I8);
+ case 2u: return builtin_id(KIT_CG_BUILTIN_I16);
+ case 8u: return builtin_id(KIT_CG_BUILTIN_I64);
+ default: return builtin_id(KIT_CG_BUILTIN_I32);
+ }
+}
+
+/* AAPCS pair rule: an 8-byte scalar argument (i64/double — surfaced by the ABI
+ * as a 2-word DIRECT arg whose parts carry align=8, the slot-exceeding marker)
+ * must occupy an even/odd core-register pair (r0:r1 or r2:r3); the cursor rounds
+ * up to an even index first. Returns 1 if `ai`'s first part demands that. A
+ * 4-byte-aligned two-word aggregate (struct{int;int;}) has align=4 and is NOT
+ * pair-aligned, so the shape (nparts==2) alone is insufficient — the alignment
+ * is the discriminator. */
+static int arm_arg_is_aligned_pair(const ABIArgInfo* ai) {
+ return ai->kind == ABI_ARG_DIRECT && ai->nparts == 2u && ai->parts &&
+ ai->parts[0].align >= 2u * ARM32_GPR_BYTES;
+}
+
static void arm_bind_native_param(NativeTarget* t, const CGParamDesc* p,
NativeLoc dst) {
Arm32NativeTarget* a = arm_of(t);
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
const ABIArgInfo* ai = p->index < abi->nparams ? &abi->params[p->index] : NULL;
int to_reg = dst.kind == NATIVE_LOC_REG;
+ int pair = ai && arm_arg_is_aligned_pair(ai);
u32 i;
if (!ai || ai->kind == ABI_ARG_IGNORE) return;
if (ai->kind == ABI_ARG_INDIRECT)
arm_panic(a, "indirect (by-ref aggregate) params are Phase 2");
+ /* AAPCS: round NCRN up to even before an 8-byte aligned pair, and if the pair
+ * does not fully fit in the remaining core registers it passes ENTIRELY on the
+ * (8-byte-aligned) stack — it never straddles the r3/stack boundary. */
+ if (pair) {
+ a->next_param_int = align_up_u32(a->next_param_int, 2u);
+ if (a->next_param_int + 2u > 4u) {
+ a->next_param_int = 4u;
+ a->next_param_stack = align_up_u32(a->next_param_stack, 2u * ARM32_GPR_BYTES);
+ }
+ }
for (i = 0; i < ai->nparts; ++i) {
const ABIArgPart* part = &ai->parts[i];
+ KitCgTypeId lty = arm_part_scalar_type(part);
NativeLoc src;
if (a->next_param_int < 4u) {
- src = native_loc_reg(p->type, NATIVE_REG_INT, (Reg)(a->next_param_int++));
+ src = native_loc_reg(lty, NATIVE_REG_INT, (Reg)(a->next_param_int++));
} else {
/* Incoming stack arg: above the saved register block = [r7 + N + k],
* where N = saved_block_bytes (8 for the bare {r7,lr} -O0 prologue; larger
* when the -O1 known-frame prologue also PUSHes callee-saves). */
NativeAddr sa;
- src = native_loc_reg(p->type, NATIVE_REG_INT, ARM_SCRATCH);
+ src = native_loc_reg(lty, NATIVE_REG_INT, ARM_SCRATCH);
memset(&sa, 0, sizeof sa);
sa.base_kind = NATIVE_ADDR_BASE_REG;
sa.base.reg = ARM_FP;
- sa.base_type = p->type;
+ sa.base_type = lty;
sa.offset = (i32)(a->saved_block_bytes + a->next_param_stack);
- arm_emit_mem(a, 1, src, sa, native_mem_for_type(t, p->type, part->size));
+ arm_emit_mem(a, 1, src, sa, native_mem_for_type(t, lty, part->size));
a->next_param_stack += 4u;
}
if (dst.kind == NATIVE_LOC_NONE) {
@@ -1078,34 +1125,51 @@ static void arm_bind_native_param(NativeTarget* t, const CGParamDesc* p,
memset(&home, 0, sizeof home);
home.base_kind = NATIVE_ADDR_BASE_FRAME;
home.base.frame = dst.v.frame;
- home.base_type = p->type;
+ home.base_type = lty;
home.offset = (i32)part->src_offset;
- arm_emit_mem(a, 0, src, home, native_mem_for_type(t, p->type, part->size));
+ arm_emit_mem(a, 0, src, home, native_mem_for_type(t, lty, part->size));
}
}
}
/* ============================ calls / returns ============================ */
-/* Outgoing stack-argument bytes for a call: int parts beyond r0..r3. */
+/* Advance the AAPCS argument-placement cursor across ONE argument `ai`, updating
+ * the core-register count *next_int (r0..r3) and the outgoing-stack byte count
+ * *stack. The single authority for the pair rule, shared by the stack-size
+ * accounting and the actual plan_call/bind_param emission so they cannot drift:
+ * - an 8-byte aligned pair rounds NCRN up to even and, if it does not fully
+ * fit in r0..r3, passes ENTIRELY on the 8-byte-aligned stack (no straddle);
+ * - every other part takes the next core register, else a 4-byte stack slot. */
+static void arm_arg_advance(const ABIArgInfo* ai, u32* next_int, u32* stack) {
+ u32 p;
+ if (ai->kind == ABI_ARG_IGNORE) return;
+ if (ai->kind == ABI_ARG_INDIRECT) {
+ if (*next_int < 4u) (*next_int)++;
+ else *stack += 4u;
+ return;
+ }
+ if (arm_arg_is_aligned_pair(ai)) {
+ *next_int = align_up_u32(*next_int, 2u);
+ if (*next_int + 2u > 4u) {
+ *next_int = 4u;
+ *stack = align_up_u32(*stack, 2u * ARM32_GPR_BYTES);
+ }
+ }
+ for (p = 0; p < ai->nparts; ++p) {
+ if (*next_int < 4u) (*next_int)++;
+ else *stack += 4u;
+ }
+}
+
+/* Outgoing stack-argument bytes for a call: parts beyond r0..r3 (8-aligned). */
static u32 arm_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
- u32 stack = 0, i, p;
+ u32 stack = 0, i;
if (!abi) return 0;
- for (i = 0; i < desc->nargs && i < abi->nparams; ++i) {
- const ABIArgInfo* ai = &abi->params[i];
- if (ai->kind == ABI_ARG_IGNORE) continue;
- if (ai->kind == ABI_ARG_INDIRECT) {
- if (next_int < 4u) next_int++;
- else stack += 4u;
- continue;
- }
- for (p = 0; p < ai->nparts; ++p) {
- if (next_int < 4u) next_int++;
- else stack += 4u;
- }
- }
+ for (i = 0; i < desc->nargs && i < abi->nparams; ++i)
+ arm_arg_advance(&abi->params[i], &next_int, &stack);
return align_up_u32(stack, 8u);
}
@@ -1113,23 +1177,12 @@ static u32 arm_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
int* variadic, u32* nparams) {
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fn_type);
u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
- u32 stack = 0, i, p;
+ u32 stack = 0, i;
if (variadic) *variadic = abi && abi->variadic;
if (nparams) *nparams = abi ? abi->nparams : 0u;
if (!abi) return 0;
- for (i = 0; i < abi->nparams; ++i) {
- const ABIArgInfo* ai = &abi->params[i];
- if (ai->kind == ABI_ARG_IGNORE) continue;
- if (ai->kind == ABI_ARG_INDIRECT) {
- if (next_int < 4u) next_int++;
- else stack += 4u;
- continue;
- }
- for (p = 0; p < ai->nparts; ++p) {
- if (next_int < 4u) next_int++;
- else stack += 4u;
- }
- }
+ for (i = 0; i < abi->nparams; ++i)
+ arm_arg_advance(&abi->params[i], &next_int, &stack);
return align_up_u32(stack, 8u);
}
@@ -1239,19 +1292,28 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
if (!ai || ai->kind == ABI_ARG_IGNORE) continue;
if (ai->kind == ABI_ARG_INDIRECT)
arm_panic(a, "indirect (by-ref aggregate) args are Phase 2");
+ /* AAPCS pair rule (mirrors arm_arg_advance / arm_bind_native_param): round
+ * NCRN to even before an 8-byte aligned pair; if it does not fully fit in
+ * r0..r3, the whole argument moves to the 8-byte-aligned stack. */
+ if (arm_arg_is_aligned_pair(ai)) {
+ next_int = align_up_u32(next_int, 2u);
+ if (next_int + 2u > 4u) {
+ next_int = 4u;
+ stack = align_up_u32(stack, 2u * ARM32_GPR_BYTES);
+ }
+ }
for (p = 0; p < ai->nparts; ++p) {
const ABIArgPart* part = &ai->parts[p];
+ KitCgTypeId lty = arm_part_scalar_type(part);
if (next_int < 4u) {
NativeArgMove* m = &moves[nmoves++];
- m->dst = native_loc_reg(desc->args[i].type, NATIVE_REG_INT,
- (Reg)(next_int++));
+ m->dst = native_loc_reg(lty, NATIVE_REG_INT, (Reg)(next_int++));
m->src = desc->args[i];
m->src_offset = part->src_offset;
m->size = part->size;
m->is_addr = 0;
} else {
- NativeLoc tmp = native_loc_reg(desc->args[i].type, NATIVE_REG_INT,
- ARM_SCRATCH);
+ NativeLoc tmp = native_loc_reg(lty, NATIVE_REG_INT, ARM_SCRATCH);
arm_load_part(t, tmp, desc->args[i], part->src_offset, part->size);
stack = align_up_u32(stack, 4u);
arm_store_outgoing(t, stack, tmp, part->size);
@@ -1271,7 +1333,10 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
u32 nr = 0, ni = 0, p;
for (p = 0; p < abi->ret.nparts; ++p) {
const ABIArgPart* part = &abi->ret.parts[p];
- KitCgTypeId pty = desc->results[0].type;
+ /* Per-lane type so each NDT-emitted result move is sized to the lane (4),
+ * not the full 8-byte scalar (which the 32-bit backend cannot single-step
+ * load/store). The ret pair is always r0:r1 (no even-rounding needed). */
+ KitCgTypeId pty = arm_part_scalar_type(part);
rets[nr].src = native_loc_reg(pty, NATIVE_REG_INT, (Reg)(ni++));
rets[nr].dst = desc->results[0];
if (rets[nr].dst.kind == NATIVE_LOC_FRAME)
@@ -1324,7 +1389,10 @@ static void arm_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
u32 ni = 0, p;
for (p = 0; p < abi->ret.nparts; ++p) {
const ABIArgPart* part = &abi->ret.parts[p];
- KitCgTypeId pty = value->type;
+ /* Per-lane type: an i64/double return is two i32 lanes in r0:r1; sizing
+ * each move from the lane type keeps the 32-bit backend off the 8-byte
+ * memory path. (See arm_part_scalar_type.) */
+ KitCgTypeId pty = arm_part_scalar_type(part);
rets[nr].src = *value;
if (rets[nr].src.kind == NATIVE_LOC_FRAME)
rets[nr].src =
@@ -1358,16 +1426,59 @@ static void arm_set_loc(NativeTarget* t, SrcLoc loc) {
#define ARM_UNIMPL(name) arm_panic(arm_of(t), name " not implemented in Phase 1")
+/* Assemble up to 4 little-endian bytes of `c` starting at byte `off` into a u32. */
+static u32 arm_const_lane(ConstBytes c, u32 off) {
+ u32 v = 0, i;
+ for (i = 0; i < 4u && off + i < c.size; ++i)
+ v |= (u32)c.bytes[off + i] << (i * 8u);
+ return v;
+}
+
static void arm_load_const(NativeTarget* t, NativeLoc dst, ConstBytes c) {
Arm32NativeTarget* a = arm_of(t);
- u32 v = 0, i;
/* Scalar constant <=4 bytes: reinterpret the ABI bytes (little-endian on
- * arm32) as a u32 and materialize via arm_load_imm (MOV.W/MVN.W/MOVW+MOVT).
- * 8-byte constants are the wide8 track's concern. */
- if (c.size > 4u)
- arm_panic(a, "8-byte load_const is the wide8 track");
- for (i = 0; i < c.size && i < 4u; ++i) v |= (u32)c.bytes[i] << (i * 8u);
- arm_load_imm(t, dst, (i64)(i32)v);
+ * arm32) as a u32 and materialize via arm_load_imm (MOV.W/MVN.W/MOVW+MOVT). */
+ if (c.size <= 4u) {
+ arm_load_imm(t, dst, (i64)(i32)arm_const_lane(c, 0u));
+ return;
+ }
+ /* 8-byte (i64/double) backstop. At -O0 the NDT lowers wide8 constants into two
+ * 32-bit lanes before they reach the backend (nd_load_const guards it), so
+ * this is only exercised by paths that hand the full 8-byte constant straight
+ * to a destination — materialize the low lane then the high lane, little-end
+ * first. A memory destination (frame/stack home) carries both lanes 4 bytes
+ * apart; a single 32-bit register can only hold the low lane (the high lane
+ * has no home in one NativeLoc), which matches a 32-bit reinterpret. */
+ if (c.size > 8u) arm_panic(a, "load_const wider than 8 bytes");
+ if (dst.kind == NATIVE_LOC_REG) {
+ arm_load_imm(t, dst, (i64)(i32)arm_const_lane(c, 0u));
+ return;
+ }
+ {
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ NativeLoc lane = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
+ NativeAddr home;
+ MemAccess mem = native_mem_for_type(t, i32t, 4u);
+ u32 off;
+ memset(&home, 0, sizeof home);
+ home.base_type = i32t;
+ if (dst.kind == NATIVE_LOC_FRAME) {
+ home.base_kind = NATIVE_ADDR_BASE_FRAME;
+ home.base.frame = dst.v.frame;
+ } else if (dst.kind == NATIVE_LOC_STACK) {
+ home.base_kind = NATIVE_ADDR_BASE_FRAME;
+ home.base.frame = dst.v.stack.slot;
+ home.offset = dst.v.stack.offset;
+ } else {
+ arm_panic(a, "8-byte load_const destination not lowered");
+ }
+ for (off = 0; off < c.size; off += 4u) {
+ NativeAddr d = home;
+ d.offset += (i32)off;
+ arm_load_imm(t, lane, (i64)(i32)arm_const_lane(c, off));
+ arm_emit_mem(a, 0, lane, d, mem);
+ }
+ }
}
static void arm_load_label_addr(NativeTarget* t, NativeLoc dst, MCLabel l) {
/* `&&label` address-take (computed goto / jump-table base): materialize the
