commit 4473fe52f7a6196d01f1c467b100c5ebe061517c
parent 319f6e7440208c4e0f7a79ec13548c5e32d6dce8
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 18:08:28 -0700
arm32 Phase 2: struct-by-value (INDIRECT) + varargs + LDREX/STREX atomics + inline asm
INDIRECT struct-by-value params/args/returns (copy via arm_copy_bytes), AAPCS32
varargs (r0-r3 spill area + 8-byte i64/double va_arg alignment), LDREX/STREX+DMB
<=4-byte atomics (rmw/cas), and a minimal inline-asm template runner. Integrated
against W8 (per-lane arg/return types) and O1 (saved_block_bytes): the arg-window
base now unifies saved_block_bytes + variadic spill so stack args resolve at -O0,
-O1, and variadic; kept W8's arm_arg_advance for signature_stack_bytes alongside
SVA's variadic arm_param_abi for the call path; deduped arm_dmb.
Diffstat:
2 files changed, 1120 insertions(+), 35 deletions(-)
diff --git a/src/arch/arm32/isa.h b/src/arch/arm32/isa.h
@@ -225,6 +225,33 @@ static inline u16 arm_mov_hi(u32 rd, u32 rm) {
return (u16)(0x4600u | ((rd >> 3) << 7) | (rm << 3) | (rd & 7u));
}
+/* --------- exclusive (atomic) loads / stores + barriers (ARMv7-M) ---------
+ * LDREX/STREX are word-only with a #imm8*4 offset (T1); the byte/half variants
+ * (LDREX{B,H}/STREX{B,H}) take no offset. STREX writes a 0/1 success status to
+ * Rd. DMB orders memory; CLREX clears the exclusive monitor. The atomic lowering
+ * uses these for <=4-byte _Atomic ops (8-byte routes to the spinlock libcall —
+ * M-profile has no LDREXD/STREXD). */
+static inline u32 arm_ldrex(u32 rt, u32 rn, u32 imm8) {
+ return arm_t32(0xe850u | rn, (rt << 12) | 0xf00u | (imm8 & 0xffu));
+}
+static inline u32 arm_strex(u32 rd, u32 rt, u32 rn, u32 imm8) {
+ return arm_t32(0xe840u | rn, (rt << 12) | (rd << 8) | (imm8 & 0xffu));
+}
+static inline u32 arm_ldrexb(u32 rt, u32 rn) {
+ return arm_t32(0xe8d0u | rn, (rt << 12) | 0xf4fu);
+}
+static inline u32 arm_strexb(u32 rd, u32 rt, u32 rn) {
+ return arm_t32(0xe8c0u | rn, (rt << 12) | 0xf40u | rd);
+}
+static inline u32 arm_ldrexh(u32 rt, u32 rn) {
+ return arm_t32(0xe8d0u | rn, (rt << 12) | 0xf5fu);
+}
+static inline u32 arm_strexh(u32 rd, u32 rt, u32 rn) {
+ return arm_t32(0xe8c0u | rn, (rt << 12) | 0xf50u | rd);
+}
+/* arm_dmb/arm_dsb/arm_isb are defined once in the barrier section below. */
+static inline u32 arm_clrex(void) { return arm_t32(0xf3bfu, 0x8f2fu); }
+
/* --------- branches (placeholders; immediate filled by reloc/label-fixup) ---
* The immediate fields use the canonical clang/gas "branch to self" pattern
* (encoded displacement -4). For ARM ELF the branch relocs are REL, so the
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -35,6 +35,7 @@
#include "cg/type.h"
#include "core/bytes.h"
#include "core/core.h"
+#include "core/pool.h"
#include "obj/obj.h"
extern void debug_emit_row(Debug*, ObjSecId text_section, u32 offset, SrcLoc);
@@ -54,6 +55,16 @@ extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
* small frame patches the first slot with one SUBW and NOP-fills the rest. */
#define ARM_NDT_SUB_WORDS 3u
+/* Incoming-arg window geometry (r7-relative). The prologue saves {r7, lr} (8
+ * bytes), so r7 sits 8 bytes below the caller's outgoing-arg block: incoming
+ * stack args begin at [r7 + 8]. A variadic callee additionally spills the GP
+ * argument registers r0..r3 (16 bytes) into a save area pushed ABOVE the saved
+ * pair, making them contiguous-below the named stack args; that shifts the
+ * stack-arg window down by 16 and locates the variadic register-save area at
+ * [r7 + 8]. */
+#define ARM_SAVED_PAIR_BYTES 8u
+#define ARM_VA_GP_SAVE_BYTES 16u /* r0..r3 spilled for variadics */
+
/* ============================ state ============================ */
typedef struct Arm32NativeTarget {
@@ -91,6 +102,12 @@ static _Noreturn void arm_panic(Arm32NativeTarget* a, const char* msg) {
static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0xfu; }
+/* Forward declarations for helpers referenced before their definitions (the
+ * aggregate copy is shared by INDIRECT param binding, INDIRECT returns, and the
+ * wide va_arg path). */
+static void arm_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
+ AggregateAccess acc);
+
/* ============================ emit ============================ */
static void arm_emit_t16(MCEmitter* mc, u16 hw) {
@@ -758,6 +775,18 @@ static u32 arm_frame_size(const Arm32NativeTarget* a) {
return align_up_u32(a->frame.cum_off + a->frame.max_outgoing, 8u);
}
+/* r7-relative offset where the incoming stack-argument window begins. The saved
+ * {r7, lr} pair is 8 bytes; a variadic callee also spilled r0..r3 (16 bytes) in
+ * a save area above the pair, pushing the named stack args 16 bytes higher. The
+ * spilled r0..r3 occupy [r7 + ARM_SAVED_PAIR_BYTES, r7 + arg_window_base). */
+static u32 arm_arg_window_base(const Arm32NativeTarget* a) {
+ /* saved_block_bytes covers the saved-register block ({r7,lr} = 8 at -O0, plus
+ * any callee-saves the -O1 known-frame prologue PUSHes); a variadic callee
+ * also spilled r0..r3 (16 bytes) just above it. Both push the named incoming
+ * stack args higher, so the window base accounts for them together. */
+ return a->saved_block_bytes + (a->is_variadic ? ARM_VA_GP_SAVE_BYTES : 0u);
+}
+
static void arm_reserve_callee_saves(NativeTarget* t, const u32* used_by_class,
u32 nclasses) {
native_frame_set_callee_saves(&arm_of(t)->frame, used_by_class, nclasses,
@@ -875,7 +904,12 @@ static void arm_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
* The deferred `SUB sp` is patched in arm_func_end once cum_off/max_outgoing
* are final. Reserve ARM_NDT_SUB_WORDS T32 slots so a large frame (> 4095) can
* be patched as an IP-staged MOVW/MOVT/SUB sequence; a small frame patches the
- * first slot with SUBW and fills the rest with NOPs. */
+ * first slot with SUBW and fills the rest with NOPs.
+ * A variadic callee first spills the GP arg registers r0..r3 into a save area
+ * just above the saved pair (so they are contiguous-below the named incoming
+ * stack args at [r7 + 8 + 16]); va_start/va_arg walk forward across both. */
+ if (a->is_variadic)
+ arm_emit_t32(mc, arm_push_w(0xfu)); /* PUSH {r0-r3} */
arm_emit_t32(mc, arm_push_w((1u << ARM_FP) | (1u << 14u))); /* PUSH {r7, lr} */
arm_emit_t16(mc, arm_mov_hi(ARM_FP, 13u)); /* MOV r7, sp */
a->prologue_sub_pos = mc_pos(mc);
@@ -1081,8 +1115,46 @@ static void arm_bind_native_param(NativeTarget* t, const CGParamDesc* p,
int pair = ai && arm_arg_is_aligned_pair(ai);
u32 i;
if (!ai || ai->kind == ABI_ARG_IGNORE) return;
- if (ai->kind == ABI_ARG_INDIRECT)
- arm_panic(a, "indirect (by-ref aggregate) params are Phase 2");
+ if (ai->kind == ABI_ARG_INDIRECT) {
+ /* Large aggregate passed by reference: the incoming arg is a pointer to the
+ * caller's copy. Copy it into the parameter's frame home so the body owns a
+ * private copy (matching the by-value semantics). */
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ NativeLoc src = native_loc_reg(i32t, NATIVE_REG_INT,
+ a->next_param_int < 4u
+ ? (Reg)a->next_param_int
+ : ARM_SCRATCH);
+ NativeAddr d_addr, from;
+ AggregateAccess access;
+ if (a->next_param_int < 4u) {
+ a->next_param_int++;
+ } else {
+ NativeAddr sa;
+ memset(&sa, 0, sizeof sa);
+ sa.base_kind = NATIVE_ADDR_BASE_REG;
+ sa.base.reg = ARM_FP;
+ sa.base_type = i32t;
+ sa.offset = (i32)(arm_arg_window_base(a) + a->next_param_stack);
+ arm_emit_mem(a, 1, src, sa, native_mem_for_type(t, i32t, 4));
+ a->next_param_stack += 4u;
+ }
+ if (dst.kind != NATIVE_LOC_FRAME)
+ arm_panic(a, "indirect parameter requires a frame destination");
+ memset(&d_addr, 0, sizeof d_addr);
+ d_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ d_addr.base.frame = dst.v.frame;
+ d_addr.base_type = p->type;
+ memset(&from, 0, sizeof from);
+ from.base_kind = NATIVE_ADDR_BASE_REG;
+ from.base.reg = loc_reg(src);
+ from.base_type = p->type;
+ memset(&access, 0, sizeof access);
+ access.type = p->type;
+ access.size = p->size ? p->size : (u32)cg_type_size(t->c, p->type);
+ access.align = p->align ? p->align : native_type_align(t, p->type);
+ arm_copy_bytes(t, d_addr, from, access);
+ return;
+ }
/* AAPCS: round NCRN up to even before an 8-byte aligned pair, and if the pair
* does not fully fit in the remaining core registers it passes ENTIRELY on the
* (8-byte-aligned) stack — it never straddles the r3/stack boundary. */
@@ -1109,7 +1181,7 @@ static void arm_bind_native_param(NativeTarget* t, const CGParamDesc* p,
sa.base_kind = NATIVE_ADDR_BASE_REG;
sa.base.reg = ARM_FP;
sa.base_type = lty;
- sa.offset = (i32)(a->saved_block_bytes + a->next_param_stack);
+ sa.offset = (i32)(arm_arg_window_base(a) + a->next_param_stack);
arm_emit_mem(a, 1, src, sa, native_mem_for_type(t, lty, part->size));
a->next_param_stack += 4u;
}
@@ -1136,8 +1208,8 @@ static void arm_bind_native_param(NativeTarget* t, const CGParamDesc* p,
/* Advance the AAPCS argument-placement cursor across ONE argument `ai`, updating
* the core-register count *next_int (r0..r3) and the outgoing-stack byte count
- * *stack. The single authority for the pair rule, shared by the stack-size
- * accounting and the actual plan_call/bind_param emission so they cannot drift:
+ * *stack. The single authority for the pair rule, shared by the signature
+ * stack-size accounting so it cannot drift from emission:
* - an 8-byte aligned pair rounds NCRN up to even and, if it does not fully
* fit in r0..r3, passes ENTIRELY on the 8-byte-aligned stack (no straddle);
* - every other part takes the next core register, else a 4-byte stack slot. */
@@ -1162,14 +1234,82 @@ static void arm_arg_advance(const ABIArgInfo* ai, u32* next_int, u32* stack) {
}
}
-/* Outgoing stack-argument bytes for a call: parts beyond r0..r3 (8-aligned). */
+/* Resolve the ABI classification of call argument `i`. Named params come from
+ * the callee's ABIFuncInfo; unnamed (variadic) args are synthesized here. A
+ * scalar wider than one GPR (8-byte i64 / soft double) becomes two 4-byte INT
+ * parts (AAPCS32 passes them in an even/odd register pair or 8-byte-aligned on
+ * the stack — the even/8-byte alignment is applied by the caller below). */
+static const ABIArgInfo* arm_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
+ const NativeCallDesc* desc, u32 i,
+ ABIArgInfo* scratch) {
+ u32 sz, align;
+ if (abi && i < abi->nparams) return &abi->params[i];
+ sz = native_type_size(t, desc->args[i].type);
+ align = native_type_align(t, desc->args[i].type);
+ memset(scratch, 0, sizeof *scratch);
+ scratch->kind = ABI_ARG_DIRECT;
+ if (sz > 4u) {
+ u32 nparts = (sz + 3u) / 4u, p;
+ ABIArgPart* parts = arena_zarray(t->c->tu, ABIArgPart, nparts);
+ for (p = 0; p < nparts; ++p) {
+ u32 off = p * 4u;
+ parts[p].cls = ABI_CLASS_INT;
+ parts[p].loc = ABI_LOC_REG;
+ parts[p].size = (sz - off) < 4u ? (sz - off) : 4u;
+ parts[p].align = 4u;
+ parts[p].src_offset = off;
+ }
+ scratch->nparts = (u16)nparts;
+ scratch->parts = parts;
+ return scratch;
+ }
+ scratch->nparts = 1;
+ {
+ ABIArgPart* part = arena_zarray(t->c->tu, ABIArgPart, 1);
+ part->cls = ABI_CLASS_INT;
+ part->loc = ABI_LOC_REG;
+ part->size = sz;
+ part->align = align;
+ scratch->parts = part;
+ }
+ return scratch;
+}
+
+/* An AAPCS32 8-byte argument (i64 / soft double) requires even-register pair
+ * alignment in the core registers and 8-byte stack alignment. Detect it from
+ * the argument's natural alignment (8) — robust for both the named two-part
+ * classification and the synthesized variadic form. */
+static int arm_arg_needs_8align(NativeTarget* t, const NativeCallDesc* desc,
+ u32 i) {
+ return native_type_align(t, desc->args[i].type) >= 8u;
+}
+
+/* Outgoing stack-argument bytes for a call: int parts beyond r0..r3. */
static u32 arm_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
- u32 stack = 0, i;
+ u32 stack = 0, i, p;
+ ABIArgInfo scratch;
if (!abi) return 0;
- for (i = 0; i < desc->nargs && i < abi->nparams; ++i)
- arm_arg_advance(&abi->params[i], &next_int, &stack);
+ for (i = 0; i < desc->nargs; ++i) {
+ const ABIArgInfo* ai = arm_param_abi(t, abi, desc, i, &scratch);
+ int eight = arm_arg_needs_8align(t, desc, i);
+ if (ai->kind == ABI_ARG_IGNORE) continue;
+ if (ai->kind == ABI_ARG_INDIRECT) {
+ if (next_int < 4u) next_int++;
+ else {
+ stack = align_up_u32(stack, 4u);
+ stack += 4u;
+ }
+ continue;
+ }
+ if (eight) next_int = align_up_u32(next_int, 2u); /* even-pair */
+ for (p = 0; p < ai->nparts; ++p) {
+ if (p == 0 && eight && next_int >= 4u) stack = align_up_u32(stack, 8u);
+ if (next_int < 4u) next_int++;
+ else stack += 4u;
+ }
+ }
return align_up_u32(stack, 8u);
}
@@ -1235,9 +1375,28 @@ static void arm_store_outgoing(NativeTarget* t, u32 stack_off, NativeLoc src,
arm_emit_mem(arm_of(t), 0, src, addr, mem);
}
+/* Materialize the address of a frame/stack-resident NativeLoc into `dst` (used
+ * to pass a large INDIRECT aggregate argument by reference). */
+static void arm_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
+ NativeAddr addr;
+ memset(&addr, 0, sizeof addr);
+ if (src.kind == NATIVE_LOC_FRAME) {
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ addr.base.frame = src.v.frame;
+ } else if (src.kind == NATIVE_LOC_STACK) {
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ addr.base.frame = src.v.stack.slot;
+ addr.offset = src.v.stack.offset;
+ } else {
+ arm_panic(arm_of(t), "address-of non-memory location");
+ }
+ addr.base_type = dst.type;
+ arm_load_addr(t, dst, addr);
+}
+
static void arm_emit_one_arg_move(NativeTarget* t, const NativeArgMove* m) {
if (m->is_addr)
- arm_load_addr(t, m->dst, m->src.v.addr); /* unused in smoke set */
+ arm_addr_of_loc(t, m->dst, m->src);
else
arm_load_part(t, m->dst, m->src, m->src_offset, m->size);
}
@@ -1287,20 +1446,37 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
u32 stack = 0, nmoves = 0, i, p;
NativeArgMove moves[ARM_MAX_REG_ARG_MOVES];
KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ ABIArgInfo scratch;
for (i = 0; i < desc->nargs; ++i) {
- const ABIArgInfo* ai = i < abi->nparams ? &abi->params[i] : NULL;
- if (!ai || ai->kind == ABI_ARG_IGNORE) continue;
- if (ai->kind == ABI_ARG_INDIRECT)
- arm_panic(a, "indirect (by-ref aggregate) args are Phase 2");
- /* AAPCS pair rule (mirrors arm_arg_advance / arm_bind_native_param): round
- * NCRN to even before an 8-byte aligned pair; if it does not fully fit in
- * r0..r3, the whole argument moves to the 8-byte-aligned stack. */
- if (arm_arg_is_aligned_pair(ai)) {
- next_int = align_up_u32(next_int, 2u);
- if (next_int + 2u > 4u) {
- next_int = 4u;
- stack = align_up_u32(stack, 2u * ARM32_GPR_BYTES);
+ const ABIArgInfo* ai = arm_param_abi(t, abi, desc, i, &scratch);
+ int eight = arm_arg_needs_8align(t, desc, i);
+ if (ai->kind == ABI_ARG_IGNORE) continue;
+ if (ai->kind == ABI_ARG_INDIRECT) {
+ /* Pass a large aggregate by reference: a pointer to the caller's copy
+ * (the callee makes its own copy in bind_param). The pointer rides a
+ * core reg when one is free, else the outgoing stack. */
+ if (next_int < 4u) {
+ NativeArgMove* m = &moves[nmoves++];
+ m->dst = native_loc_reg(i32t, NATIVE_REG_INT, (Reg)(next_int++));
+ m->src = desc->args[i];
+ m->src_offset = 0;
+ m->size = 4u;
+ m->is_addr = 1;
+ } else {
+ NativeLoc ptr = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
+ arm_addr_of_loc(t, ptr, desc->args[i]);
+ stack = align_up_u32(stack, 4u);
+ arm_store_outgoing(t, stack, ptr, 4u);
+ stack += 4u;
}
+ continue;
+ }
+ /* AAPCS32: an 8-byte arg starts in an even register pair; if it cannot fit
+ * (an odd reg remains, or none), the whole value goes to 8-byte-aligned
+ * stack. */
+ if (eight) {
+ next_int = align_up_u32(next_int, 2u);
+ if (next_int + ai->nparts > 4u) next_int = 4u; /* spill the whole arg */
}
for (p = 0; p < ai->nparts; ++p) {
const ABIArgPart* part = &ai->parts[p];
@@ -1315,7 +1491,8 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
} else {
NativeLoc tmp = native_loc_reg(lty, NATIVE_REG_INT, ARM_SCRATCH);
arm_load_part(t, tmp, desc->args[i], part->src_offset, part->size);
- stack = align_up_u32(stack, 4u);
+ if (p == 0 && eight) stack = align_up_u32(stack, 8u);
+ else stack = align_up_u32(stack, 4u);
arm_store_outgoing(t, stack, tmp, part->size);
stack += 4u;
}
@@ -1383,7 +1560,40 @@ static void arm_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
u32 nr = 0;
if (value) rets = arena_zarray(t->c->tu, NativeCallPlanRet, 4);
if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
- arm_panic(a, "indirect (by-ref aggregate) return is Phase 2");
+ /* Large aggregate return: copy the value into the caller-provided
+ * destination via the hidden sret pointer spilled at entry. The body's
+ * return slot is *value; load the saved pointer into ip, then copy. No
+ * register return parts. */
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ NativeLoc dstp = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
+ NativeLoc saved = native_loc_stack(i32t, a->sret_ptr_slot, 0);
+ NativeAddr dst_addr, src_addr;
+ AggregateAccess access;
+ arm_load_part(t, dstp, saved, 0, 4u);
+ memset(&dst_addr, 0, sizeof dst_addr);
+ dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
+ dst_addr.base.reg = ARM_SCRATCH;
+ dst_addr.base_type = value->type;
+ memset(&src_addr, 0, sizeof src_addr);
+ if (value->kind == NATIVE_LOC_FRAME) {
+ src_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ src_addr.base.frame = value->v.frame;
+ } else if (value->kind == NATIVE_LOC_STACK) {
+ src_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ src_addr.base.frame = value->v.stack.slot;
+ src_addr.offset = value->v.stack.offset;
+ } else {
+ arm_panic(a, "indirect return value must be in memory");
+ }
+ src_addr.base_type = value->type;
+ memset(&access, 0, sizeof access);
+ access.type = value->type;
+ access.size = (u32)cg_type_size(t->c, value->type);
+ access.align = native_type_align(t, value->type);
+ arm_copy_bytes(t, dst_addr, src_addr, access);
+ *out_rets = NULL;
+ *out_nrets = 0;
+ return;
}
if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
u32 ni = 0, p;
@@ -1618,21 +1828,87 @@ static void arm_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
(void)align;
ARM_UNIMPL("alloca");
}
+/* ============================ atomics ============================ */
+/* ARMv7-M atomics over LDREX/STREX (word/halfword/byte) + DMB. The cg layer
+ * routes 8-byte _Atomic to the spinlock libcall (atomic_lock_free_max=4), so
+ * only <=4-byte ops reach here.
+ *
+ * Register budget: lr (ARM_TMP) holds the access ADDRESS. The NDT keeps the
+ * value operands in the r0..r3 pool OR in ip (ARM_SCRATCH, its materialization
+ * scratch), so the atomic sequences must NOT use ip as a temp — they would
+ * clobber an operand that happens to live there. STREX additionally needs its
+ * status, value and base registers mutually distinct. The rmw/cas loops
+ * therefore borrow callee-saved registers (r4/r5, free at -O0, push/pop-balanced
+ * so safe at -O1) for the private temporaries (new value + STREX status). */
+
+static int arm_order_acquire(KitCgMemOrder o) {
+ return o == KIT_CG_MO_CONSUME || o == KIT_CG_MO_ACQUIRE ||
+ o == KIT_CG_MO_ACQ_REL || o == KIT_CG_MO_SEQ_CST;
+}
+static int arm_order_release(KitCgMemOrder o) {
+ return o == KIT_CG_MO_RELEASE || o == KIT_CG_MO_ACQ_REL ||
+ o == KIT_CG_MO_SEQ_CST;
+}
+
+/* Materialize the access address into lr (ARM_TMP) and return it. lr is never an
+ * NDT operand and is dead in the body, so it never collides with the value
+ * operands (which the value-cache keeps in r0..r3 / ip). The incoming NativeAddr
+ * may be a frame slot or a register; load_addr lands the pointer in lr, which
+ * the LDREX/STREX sequence then uses as the base. */
+static u32 arm_atomic_addr_reg(Arm32NativeTarget* a, NativeAddr addr) {
+ NativeLoc dst =
+ native_loc_reg(builtin_id(KIT_CG_BUILTIN_I32), NATIVE_REG_INT, ARM_TMP);
+ arm_load_addr(&a->base, dst, addr);
+ return ARM_TMP;
+}
+
+/* LDREX-family load for the access width (word / halfword / byte). */
+static void arm_emit_ldrex(Arm32NativeTarget* a, u32 sz, u32 rt, u32 base) {
+ MCEmitter* mc = a->base.mc;
+ if (sz >= 4u) arm_emit_t32(mc, arm_ldrex(rt, base, 0u));
+ else if (sz == 2u) arm_emit_t32(mc, arm_ldrexh(rt, base));
+ else arm_emit_t32(mc, arm_ldrexb(rt, base));
+}
+/* STREX-family store; writes the 0/1 success status into `rd`. */
+static void arm_emit_strex(Arm32NativeTarget* a, u32 sz, u32 rd, u32 rt,
+ u32 base) {
+ MCEmitter* mc = a->base.mc;
+ if (sz >= 4u) arm_emit_t32(mc, arm_strex(rd, rt, base, 0u));
+ else if (sz == 2u) arm_emit_t32(mc, arm_strexh(rd, rt, base));
+ else arm_emit_t32(mc, arm_strexb(rd, rt, base));
+}
+
static void arm_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
MemAccess mem, KitCgMemOrder order) {
- (void)dst;
- (void)addr;
- (void)mem;
- (void)order;
- ARM_UNIMPL("atomic_load");
+ Arm32NativeTarget* a = arm_of(t);
+ MCEmitter* mc = t->mc;
+ u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
+ u32 base = arm_atomic_addr_reg(a, addr);
+ NativeAddr m;
+ if (sz > 4u) arm_panic(a, "8-byte atomic load not lowered (spinlock libcall)");
+ memset(&m, 0, sizeof m);
+ m.base_kind = NATIVE_ADDR_BASE_REG;
+ m.base.reg = base;
+ m.base_type = dst.type;
+ arm_emit_mem(a, 1, dst, m, mem); /* plain LDR/LDRH/LDRB is atomic for <=4B */
+ if (arm_order_acquire(order)) arm_emit_t32(mc, arm_dmb(0xfu));
}
+
static void arm_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc v,
MemAccess mem, KitCgMemOrder order) {
- (void)addr;
- (void)v;
- (void)mem;
- (void)order;
- ARM_UNIMPL("atomic_store");
+ Arm32NativeTarget* a = arm_of(t);
+ MCEmitter* mc = t->mc;
+ u32 sz = mem.size ? mem.size : native_type_size(t, v.type);
+ u32 base = arm_atomic_addr_reg(a, addr);
+ NativeAddr m;
+ if (sz > 4u) arm_panic(a, "8-byte atomic store not lowered (spinlock libcall)");
+ if (arm_order_release(order)) arm_emit_t32(mc, arm_dmb(0xfu));
+ memset(&m, 0, sizeof m);
+ m.base_kind = NATIVE_ADDR_BASE_REG;
+ m.base.reg = base;
+ m.base_type = v.type;
+ arm_emit_mem(a, 0, v, m, mem); /* plain STR/STRH/STRB is atomic for <=4B */
+ if (order == KIT_CG_MO_SEQ_CST) arm_emit_t32(mc, arm_dmb(0xfu));
}
/* The ARMv7-M barrier option for a full-system barrier ("sy"). 64-bit clz/ctz/
* bswap on a 32-bit target are already routed to __*di2 libcalls by cg, so the
@@ -1640,6 +1916,249 @@ static void arm_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc v,
* stay in lockstep with arm32_supports_intrinsic (arch.c). */
#define ARM_BARRIER_SY 0xfu
+static void arm_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
+ NativeAddr addr, NativeLoc val, MemAccess mem,
+ KitCgMemOrder order) {
+ Arm32NativeTarget* a = arm_of(t);
+ MCEmitter* mc = t->mc;
+ u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
+ u32 rd = loc_reg(dst);
+ u32 rv = loc_reg(val);
+ u32 base, newv = 4u, status = 5u; /* r4 = new value, r5 = STREX status */
+ MCLabel retry = mc_label_new(mc);
+ if (sz > 4u) arm_panic(a, "8-byte atomic rmw not lowered (spinlock libcall)");
+ /* Borrow r4 (new value) + r5 (STREX status) across a balanced push/pop so the
+ * sequence touches no NDT-allocated register beyond lr=addr, rd=old value
+ * (returned), rv=operand. ip is left alone (an operand may live there). */
+ arm_emit_t32(mc, arm_push_w((1u << newv) | (1u << status)));
+ base = arm_atomic_addr_reg(a, addr); /* lr */
+ if (arm_order_release(order)) arm_emit_t32(mc, arm_dmb(0xfu));
+ /* Retry loop: rd = *base (LDREX); r4 = rd OP val; STREX r5,r4,[base]; retry
+ * while r5 != 0. */
+ mc_label_place(mc, retry);
+ arm_emit_ldrex(a, sz, rd, base);
+ switch (op) {
+ case KIT_CG_ATOMIC_XCHG:
+ arm_emit_t16(mc, arm_mov_hi(newv, rv));
+ break;
+ case KIT_CG_ATOMIC_ADD:
+ arm_emit_t32(mc, arm_add_reg(newv, rd, rv));
+ break;
+ case KIT_CG_ATOMIC_SUB:
+ arm_emit_t32(mc, arm_sub_reg(newv, rd, rv));
+ break;
+ case KIT_CG_ATOMIC_AND:
+ arm_emit_t32(mc, arm_and_reg(newv, rd, rv));
+ break;
+ case KIT_CG_ATOMIC_OR:
+ arm_emit_t32(mc, arm_orr_reg(newv, rd, rv));
+ break;
+ case KIT_CG_ATOMIC_XOR:
+ arm_emit_t32(mc, arm_eor_reg(newv, rd, rv));
+ break;
+ case KIT_CG_ATOMIC_NAND:
+ arm_emit_t32(mc, arm_and_reg(newv, rd, rv));
+ arm_emit_t32(mc, arm_mvn_reg(newv, newv));
+ break;
+ default:
+ arm_panic(a, "unsupported atomic rmw op");
+ }
+ arm_emit_strex(a, sz, status, newv, base); /* STREX r5, r4, [lr] */
+ arm_emit_t32(mc, arm_cmp_imm(status, 0u)); /* CMP r5, #0 */
+ arm_emit_t32(mc, arm_b_cond_w(ARM_CC_NE)); /* BNE retry */
+ mc_emit_label_ref(mc, retry, R_ARM_THM_JUMP19, 4, 0);
+ if (arm_order_acquire(order)) arm_emit_t32(mc, arm_dmb(0xfu));
+ arm_emit_t32(mc, arm_pop_w((1u << newv) | (1u << status)));
+}
+
+static void arm_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
+ NativeAddr addr, NativeLoc expected,
+ NativeLoc desired, MemAccess mem,
+ KitCgMemOrder success, KitCgMemOrder failure) {
+ Arm32NativeTarget* a = arm_of(t);
+ MCEmitter* mc = t->mc;
+ u32 sz = mem.size ? mem.size : native_type_size(t, prior.type);
+ u32 rprior = loc_reg(prior);
+ u32 rexp = loc_reg(expected);
+ u32 rdes = loc_reg(desired);
+ u32 rok = loc_reg(ok);
+ u32 base, status = 4u; /* r4 = STREX status (never ip — an operand may live there) */
+ u32 enc0, enc1;
+ MCLabel retry = mc_label_new(mc);
+ MCLabel fail = mc_label_new(mc);
+ MCLabel done = mc_label_new(mc);
+ (void)failure;
+ if (sz > 4u) arm_panic(a, "8-byte atomic cas not lowered (spinlock libcall)");
+ thumb_expand_imm_encode(0u, &enc0);
+ thumb_expand_imm_encode(1u, &enc1);
+ arm_emit_t32(mc, arm_push_w(1u << status));
+ base = arm_atomic_addr_reg(a, addr); /* lr */
+ if (arm_order_release(success)) arm_emit_t32(mc, arm_dmb(0xfu));
+ /* retry: prior = *base (LDREX); if prior != expected goto fail; STREX
+ * r4,desired,[base]; retry on failure; ok = 1; goto done. fail: clear the
+ * monitor (CLREX) and ok = 0. */
+ mc_label_place(mc, retry);
+ arm_emit_ldrex(a, sz, rprior, base);
+ arm_emit_t32(mc, arm_cmp_reg(rprior, rexp));
+ arm_emit_t32(mc, arm_b_cond_w(ARM_CC_NE)); /* BNE fail */
+ mc_emit_label_ref(mc, fail, R_ARM_THM_JUMP19, 4, 0);
+ arm_emit_strex(a, sz, status, rdes, base);
+ arm_emit_t32(mc, arm_cmp_imm(status, 0u));
+ arm_emit_t32(mc, arm_b_cond_w(ARM_CC_NE)); /* BNE retry */
+ mc_emit_label_ref(mc, retry, R_ARM_THM_JUMP19, 4, 0);
+ arm_emit_t32(mc, arm_mov_imm(rok, enc1)); /* ok = 1 */
+ arm_emit_t32(mc, arm_b_w()); /* B done */
+ mc_emit_label_ref(mc, done, R_ARM_THM_JUMP24, 4, 0);
+ mc_label_place(mc, fail);
+ arm_emit_t32(mc, arm_clrex()); /* drop the exclusive reservation */
+ arm_emit_t32(mc, arm_mov_imm(rok, enc0)); /* ok = 0 */
+ mc_label_place(mc, done);
+ if (arm_order_acquire(success)) arm_emit_t32(mc, arm_dmb(0xfu));
+ arm_emit_t32(mc, arm_pop_w(1u << status));
+}
+
+static void arm_fence(NativeTarget* t, KitCgMemOrder order) {
+ if (order == KIT_CG_MO_RELAXED) return;
+ arm_emit_t32(t->mc, arm_dmb(0xfu));
+}
+
+/* ============================ varargs ============================ */
+/* AAPCS32 va_list is a plain 4-byte pointer to the next argument slot. The
+ * prologue spilled the unconsumed GP arg registers r0..r3 into a save area just
+ * above the saved {r7, lr} pair (at [r7 + ARM_SAVED_PAIR_BYTES]); the named
+ * incoming stack args follow contiguously at [r7 + arg_window_base], so a
+ * uniform 4-byte stride (8-byte-aligned for i64/double) walks both regions.
+ * `ap` is a NativeAddr addressing the va_list object itself. */
+
+static void arm_va_start_core(Arm32NativeTarget* a, NativeAddr ap) {
+ NativeTarget* t = &a->base;
+ MCEmitter* mc = t->mc;
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ NativeLoc ptr = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
+ ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
+ if (vai.kind != ABI_VA_LIST_POINTER) arm_panic(a, "unsupported va_list layout");
+ if (!a->is_variadic) arm_panic(a, "va_start: function not variadic");
+ /* *ap = r7 + ARM_SAVED_PAIR_BYTES + next_param_int*4 (skip the named GP slots
+ * already consumed by the fixed params). lr (ARM_TMP) is the staging temp. */
+ arm_emit_t32(mc, arm_add_imm12(ARM_TMP, ARM_FP,
+ ARM_SAVED_PAIR_BYTES + a->next_param_int * 4u));
+ arm_emit_mem(a, 0, ptr, ap, native_mem_for_type(t, i32t, 4));
+}
+
+/* Whether a va_arg value is too wide to move through one core register (an
+ * 8-byte i64 / soft-double): it occupies two GP slots and is copied straight
+ * from the save area into its destination memory. */
+static int arm_va_arg_is_wide(NativeTarget* t, KitCgTypeId type) {
+ return native_type_size(t, type) > 4u;
+}
+
+/* Wide / aggregate va_arg: read the cursor, 8-byte-align it (i64/double slots
+ * are 8-byte aligned on the stack per AAPCS32), advance past the whole span,
+ * then byte-copy from the (aligned) cursor into the destination memory. ip holds
+ * the cursor across the copy; arm_copy_bytes uses lr internally. */
+static void arm_va_arg_wide(Arm32NativeTarget* a, NativeAddr dst, NativeAddr ap,
+ u32 sz) {
+ NativeTarget* t = &a->base;
+ MCEmitter* mc = t->mc;
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ u32 span = align_up_u32(sz, 4u);
+ u32 enc7;
+ NativeLoc cur = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
+ NativeLoc nxt = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
+ NativeAddr src;
+ AggregateAccess acc;
+ /* cur = *ap. */
+ arm_emit_mem(a, 1, cur, ap, native_mem_for_type(t, i32t, 4));
+ /* 8-byte align the cursor: cur = (cur + 7) & ~7. ~7 (0xFFFFFFF8) is not a
+ * Thumb modified immediate, so clear the low 3 bits with BIC #7 (op4=1; #7 IS
+ * encodable). */
+ arm_emit_t32(mc, arm_add_imm12(ARM_SCRATCH, ARM_SCRATCH, 7u));
+ thumb_expand_imm_encode(7u, &enc7);
+ arm_emit_t32(mc, arm_dp_imm(1u, 0u, ARM_SCRATCH, ARM_SCRATCH, enc7)); /* BIC */
+ /* *ap = cur + span. */
+ arm_emit_t32(mc, arm_add_imm12(ARM_TMP, ARM_SCRATCH, span));
+ arm_emit_mem(a, 0, nxt, ap, native_mem_for_type(t, i32t, 4));
+ /* Copy sz bytes from [cur] to the destination. */
+ memset(&src, 0, sizeof src);
+ src.base_kind = NATIVE_ADDR_BASE_REG;
+ src.base.reg = ARM_SCRATCH;
+ src.base_type = i32t;
+ memset(&acc, 0, sizeof acc);
+ acc.type = i32t;
+ acc.size = sz;
+ acc.align = 4u;
+ arm_copy_bytes(t, dst, src, acc);
+}
+
+static void arm_va_arg_core(Arm32NativeTarget* a, NativeLoc dst, NativeAddr ap,
+ KitCgTypeId type) {
+ NativeTarget* t = &a->base;
+ MCEmitter* mc = t->mc;
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ u32 sz = native_type_size(t, type);
+ NativeLoc cur = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
+ NativeAddr from;
+ ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
+ if (vai.kind != ABI_VA_LIST_POINTER) arm_panic(a, "unsupported va_list layout");
+ if (dst.kind != NATIVE_LOC_REG) arm_panic(a, "va_arg destination must be reg");
+ /* cur = *ap; load value from [cur]; *ap = cur + 4 (one GP-slot stride). */
+ arm_emit_mem(a, 1, cur, ap, native_mem_for_type(t, i32t, 4));
+ memset(&from, 0, sizeof from);
+ from.base_kind = NATIVE_ADDR_BASE_REG;
+ from.base.reg = ARM_SCRATCH;
+ from.base_type = type;
+ arm_emit_mem(a, 1, dst, from, native_mem_for_type(t, type, sz));
+ arm_emit_t32(mc, arm_add_imm12(ARM_SCRATCH, ARM_SCRATCH, 4u));
+ arm_emit_mem(a, 0, cur, ap, native_mem_for_type(t, i32t, 4));
+}
+
+static void arm_va_copy_core(Arm32NativeTarget* a, NativeAddr dst_ap,
+ NativeAddr src_ap) {
+ NativeTarget* t = &a->base;
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ NativeLoc tmp = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
+ arm_emit_mem(a, 1, tmp, src_ap, native_mem_for_type(t, i32t, 4));
+ arm_emit_mem(a, 0, tmp, dst_ap, native_mem_for_type(t, i32t, 4));
+}
+
+/* ---- NativeTarget (optimizer) hooks: ap is a materialized register ---- */
+static NativeAddr arm_va_addr_from_ptr(NativeLoc ap_ptr) {
+ NativeAddr addr;
+ memset(&addr, 0, sizeof addr);
+ addr.base_kind = NATIVE_ADDR_BASE_REG;
+ addr.cls = NATIVE_REG_INT;
+ addr.base.reg = ap_ptr.v.reg;
+ addr.base_type = ap_ptr.type;
+ return addr;
+}
+static void arm_va_start_native(NativeTarget* t, NativeLoc ap_ptr) {
+ arm_va_start_core(arm_of(t), arm_va_addr_from_ptr(ap_ptr));
+}
+static void arm_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
+ KitCgTypeId type) {
+ Arm32NativeTarget* a = arm_of(t);
+ if (arm_va_arg_is_wide(t, type)) {
+ NativeAddr dstm;
+ memset(&dstm, 0, sizeof dstm);
+ dstm.base_kind = NATIVE_ADDR_BASE_REG;
+ dstm.base.reg = dst.v.reg;
+ dstm.base_type = type;
+ arm_va_arg_wide(a, dstm, arm_va_addr_from_ptr(ap_ptr),
+ native_type_size(t, type));
+ return;
+ }
+ arm_va_arg_core(a, dst, arm_va_addr_from_ptr(ap_ptr), type);
+}
+static void arm_va_end_native(NativeTarget* t, NativeLoc ap_ptr) {
+ (void)t;
+ (void)ap_ptr;
+}
+static void arm_va_copy_native(NativeTarget* t, NativeLoc dst_ap_ptr,
+ NativeLoc src_ap_ptr) {
+ arm_va_copy_core(arm_of(t), arm_va_addr_from_ptr(dst_ap_ptr),
+ arm_va_addr_from_ptr(src_ap_ptr));
+}
+
static void arm_intrinsic(NativeTarget* t, IntrinKind kind,
const NativeLoc* dsts, u32 ndst, const NativeLoc* args,
u32 narg) {
@@ -1709,6 +2228,446 @@ static void arm_intrinsic(NativeTarget* t, IntrinKind kind,
arm_panic(a, "intrinsic not lowered (overflow/FMA/syscall/etc. are other tracks)");
}
+/* ============================ inline asm ============================ */
+/* The descriptor-driven Thumb-2 textual assembler is a separate Phase-2
+ * deliverable (ARM32.md "Standalone assembler frontend"); until it lands the
+ * inline-asm template runner here covers the constraints the kit ecosystem
+ * actually emits at -O0: register/memory operands, the "r"/"l" classes, the
+ * memory/cc clobbers and "0"-style matching constraints (all handled by the
+ * shared native_asm_* binder), and a minimal mnemonic set in the template
+ * itself — register MOV (`mov`/`movs`), the barriers (`dmb`/`dsb`/`isb`), and
+ * `nop`. The operand binding/allocation/save-restore is the arch-neutral shared
+ * driver; only operand rendering + the few mnemonics are arch-specific. Any
+ * other mnemonic is a clean panic (it needs the full assembler). */
+
+enum {
+ ARM_INLINE_OPK_REG = 0xf0u, /* bound register pseudo-operand (Operand.kind) */
+ ARM_INLINE_OPCLS_INT = 0u,
+ ARM_INLINE_OPCLS_FP = 1u, /* Operand.pad[0] discriminator */
+};
+
+_Noreturn static void arm_asm_panic_at(Compiler* c, SrcLoc loc,
+ const char* msg) {
+ compiler_panic(c, loc, "arm32 inline asm: %s", msg);
+}
+_Noreturn static void arm_asm_panic(NativeDirectTarget* d, const char* msg) {
+ arm_asm_panic_at(d->base.c, d->loc, msg);
+}
+
+static void arm_asm_bound_reg(Operand* out, KitCgTypeId type,
+ NativeAllocClass cls, Reg reg) {
+ memset(out, 0, sizeof *out);
+ out->kind = ARM_INLINE_OPK_REG;
+ out->pad[0] = (cls == NATIVE_REG_FP) ? ARM_INLINE_OPCLS_FP : ARM_INLINE_OPCLS_INT;
+ out->type = type;
+ out->v.local = (CGLocal)reg;
+}
+static void arm_asm_bound_mem(Operand* out, KitCgTypeId type, Reg base) {
+ memset(out, 0, sizeof *out);
+ out->kind = OPK_INDIRECT;
+ out->type = type;
+ out->v.ind.base = (CGLocal)base;
+ out->v.ind.index = CG_LOCAL_NONE;
+}
+
+/* Parse one clobber name ("r0".."r12", "lr"/"sp", "cc", "memory") into a
+ * register mask bit; cc/memory contribute no register. */
+static void arm_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
+ u32 nclob, u32* int_mask, u32* fp_mask) {
+ *int_mask = 0;
+ *fp_mask = 0;
+ for (u32 i = 0; i < nclob; ++i) {
+ Slice s = pool_slice(c->global, clobbers[i]);
+ char buf[16];
+ uint32_t dwarf;
+ if (!s.s || !s.len || s.len >= sizeof buf) continue;
+ memcpy(buf, s.s, s.len);
+ buf[s.len] = '\0';
+ if (!strcmp(buf, "cc") || !strcmp(buf, "memory")) continue;
+ if (arm32_register_index(buf, &dwarf) == 0 && dwarf <= 15u)
+ *int_mask |= 1u << dwarf;
+ }
+ (void)loc;
+}
+
+/* Allocate a free operand register from the core pool, honoring allowed_mask and
+ * the running used set. r7 (fp), r12 (ip), r13 (sp), r14 (lr), r15 (pc) are
+ * never handed out. */
+static Reg arm_asm_alloc_reg(NativeDirectTarget* d, NativeAllocClass cls,
+ u32 allowed_mask, u32* used_int, u32* used_fp) {
+ static const Reg int_pool[] = {0u, 1u, 2u, 3u, 4u, 5u, 6u, 8u, 9u, 10u, 11u};
+ (void)used_fp;
+ if (cls != NATIVE_REG_INT) arm_asm_panic(d, "no FP registers (soft-float)");
+ for (u32 i = 0; i < sizeof int_pool / sizeof int_pool[0]; ++i) {
+ Reg r = int_pool[i];
+ if (allowed_mask && (allowed_mask & (1u << r)) == 0) continue;
+ if ((*used_int & (1u << r)) != 0) continue;
+ *used_int |= 1u << r;
+ return r;
+ }
+ arm_asm_panic(d, "out of registers for asm operands");
+}
+
+/* Operand-address resolution for the direct (-O0) path (mirrors aa64). */
+static NativeAddr arm_direct_addr(NativeDirectTarget* d, Operand op) {
+ NativeAddr addr;
+ memset(&addr, 0, sizeof addr);
+ switch ((OpKind)op.kind) {
+ case OPK_LOCAL:
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ addr.base.frame = d->locals[op.v.local - 1u].home;
+ addr.base_type = op.type;
+ return addr;
+ case OPK_INDIRECT:
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
+ addr.base.frame = d->locals[op.v.ind.base - 1u].home;
+ addr.cls = d->locals[op.v.ind.base - 1u].cls;
+ addr.base_type = d->locals[op.v.ind.base - 1u].type;
+ addr.offset = op.v.ind.ofs;
+ return addr;
+ default:
+ arm_asm_panic(d, "operand is not addressable");
+ }
+}
+static NativeAddr arm_direct_materialize_addr(NativeDirectTarget* d,
+ Operand op) {
+ NativeAddr addr = arm_direct_addr(d, op);
+ if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
+ NativeLoc base = native_loc_reg(addr.base_type, NATIVE_REG_INT, ARM_SCRATCH);
+ NativeAddr load;
+ memset(&load, 0, sizeof load);
+ load.base_kind = NATIVE_ADDR_BASE_FRAME;
+ load.base.frame = addr.base.frame;
+ load.base_type = addr.base_type;
+ arm_emit_mem(arm_of(d->native), 1, base, load,
+ native_mem_for_type(d->native, addr.base_type, 4));
+ addr.base_kind = NATIVE_ADDR_BASE_REG;
+ addr.base.reg = ARM_SCRATCH;
+ }
+ return addr;
+}
+
+static void arm_direct_load_operand_to_reg(NativeDirectTarget* d, Operand op,
+ NativeLoc dst) {
+ NativeAddr addr;
+ memset(&addr, 0, sizeof addr);
+ switch ((OpKind)op.kind) {
+ case OPK_IMM:
+ d->native->load_imm(d->native, dst, op.v.imm);
+ return;
+ case OPK_LOCAL:
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ addr.base.frame = d->locals[op.v.local - 1u].home;
+ addr.base_type = op.type;
+ arm_emit_mem(arm_of(d->native), 1, dst, addr,
+ native_mem_for_type(d->native, op.type, 0));
+ return;
+ case OPK_GLOBAL:
+ addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
+ addr.base.global.sym = op.v.global.sym;
+ addr.base.global.addend = op.v.global.addend;
+ addr.base_type = op.type;
+ d->native->load_addr(d->native, dst, addr);
+ return;
+ case OPK_INDIRECT:
+ addr = arm_direct_materialize_addr(d, op);
+ arm_emit_mem(arm_of(d->native), 1, dst, addr,
+ native_mem_for_type(d->native, op.type, 0));
+ return;
+ }
+ arm_asm_panic(d, "unsupported asm input operand");
+}
+static void arm_direct_load_address_to_reg(NativeDirectTarget* d, Operand op,
+ NativeLoc dst) {
+ d->native->load_addr(d->native, dst, arm_direct_addr(d, op));
+}
+static void arm_direct_store_reg_to_operand(NativeDirectTarget* d, Operand op,
+ NativeLoc src) {
+ NativeAddr addr;
+ memset(&addr, 0, sizeof addr);
+ if (op.kind == OPK_LOCAL) {
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ addr.base.frame = d->locals[op.v.local - 1u].home;
+ addr.base_type = op.type;
+ } else {
+ addr = arm_direct_materialize_addr(d, op);
+ }
+ arm_emit_mem(arm_of(d->native), 0, src, addr,
+ native_mem_for_type(d->native, op.type, 0));
+}
+
+/* Callee-saved spill/restore for asm clobbers (r4..r11, excluding r7=fp). */
+typedef struct ArmAsmSavedClobber {
+ NativeFrameSlot slot;
+ Reg reg;
+ KitCgTypeId type;
+} ArmAsmSavedClobber;
+
+static ArmAsmSavedClobber* arm_asm_save_callee_clobbers(Arm32NativeTarget* a,
+ u32 int_mask,
+ u32 fp_mask,
+ u32* nsaved_out) {
+ ArmAsmSavedClobber* saved =
+ arena_zarray(a->base.c->tu, ArmAsmSavedClobber, 12u);
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ u32 n = 0;
+ (void)fp_mask;
+ for (Reg r = 4u; r <= 11u; ++r) {
+ NativeFrameSlotDesc sd;
+ NativeAddr addr;
+ if (r == ARM_FP || (int_mask & (1u << r)) == 0) continue;
+ memset(&sd, 0, sizeof sd);
+ sd.type = i32t;
+ sd.size = 4;
+ sd.align = 4;
+ sd.kind = NATIVE_FRAME_SLOT_SAVE;
+ saved[n].slot = a->base.frame_slot(&a->base, &sd);
+ saved[n].reg = r;
+ saved[n].type = i32t;
+ memset(&addr, 0, sizeof addr);
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ addr.base.frame = saved[n].slot;
+ addr.base_type = i32t;
+ arm_emit_mem(a, 0, native_loc_reg(i32t, NATIVE_REG_INT, r), addr,
+ native_mem_for_type(&a->base, i32t, 4));
+ n++;
+ }
+ *nsaved_out = n;
+ return saved;
+}
+static void arm_asm_restore_one_rec(Arm32NativeTarget* a,
+ const ArmAsmSavedClobber* s) {
+ NativeAddr addr;
+ memset(&addr, 0, sizeof addr);
+ addr.base_kind = NATIVE_ADDR_BASE_FRAME;
+ addr.base.frame = s->slot;
+ addr.base_type = s->type;
+ arm_emit_mem(a, 1, native_loc_reg(s->type, NATIVE_REG_INT, s->reg), addr,
+ native_mem_for_type(&a->base, s->type, 4));
+}
+static void* arm_asm_hook_save_callee_clobbers(NativeDirectTarget* d,
+ u32 int_mask, u32 fp_mask,
+ u32* nsaved_out) {
+ return arm_asm_save_callee_clobbers(arm_of(d->native), int_mask, fp_mask,
+ nsaved_out);
+}
+static void arm_asm_hook_restore_one(NativeDirectTarget* d, void* saved,
+ u32 idx) {
+ arm_asm_restore_one_rec(arm_of(d->native),
+ &((ArmAsmSavedClobber*)saved)[idx]);
+}
+
+/* ---- minimal template runner ----
+ * Render `%N` operand references and emit a small whitelist of mnemonics. The
+ * register operands carry their physical number in bound_*[idx].v.local. */
+static u32 arm_asm_operand_reg(NativeTarget* t, const Operand* bound_outs,
+ u32 nout, const Operand* bound_ins, u32 nin,
+ u32 idx) {
+ const Operand* op = idx < nout ? &bound_outs[idx] : &bound_ins[idx - nout];
+ if (idx >= nout + nin)
+ arm_panic(arm_of(t), "inline asm operand index out of range");
+ if (op->kind != ARM_INLINE_OPK_REG)
+ arm_panic(arm_of(t), "inline asm minimal runner: non-register operand");
+ return (u32)op->v.local & 0xfu;
+}
+
+/* Parse a single rendered token, resolving a `%N` reference to its register. */
+static u32 arm_asm_token_reg(NativeTarget* t, const char* tok, size_t len,
+ const Operand* bo, u32 nout, const Operand* bi,
+ u32 nin) {
+ uint32_t dwarf;
+ char buf[16];
+ if (len >= 2u && tok[0] == '%' && tok[1] >= '0' && tok[1] <= '9') {
+ u32 idx = (u32)(tok[1] - '0');
+ if (len >= 3u && tok[2] >= '0' && tok[2] <= '9')
+ idx = idx * 10u + (u32)(tok[2] - '0');
+ return arm_asm_operand_reg(t, bo, nout, bi, nin, idx);
+ }
+ if (len && len < sizeof buf) {
+ memcpy(buf, tok, len);
+ buf[len] = '\0';
+ if (arm32_register_index(buf, &dwarf) == 0 && dwarf <= 15u) return dwarf;
+ }
+ arm_panic(arm_of(t), "inline asm minimal runner: unrecognized operand");
+}
+
+static void arm_asm_run_one_line(NativeTarget* t, const char* s, size_t n,
+ const Operand* bo, u32 nout, const Operand* bi,
+ u32 nin) {
+ size_t i = 0, j;
+ char mnem[16];
+ size_t ml = 0;
+ /* skip leading space; an empty / whitespace-only line is a no-op. */
+ while (i < n && (s[i] == ' ' || s[i] == '\t')) i++;
+ if (i >= n) return;
+ /* read the mnemonic. */
+ while (i < n && s[i] != ' ' && s[i] != '\t' && ml + 1 < sizeof mnem)
+ mnem[ml++] = s[i++];
+ mnem[ml] = '\0';
+ while (i < n && (s[i] == ' ' || s[i] == '\t')) i++;
+ if (!strcmp(mnem, "nop")) {
+ arm_emit_t16(t->mc, arm_nop16());
+ return;
+ }
+ if (!strcmp(mnem, "dmb")) { arm_emit_t32(t->mc, arm_dmb(0xfu)); return; }
+ if (!strcmp(mnem, "dsb")) { arm_emit_t32(t->mc, arm_t32(0xf3bfu, 0x8f4fu)); return; }
+ if (!strcmp(mnem, "isb")) { arm_emit_t32(t->mc, arm_t32(0xf3bfu, 0x8f6fu)); return; }
+ if (!strcmp(mnem, "wfi")) { arm_emit_t16(t->mc, (u16)0xbf30u); return; }
+ if (!strcmp(mnem, "wfe")) { arm_emit_t16(t->mc, (u16)0xbf20u); return; }
+ if (!strcmp(mnem, "bkpt")) {
+ /* bkpt #imm8 — semihosting (SYS_*) trap. Parse the (decimal/hex) imm. */
+ u32 imm = 0;
+ if (i < n && s[i] == '#') i++;
+ if (i + 1 < n && s[i] == '0' && (s[i + 1] == 'x' || s[i + 1] == 'X')) {
+ i += 2;
+ while (i < n) {
+ char ch = s[i];
+ u32 d;
+ if (ch >= '0' && ch <= '9') d = (u32)(ch - '0');
+ else if (ch >= 'a' && ch <= 'f') d = (u32)(ch - 'a') + 10u;
+ else if (ch >= 'A' && ch <= 'F') d = (u32)(ch - 'A') + 10u;
+ else break;
+ imm = imm * 16u + d;
+ i++;
+ }
+ } else {
+ while (i < n && s[i] >= '0' && s[i] <= '9') imm = imm * 10u + (u32)(s[i++] - '0');
+ }
+ arm_emit_t16(t->mc, arm_bkpt(imm & 0xffu));
+ return;
+ }
+ if (!strcmp(mnem, "mov") || !strcmp(mnem, "movs")) {
+ /* mov <rd>, <rs> — two comma-separated register operands. */
+ const char* a0;
+ size_t a0l;
+ const char* a1;
+ size_t a1l;
+ u32 rd, rs;
+ a0 = &s[i];
+ j = i;
+ while (j < n && s[j] != ',') j++;
+ a0l = j - i;
+ while (a0l && (a0[a0l - 1] == ' ' || a0[a0l - 1] == '\t')) a0l--;
+ if (j >= n) arm_panic(arm_of(t), "inline asm mov: expected two operands");
+ j++; /* skip comma */
+ while (j < n && (s[j] == ' ' || s[j] == '\t')) j++;
+ a1 = &s[j];
+ a1l = n - j;
+ while (a1l && (a1[a1l - 1] == ' ' || a1[a1l - 1] == '\t')) a1l--;
+ rd = arm_asm_token_reg(t, a0, a0l, bo, nout, bi, nin);
+ rs = arm_asm_token_reg(t, a1, a1l, bo, nout, bi, nin);
+ arm_emit_t16(t->mc, arm_mov_hi(rd, rs));
+ return;
+ }
+ arm_panic(arm_of(t),
+ "inline asm mnemonic not supported by the minimal arm32 runner "
+ "(full Thumb-2 assembler is a follow-on)");
+}
+
+/* Walk the template, splitting on '\n' and ';', and run each line. */
+static void arm_asm_run_template(NativeTarget* t, const char* tmpl,
+ const Operand* bound_outs, u32 nout,
+ const Operand* bound_ins, u32 nin) {
+ const char* start;
+ const char* p;
+ if (!tmpl || !*tmpl) return;
+ start = tmpl;
+ for (p = tmpl;; ++p) {
+ if (*p == '\0' || *p == '\n' || *p == ';') {
+ arm_asm_run_one_line(t, start, (size_t)(p - start), bound_outs, nout,
+ bound_ins, nin);
+ if (*p == '\0') break;
+ start = p + 1;
+ }
+ }
+}
+
+static void arm_asm_hook_run_template(NativeDirectTarget* d, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ Operand* bound_outs,
+ const AsmConstraint* ins, u32 nin,
+ Operand* bound_ins, const Sym* clobbers,
+ u32 nclob) {
+ (void)outs;
+ (void)ins;
+ (void)clobbers;
+ (void)nclob;
+ arm_asm_run_template(d->native, tmpl, bound_outs, nout, bound_ins, nin);
+}
+
+static void arm_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ Operand* out_ops, const AsmConstraint* ins,
+ u32 nin, const Operand* in_ops,
+ const Sym* clobbers, u32 nclob,
+ u32 clobber_abi_sets) {
+ static const NativeAsmDirectHooks hooks = {
+ /* Reserve ip (scratch), fp, sp, lr, pc from the operand allocator. */
+ .scratch_int = (1u << ARM_SCRATCH) | (1u << ARM_FP) | (1u << 13u) |
+ (1u << 14u) | (1u << 15u),
+ .scratch_fp = 0u,
+ .opk_reg = ARM_INLINE_OPK_REG,
+ .opcls_fp = ARM_INLINE_OPCLS_FP,
+ .panic = arm_asm_panic,
+ .bound_reg = arm_asm_bound_reg,
+ .bound_mem = arm_asm_bound_mem,
+ .alloc_reg = arm_asm_alloc_reg,
+ .clobber_masks = arm_asm_clobber_masks,
+ .save_callee_clobbers = arm_asm_hook_save_callee_clobbers,
+ .restore_one = arm_asm_hook_restore_one,
+ .load_operand_to_reg = arm_direct_load_operand_to_reg,
+ .load_address_to_reg = arm_direct_load_address_to_reg,
+ .store_reg_to_operand = arm_direct_store_reg_to_operand,
+ .run_template = arm_asm_hook_run_template,
+ };
+ native_asm_bind_direct_operands(d, tmpl, outs, nout, out_ops, ins, nin,
+ in_ops, clobbers, nclob, clobber_abi_sets,
+ &hooks);
+}
+
+/* NativeTarget (optimizer / -O1) inline-asm path: bind operands to registers
+ * (no self-allocation — the optimizer pre-allocated and forwarded clobbers via
+ * plan_frame), then run the template. The minimal runner needs only the bound
+ * register numbers, which arrive in out_locs/in_locs. */
+static void arm_asm_block_native(NativeTarget* t, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ NativeLoc* out_locs, const AsmConstraint* ins,
+ u32 nin, const NativeLoc* in_locs,
+ const Sym* clobbers, u32 nclob) {
+ Compiler* c = t->c;
+ Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
+ Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
+ u32 i;
+ (void)clobbers;
+ (void)nclob;
+ for (i = 0; i < nout; ++i) {
+ KitCgTypeId type = outs[i].type ? outs[i].type : out_locs[i].type;
+ if (out_locs[i].kind != NATIVE_LOC_REG)
+ arm_asm_panic_at(c, arm_of(t)->loc,
+ "optimizer asm output not in a register");
+ arm_asm_bound_reg(&bound_outs[i], type, (NativeAllocClass)out_locs[i].cls,
+ (Reg)out_locs[i].v.reg);
+ }
+ for (i = 0; i < nin; ++i) {
+ const char* body = native_asm_constraint_body(ins[i].str);
+ int matched = native_asm_match_index(body);
+ KitCgTypeId type;
+ if (matched >= 0) {
+ if ((u32)matched >= nout)
+ arm_asm_panic_at(c, arm_of(t)->loc, "matching constraint out of range");
+ bound_ins[i] = bound_outs[matched];
+ continue;
+ }
+ type = ins[i].type ? ins[i].type : in_locs[i].type;
+ if (in_locs[i].kind != NATIVE_LOC_REG)
+ arm_asm_panic_at(c, arm_of(t)->loc,
+ "optimizer asm input not in a register");
+ arm_asm_bound_reg(&bound_ins[i], type, (NativeAllocClass)in_locs[i].cls,
+ (Reg)in_locs[i].v.reg);
+ }
+ arm_asm_run_template(t, tmpl, bound_outs, nout, bound_ins, nin);
+}
+
/* ============================ construction ============================ */
NativeTarget* arm32_native_target_new(Compiler* c, ObjBuilder* obj,
@@ -1771,7 +2730,15 @@ NativeTarget* arm32_native_target_new(Compiler* c, ObjBuilder* obj,
t->ret = arm_ret;
t->atomic_load = arm_atomic_load;
t->atomic_store = arm_atomic_store;
+ t->atomic_rmw = arm_atomic_rmw;
+ t->atomic_cas = arm_atomic_cas;
+ t->fence = arm_fence;
+ t->va_start_ = arm_va_start_native;
+ t->va_arg_ = arm_va_arg_native;
+ t->va_end_ = arm_va_end_native;
+ t->va_copy_ = arm_va_copy_native;
t->intrinsic = arm_intrinsic;
+ t->asm_block = arm_asm_block_native;
t->file_scope_asm = native_file_scope_asm;
t->trap = arm_trap;
t->set_loc = arm_set_loc;
@@ -1798,9 +2765,100 @@ static const char* arm_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
return "arm32 tail calls not implemented in Phase 1";
}
+/* ---- Direct (-O0) varargs wrappers: resolve a va_list operand's address into
+ * a register, then call the shared cores. The va cores use ip (cursor/temp) and
+ * lr (copy scratch); the va_list base register must be distinct, so the direct
+ * wrappers stage it into a callee-saved scratch (r4) before calling. ---- */
+static NativeAddr arm_direct_pointer_addr(NativeDirectTarget* d, Operand op,
+ Reg reg) {
+ NativeAddr addr;
+ KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
+ memset(&addr, 0, sizeof addr);
+ if (op.kind == OPK_LOCAL) {
+ /* The local HOLDS the pointer (&ap): load its home value. */
+ NativeLoc base = native_loc_reg(i32t, NATIVE_REG_INT, reg);
+ NativeAddr load;
+ memset(&load, 0, sizeof load);
+ load.base_kind = NATIVE_ADDR_BASE_FRAME;
+ load.base.frame = d->locals[op.v.local - 1u].home;
+ load.base_type = op.type;
+ arm_emit_mem(arm_of(d->native), 1, base, load,
+ native_mem_for_type(d->native, op.type, 4));
+ addr.base_kind = NATIVE_ADDR_BASE_REG;
+ addr.base.reg = reg;
+ addr.base_type = i32t;
+ return addr;
+ }
+ /* OPK_INDIRECT names *(base+ofs): its address base+ofs IS the pointer. */
+ {
+ NativeAddr m = arm_direct_materialize_addr(d, op);
+ NativeLoc dst = native_loc_reg(i32t, NATIVE_REG_INT, reg);
+ d->native->load_addr(d->native, dst, m);
+ addr.base_kind = NATIVE_ADDR_BASE_REG;
+ addr.base.reg = reg;
+ addr.base_type = i32t;
+ return addr;
+ }
+}
+
+static void arm_va_start_(NativeDirectTarget* d, Operand ap_addr) {
+ arm_va_start_core(arm_of(d->native), arm_direct_pointer_addr(d, ap_addr, 4u));
+}
+static void arm_va_arg_(NativeDirectTarget* d, Operand dst_op, Operand ap_addr,
+ KitCgTypeId type) {
+ Arm32NativeTarget* a = arm_of(d->native);
+ NativeAddr ap = arm_direct_pointer_addr(d, ap_addr, 4u);
+ if (arm_va_arg_is_wide(d->native, type)) {
+ /* Wide (i64/double) va_arg copies straight from the save area into the
+ * destination memory. arm_va_arg_wide uses ip (cursor) + lr (copy scratch),
+ * so an OPK_INDIRECT destination pointer must live in a distinct reg (r5),
+ * not ip — materialize it there. A FRAME (OPK_LOCAL) destination needs no
+ * register. */
+ NativeAddr dst = arm_direct_addr(d, dst_op);
+ if (dst.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
+ NativeLoc base = native_loc_reg(dst.base_type, NATIVE_REG_INT, 5u);
+ NativeAddr load;
+ memset(&load, 0, sizeof load);
+ load.base_kind = NATIVE_ADDR_BASE_FRAME;
+ load.base.frame = dst.base.frame;
+ load.base_type = dst.base_type;
+ arm_emit_mem(a, 1, base, load,
+ native_mem_for_type(d->native, dst.base_type, 4));
+ dst.base_kind = NATIVE_ADDR_BASE_REG;
+ dst.base.reg = 5u;
+ }
+ arm_va_arg_wide(a, dst, ap, native_type_size(d->native, type));
+ return;
+ }
+ {
+ NativeLoc res = native_loc_reg(type, NATIVE_REG_INT, 5u);
+ NativeAddr dst;
+ arm_va_arg_core(a, res, ap, type);
+ dst = arm_direct_materialize_addr(d, dst_op);
+ arm_emit_mem(a, 0, res, dst,
+ native_mem_for_type(d->native, type,
+ native_type_size(d->native, type)));
+ }
+}
+static void arm_va_end_(NativeDirectTarget* d, Operand ap_addr) {
+ (void)d;
+ (void)ap_addr;
+}
+static void arm_va_copy_(NativeDirectTarget* d, Operand dst_op, Operand src_op) {
+ Arm32NativeTarget* a = arm_of(d->native);
+ NativeAddr src = arm_direct_pointer_addr(d, src_op, 4u);
+ NativeAddr dst = arm_direct_pointer_addr(d, dst_op, 5u);
+ arm_va_copy_core(a, dst, src);
+}
+
static const NativeOps arm_direct_ops = {
.bind_param = arm_bind_param,
.tail_call_unrealizable_reason = arm_no_tail,
+ .va_start_ = arm_va_start_,
+ .va_arg_ = arm_va_arg_,
+ .va_end_ = arm_va_end_,
+ .va_copy_ = arm_va_copy_,
+ .asm_block = arm_direct_asm_block,
};
const NativeOps* arm32_native_direct_ops(void) { return &arm_direct_ops; }
