commit 09c0cfd167b6640c0da40184f4a2576da51b6ee5
parent 6a21d4a06c4b95ff4f586305994be0628b49d280
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 21:36:58 -0700
arm32 §1: unified memory-access address layer (correct by construction)
Replace the ad-hoc lr/ip rescue paths in the load/store/copy layer with a
single address resolver (arm_resolve_mem) carrying ONE scratch-reservation
invariant, asserted in arm_pick_scratch:
- Every access maps a NativeAddr × offset × width to a legal (base, in-range
displacement), or materializes the full effective address once into a reserved
scratch (LR preferred; IP is the NDT operand scratch) chosen to alias neither
the transfer reg rt nor a live base. Fixes the i64/wide store-to-global
miscompile (str lr,[lr]) — the resolver never lands a global address in the
value register.
- Wide (i64/double) accesses resolve ONE shared base pinning both lane regs, so
the high lane's address is correct for frame/reg/global uniformly.
- copy_bytes resolves each side once (direct in-range, else materialized) with an
explicit register budget (IP/LR + one borrowed callee-save for the rare
both-materialized case) instead of the old fast/slow fork; arm_emit_base_off
stages into rd (no ARM_TMP dependency).
- arm_emit_global_addr applies a non-zero byte offset as a SEPARATE add (reloc
addend stays 0): ARM REL MOVW/MOVT addends are carry-lossy across the 16-bit
split, so the separate-add form is always-correct and matches the contract
elf_arm_reloc_field_addend documents. Fixes initialized global array/record/
tuple data reads at a non-zero element offset (table[1] read table[0]).
Also fixes the switch jump-table crashes (the .Lkit_jt table read went through
the global path) and static-local data. Validates clean: full toy corpus
281 pass (was 270), no regressions; arm32_decode_test 314/0.
scripts/arm32probe.sh + arm32batch.sh: fast single-case / full-corpus arm32
bring-up harnesses over the qemu bare lane.
Diffstat:
3 files changed, 328 insertions(+), 234 deletions(-)
diff --git a/scripts/arm32batch.sh b/scripts/arm32batch.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# arm32batch.sh — run the whole toy corpus through the arm32 bare lane at O0+O1,
+# print a compact PASS/FAIL summary + the failing list. Reuses one stub setup.
+set -u
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+export KIT="${KIT:-$ROOT/build/release/kit}"
+source "$ROOT/test/lib/exec_bare.sh"
+WORK="${ARM32PROBE_WORK:-/tmp/arm32batch}"; mkdir -p "$WORK"
+exec_bare_setup arm32 "$WORK" main >"$WORK/setup.log" 2>&1 || { echo "setup failed"; cat "$WORK/setup.log"; exit 3; }
+pass=0; fail=0; fails=""
+for f in "$ROOT"/test/toy/cases/*.toy; do
+ name=$(basename "${f%.toy}")
+ [ -e "${f%.toy}.arm32.skip" ] && continue
+ grep -q 'asmnop' "$f" 2>/dev/null && continue
+ exp=$(( $(cat "${f%.toy}.expected" 2>/dev/null || echo 0) & 255 ))
+ for opt in 0 1; do
+ obj="$WORK/$name.O$opt.o"
+ if ! "$KIT" cc -O"$opt" -target arm-none-eabi -mcpu=cortex-m3 -mfloat-abi=soft \
+ -ffreestanding -c "$f" -o "$obj" >"$WORK/cc.out" 2>"$WORK/cc.err"; then
+ fail=$((fail+1)); fails="$fails $name/O$opt(cc)"; continue
+ fi
+ [ -s "$WORK/cc.err" ] && { fail=$((fail+1)); fails="$fails $name/O$opt(stderr)"; continue; }
+ if ! exec_bare_run arm32 "$obj" "$WORK" "$WORK/rc" >/dev/null 2>&1; then
+ fail=$((fail+1)); fails="$fails $name/O$opt(link)"; continue
+ fi
+ rc=$(cat "$WORK/rc" 2>/dev/null || echo 99)
+ if [ "$rc" -eq "$exp" ]; then pass=$((pass+1)); else fail=$((fail+1)); fails="$fails $name/O$opt(exp=$exp,got=$rc)"; fi
+ done
+done
+echo "PASS=$pass FAIL=$fail"
+echo "FAILS:"; for x in $fails; do echo " $x"; done
diff --git a/scripts/arm32probe.sh b/scripts/arm32probe.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# arm32probe.sh — fast single-case arm32 bring-up probe.
+# arm32probe.sh <case.toy|case.c> [opt] compile+link+run under qemu, vs .expected
+# arm32probe.sh --dis <case.toy> [opt] compile + objdump the .text
+# KIT defaults to build/release/kit. Sets up the bare stub once under /tmp/arm32probe.
+set -u
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+export KIT="${KIT:-$ROOT/build/release/kit}"
+source "$ROOT/test/lib/exec_bare.sh"
+WORK="${ARM32PROBE_WORK:-/tmp/arm32probe}"; mkdir -p "$WORK"
+exec_bare_setup arm32 "$WORK" main >/"$WORK/setup.log" 2>&1 || { echo "setup failed"; cat "$WORK/setup.log"; exit 3; }
+
+dis=0
+if [ "${1:-}" = "--dis" ]; then dis=1; shift; fi
+f="$1"; opt="${2:-0}"
+name=$(basename "${f%.*}")
+obj="$WORK/$name.O$opt.o"
+if ! "$KIT" cc -O"$opt" -target arm-none-eabi -mcpu=cortex-m3 -mfloat-abi=soft \
+ -ffreestanding -c "$f" -o "$obj" >"$WORK/cc.out" 2>"$WORK/cc.err"; then
+ echo "$name/O$opt: CC-FAIL"; sed 's/^/ | /' "$WORK/cc.err"; exit 1
+fi
+if [ -s "$WORK/cc.err" ]; then echo "$name/O$opt: CC-STDERR"; sed 's/^/ | /' "$WORK/cc.err"; fi
+if [ "$dis" -eq 1 ]; then "$KIT" objdump -d "$obj"; exit 0; fi
+reason="$(exec_bare_run arm32 "$obj" "$WORK" "$WORK/rc" 2>&1)"
+if [ $? -eq 2 ]; then echo "$name/O$opt: LINK/RUN-FAIL: $reason"; exit 2; fi
+rc=$(cat "$WORK/rc" 2>/dev/null || echo 99)
+exp=$(( $(cat "${f%.*}.expected" 2>/dev/null || echo 0) & 255 ))
+if [ "$rc" -eq "$exp" ]; then echo "$name/O$opt: PASS (rc=$rc)"; else echo "$name/O$opt: FAIL exp=$exp got=$rc"; fi
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -312,58 +312,93 @@ static int arm_addr_legal(NativeTarget* t, const NativeAddr* addr,
/* ============================ memory ============================ */
+static void arm_emit_load_u32(Arm32NativeTarget* a, u32 rd, u32 v);
+static u32 arm_pick_scratch(Arm32NativeTarget* a, u32 rt, u32 pinned);
+
+/* Add a signed compile-time constant to rd in place (rd += k), using a reserved
+ * scratch only for a magnitude exceeding the inline immediate forms. */
+static void arm_emit_add_const(Arm32NativeTarget* a, u32 rd, i64 k) {
+ MCEmitter* mc = a->base.mc;
+ u32 enc, mag;
+ if (k == 0) return;
+ mag = (u32)(k < 0 ? -k : k);
+ if (mag <= 0xfffu) {
+ arm_emit_t32(mc, k < 0 ? arm_sub_imm12(rd, rd, mag)
+ : arm_add_imm12(rd, rd, mag));
+ } else if (thumb_expand_imm_encode(mag, &enc)) {
+ arm_emit_t32(mc, arm_dp_imm(k < 0 ? 13u : 8u, 0u, rd, rd, enc));
+ } else {
+ u32 s = arm_pick_scratch(a, rd, 0u);
+ arm_emit_load_u32(a, s, mag);
+ arm_emit_t32(mc, k < 0 ? arm_sub_reg(rd, rd, s) : arm_add_reg(rd, rd, s));
+ }
+}
+
/* Materialize the runtime address of `sym` (+ addend) into register `rd`.
*
* arm32's house strategy is MOVW/MOVT absolute (Cortex-M has no literal pools
* and the bare-metal lane is non-PIC):
- * MOVW rd, #:lower16:sym ; R_ARM_THM_MOVW_ABS_NC -> (S+addend)[15:0]
- * MOVT rd, #:upper16:sym ; R_ARM_THM_MOVT_ABS -> (S+addend)[31:16]
- * The 16-bit imm placeholders the encoders write are overwritten by the
- * patcher (src/arch/arm32/reloc.c); the addend rides in the reloc record (an
- * explicit addend), so the symbol's full address+addend lands in `rd` and no
- * post-add is emitted. (A GOT-indirect variant would belong here if a PIC
- * arm32 lane were ever added; bare-metal ELF resolves these absolutely.) */
+ * MOVW rd, #:lower16:sym ; R_ARM_THM_MOVW_ABS_NC -> S[15:0]
+ * MOVT rd, #:upper16:sym ; R_ARM_THM_MOVT_ABS -> S[31:16]
+ * A constant byte `addend` is applied as a SEPARATE add, NOT folded into the
+ * reloc: ARM uses REL relocations (the addend rides in the relocated field), and
+ * the MOVW/MOVT pair splits a 32-bit value across two 16-bit fields whose
+ * per-half (S+A) computation is carry-lossy across the 16-bit boundary. Keeping
+ * the reloc addend 0 and adding the offset here is both always-correct and the
+ * contract elf_arm_reloc_field_addend documents (and ld.lld-compatible). */
static void arm_emit_global_addr(Arm32NativeTarget* a, u32 rd, ObjSymId sym,
i64 addend) {
MCEmitter* mc = a->base.mc;
u32 pos = mc_pos(mc);
arm_emit_t32(mc, arm_movw(rd, 0u));
- mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_MOVW_ABS_NC, sym, addend,
- 1, 0);
+ mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_MOVW_ABS_NC, sym, 0, 1, 0);
pos = mc_pos(mc);
arm_emit_t32(mc, arm_movt(rd, 0u));
- mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_MOVT_ABS, sym, addend, 1,
- 0);
-}
-
-/* Resolve a NativeAddr to a (base reg, signed byte offset) pair. A GLOBAL base
- * is materialized via MOVW/MOVT into ARM_TMP (lr) — NOT the IP scratch (r12),
- * which the NDT uses to materialize operands: in a store `str rt, [base]` the
- * transfer reg rt may itself be r12 (an NDT-materialized value or scratch),
- * which the IP global-address would clobber. lr is reserved (never an NDT
- * operand) and dead in the body, so it can never alias rt. */
-static u32 arm_addr_base(Arm32NativeTarget* a, const NativeAddr* addr,
- i32* off_out) {
- switch (addr->base_kind) {
- case NATIVE_ADDR_BASE_FRAME: {
- NativeFrameSlotEntry* s =
- native_frame_slot_at(&a->frame, addr->base.frame);
- *off_out = -(i32)s->off + addr->offset; /* [r7, #-(off) + extra] */
- return ARM_FP;
- }
- case NATIVE_ADDR_BASE_REG:
- *off_out = addr->offset;
- return addr->base.reg & 0xfu;
- case NATIVE_ADDR_BASE_GLOBAL:
- arm_emit_global_addr(a, ARM_TMP, addr->base.global.sym,
- addr->base.global.addend);
- *off_out = addr->offset;
- return ARM_TMP;
- default:
- arm_panic(a, "unsupported addressing mode");
- }
+ mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_MOVT_ABS, sym, 0, 1, 0);
+ arm_emit_add_const(a, rd, addend);
}
+/* The unified memory-access address layer (ARM32.md §1) lives here. Every
+ * load/store/copy resolves a NativeAddr to a legal (base register, in-range
+ * displacement) through ONE function, arm_resolve_mem, with ONE scratch-
+ * reservation invariant asserted in one place.
+ *
+ * Scratch budget: two registers are reserved as emit scratch in the function
+ * body — IP (r12, ARM_SCRATCH) and LR (r14, ARM_TMP); r7 (FP) holds the frame
+ * anchor and is never a scratch. A single access needs at most one
+ * address-staging scratch, distinct from the transfer register `rt`. The picker
+ * draws it from {IP,LR} minus {rt} minus a caller `pinned` mask (held bases the
+ * access must preserve) and panics if none is free, so the invariant is explicit
+ * rather than relying on incidental liveness. */
+static void arm_emit_base_off(Arm32NativeTarget* a, u32 rd, u32 base, i32 off);
+
+/* True if every byte of a `span`-byte access at signed displacement `off` fits
+ * the Thumb-2 single-granule immediate forms (T4 negative -255..0, T3 positive
+ * 0..4095): off >= -255 and off+span <= 4096 implies every granule offset in
+ * [off, off+span-1] lands in [-255, 4095]. */
+static int arm_span_inline(i32 off, u32 span) {
+ return off >= -255 && off + (i32)span <= 4096;
+}
+
+/* Pick a reserved staging register from {LR, IP} that is neither the transfer
+ * register `rt` (0xff = none) nor in `pinned` (held base registers the access
+ * must preserve). LR (ARM_TMP) is preferred because IP (ARM_SCRATCH) is the NDT's
+ * operand-materialization scratch and can hold a live value across the access
+ * (e.g. a store whose value the NDT homed in IP); LR is only ever a transient
+ * backend temp within a single emit, so staging an address there is free. The
+ * single point that enforces the scratch invariant. */
+static u32 arm_pick_scratch(Arm32NativeTarget* a, u32 rt, u32 pinned) {
+ u32 avoid = (rt < 16u ? (1u << rt) : 0u) | pinned;
+ if (!(avoid & (1u << ARM_TMP))) return ARM_TMP;
+ if (!(avoid & (1u << ARM_SCRATCH))) return ARM_SCRATCH;
+ arm_panic(a, "memory access: no free address-staging scratch");
+}
+
+typedef struct ArmMemAddr {
+ u32 base; /* base register to address through */
+ i32 off; /* signed displacement; access [base, #off (+lane/granule)] */
+} ArmMemAddr;
+
/* Materialize the unsigned 32-bit constant `v` into `rd` (MOVW + optional MOVT).
* Used to stage large frame/memory offsets and SUB-sp amounts that exceed the
* inline immediate forms. */
@@ -382,39 +417,63 @@ static void arm_emit_load_u32(Arm32NativeTarget* a, u32 rd, u32 v) {
if ((v >> 16) != 0u) arm_emit_t32(mc, arm_movt(rd, (v >> 16) & 0xffffu));
}
-/* Emit a load (is_load=1) or store of `reg` to [base, #off], dispatching the
- * width from mem.size. Loads zero-extend (LDR/LDRB/LDRH); signed narrowing is a
+/* Resolve `addr` for a `span`-byte access whose transfer register is `rt`
+ * (0xff = none; the wide path pins both lane registers via `pinned` instead),
+ * returning a legal (base, off). A FRAME/REG base already in range is used
+ * directly; otherwise the full effective address is materialized once into a
+ * picked scratch (off := 0). A GLOBAL base is always materialized via MOVW/MOVT,
+ * folding addr.offset into the reloc addend so off := 0 — this is the single
+ * point that guarantees a store's global address never lands in the value
+ * register (the i64-store-to-global miscompile). */
+static ArmMemAddr arm_resolve_mem(Arm32NativeTarget* a, const NativeAddr* addr,
+ u32 rt, u32 span, u32 pinned) {
+ ArmMemAddr r;
+ switch (addr->base_kind) {
+ case NATIVE_ADDR_BASE_FRAME: {
+ NativeFrameSlotEntry* s =
+ native_frame_slot_at(&a->frame, addr->base.frame);
+ i32 off = -(i32)s->off + addr->offset;
+ if (arm_span_inline(off, span)) {
+ r.base = ARM_FP;
+ r.off = off;
+ return r;
+ }
+ r.base = arm_pick_scratch(a, rt, pinned | (1u << ARM_FP));
+ arm_emit_base_off(a, r.base, ARM_FP, off);
+ r.off = 0;
+ return r;
+ }
+ case NATIVE_ADDR_BASE_REG: {
+ u32 base = addr->base.reg & 0xfu;
+ if (arm_span_inline(addr->offset, span)) {
+ r.base = base;
+ r.off = addr->offset;
+ return r;
+ }
+ r.base = arm_pick_scratch(a, rt, pinned | (1u << base));
+ arm_emit_base_off(a, r.base, base, addr->offset);
+ r.off = 0;
+ return r;
+ }
+ case NATIVE_ADDR_BASE_GLOBAL:
+ r.base = arm_pick_scratch(a, rt, pinned);
+ arm_emit_global_addr(a, r.base, addr->base.global.sym,
+ addr->base.global.addend + addr->offset);
+ r.off = 0;
+ return r;
+ default:
+ arm_panic(a, "unsupported addressing mode");
+ }
+}
+
+/* Emit one in-range granule access: LDR/STR (+B/H) of `rt` at [base, #off],
+ * dispatching width. The caller (via arm_resolve_mem) guarantees `off` is in the
+ * T3/T4 immediate range for this width. Loads zero-extend; signed narrowing is a
* separate convert. */
-static void arm_emit_mem(Arm32NativeTarget* a, int is_load, NativeLoc reg,
- NativeAddr addr, MemAccess mem) {
+static void arm_emit_mem_one(Arm32NativeTarget* a, int is_load, u32 rt, u32 base,
+ i32 off, u32 size) {
MCEmitter* mc = a->base.mc;
- i32 off;
- u32 base = arm_addr_base(a, &addr, &off);
- u32 rt = loc_reg(reg);
- u32 size = mem.size ? mem.size : native_type_size(&a->base, reg.type);
u32 t3hw1, t4hw1;
- if (size >= 8u) {
- /* Split a wide (8-byte) access into two 4-byte halves over the consecutive
- * register pair: low lane = rt at [base, off], high lane = rt+1 at
- * [base, off+4] (little-endian). The cg hands a few paths — notably an i64
- * switch scrutinee — a single 8-byte access into an even/odd pair; rv32
- * splits identically. (Wider-than-8 is not produced by the C frontend.) */
- KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
- NativeLoc lo = reg, hi = reg;
- NativeAddr addr_hi = addr;
- MemAccess m = mem;
- if (size != 8u) arm_panic(a, "memory access wider than 8 bytes");
- m.size = 4u;
- m.type = i32t;
- lo.type = i32t;
- hi.type = i32t;
- hi.v.reg = (rt + 1u) & 0xfu;
- addr_hi.offset += 4;
- arm_emit_mem(a, is_load, lo, addr, m);
- arm_emit_mem(a, is_load, hi, addr_hi, m);
- return;
- }
- /* pick the op (T3 positive, T4 +/-) for this width. */
if (size == 1u) {
t3hw1 = is_load ? 0xf890u : 0xf880u;
t4hw1 = is_load ? 0xf810u : 0xf800u;
@@ -429,27 +488,31 @@ static void arm_emit_mem(Arm32NativeTarget* a, int is_load, NativeLoc reg,
arm_emit_t32(mc, arm_t32(t3hw1 | base, (rt << 12) | (u32)off));
} else if (off < 0 && off >= -255) {
arm_emit_t32(mc, arm_ldst_t4(t4hw1, rt, base, (u32)(-off), 0u));
- } else if (off >= 0 && off <= 255) {
- arm_emit_t32(mc, arm_ldst_t4(t4hw1, rt, base, (u32)off, 1u));
} else {
- /* Large frame/memory offset (-O1 big frames, or a deep -O0 wide8/aggregate
- * copy): stage base+off into a scratch register and access [scratch]. The
- * staging register must alias neither the transfer reg `rt` nor `base`,
- * because it is overwritten before the access. Both reserved scratch regs
- * (lr=ARM_TMP, ip=ARM_SCRATCH) are dead in the body. Normally lr is used,
- * but `rt` can itself be lr (arm_copy_bytes / arm_bitfield_store route their
- * transfer through ARM_TMP) — staging the address into lr would then clobber
- * the value mid-copy (str lr,[lr]); fall back to ip in that case. `base` is
- * never ARM_TMP here (a GLOBAL base — the only ARM_TMP producer — folds its
- * byte offset into the MOVW/MOVT reloc addend, so it reaches arm_emit_mem in
- * range), but it can be ip; the picked register avoids it either way. */
- u32 stg = (rt == ARM_TMP) ? ARM_SCRATCH : ARM_TMP;
- if (base == ARM_TMP) arm_panic(a, "large offset on lr-staged base");
- if (stg == base || stg == rt) arm_panic(a, "no free scratch for large offset");
- arm_emit_load_u32(a, stg, (u32)(off < 0 ? -off : off));
- arm_emit_t32(mc, off < 0 ? arm_sub_reg(stg, base, stg)
- : arm_add_reg(stg, base, stg));
- arm_emit_t32(mc, arm_t32(t3hw1 | stg, (rt << 12) | 0u));
+ arm_panic(a, "arm_emit_mem_one: offset out of range (resolver bug)");
+ }
+}
+
+/* Emit a load (is_load=1) or store of `reg` at `addr`, dispatching width from
+ * mem.size. A wide (8-byte i64/double) access splits into two 4-byte lanes over
+ * the register pair rt:rt+1 sharing ONE resolved base — the high lane's address
+ * is thus computed correctly for frame/reg/global uniformly. */
+static void arm_emit_mem(Arm32NativeTarget* a, int is_load, NativeLoc reg,
+ NativeAddr addr, MemAccess mem) {
+ u32 rt = loc_reg(reg);
+ u32 size = mem.size ? mem.size : native_type_size(&a->base, reg.type);
+ if (size > 8u) arm_panic(a, "memory access wider than 8 bytes");
+ if (size == 8u) {
+ u32 hi = (rt + 1u) & 0xfu;
+ ArmMemAddr r =
+ arm_resolve_mem(a, &addr, 0xffu, 8u, (1u << rt) | (1u << hi));
+ arm_emit_mem_one(a, is_load, rt, r.base, r.off, 4u);
+ arm_emit_mem_one(a, is_load, hi, r.base, r.off + 4, 4u);
+ return;
+ }
+ {
+ ArmMemAddr r = arm_resolve_mem(a, &addr, rt, size, 0u);
+ arm_emit_mem_one(a, is_load, rt, r.base, r.off, size);
}
}
@@ -516,12 +579,15 @@ static void arm_emit_base_off(Arm32NativeTarget* a, u32 rd, u32 base, i32 off) {
} else if (off < 0 && thumb_expand_imm_encode((u32)(-off), &enc)) {
arm_emit_t32(mc, arm_dp_imm(13u, 0u, rd, base, enc)); /* SUB rd, base, #-off */
} else {
- /* Large frame/array offset (-O1 big frames): stage |off| in ARM_TMP (lr) so
- * the base register survives even when rd aliases it, then add/sub into rd.
- * lr is reserved and dead in the body. */
- arm_emit_load_u32(a, ARM_TMP, (u32)(off < 0 ? -off : off));
- arm_emit_t32(mc, off < 0 ? arm_sub_reg(rd, base, ARM_TMP)
- : arm_add_reg(rd, base, ARM_TMP));
+ /* Large frame/array offset: stage |off| in a temp then add/sub into rd. The
+ * temp must not alias `base` (read after write). Use rd itself when rd !=
+ * base (the resolver guarantees a staged base reg differs from its source);
+ * only when rd aliases base (load_addr's address-of-an-indexed-REG case)
+ * fall back to the reserved ARM_TMP, which is dead in that context. */
+ u32 tmp = (rd != base) ? rd : ARM_TMP;
+ arm_emit_load_u32(a, tmp, (u32)(off < 0 ? -off : off));
+ arm_emit_t32(mc, off < 0 ? arm_sub_reg(rd, base, tmp)
+ : arm_add_reg(rd, base, tmp));
}
}
@@ -536,14 +602,17 @@ static void arm_emit_base_off(Arm32NativeTarget* a, u32 rd, u32 base, i32 off) {
static void arm_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr) {
Arm32NativeTarget* a = arm_of(t);
MCEmitter* mc = t->mc;
- i32 off;
- u32 base, rd = loc_reg(dst);
+ u32 rd = loc_reg(dst);
if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL) {
arm_emit_global_addr(a, rd, addr.base.global.sym,
addr.base.global.addend + addr.offset);
+ } else if (addr.base_kind == NATIVE_ADDR_BASE_FRAME) {
+ NativeFrameSlotEntry* s = native_frame_slot_at(&a->frame, addr.base.frame);
+ arm_emit_base_off(a, rd, ARM_FP, -(i32)s->off + addr.offset);
+ } else if (addr.base_kind == NATIVE_ADDR_BASE_REG) {
+ arm_emit_base_off(a, rd, addr.base.reg & 0xfu, addr.offset);
} else {
- base = arm_addr_base(a, &addr, &off);
- arm_emit_base_off(a, rd, base, off);
+ arm_panic(a, "unsupported addressing mode in load_addr");
}
if (addr.index_kind == NATIVE_ADDR_INDEX_REG) {
u32 idx = addr.index.reg & 0xfu;
@@ -1789,151 +1858,117 @@ static void arm_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
* with a software thread pointer is a follow-on for a multi-threaded lane.) */
arm_emit_global_addr(arm_of(t), loc_reg(dst), sym, addend);
}
-/* copy_bytes / set_bytes: aggregate (struct/array, and the wide8 i64/double)
- * memory ops, transferring a word/halfword/byte ladder through ARM_TMP (lr),
- * mirroring the aa64 sibling backend.
+/* Aggregate / wide byte copy (struct/array, INDIRECT param/return copies, the
+ * wide va_arg copy, and memmove's loops). Each side is resolved ONCE to a stable
+ * (base register, displacement): a FRAME/REG side in range for the whole span is
+ * addressed directly; a GLOBAL or out-of-range side has its effective address
+ * materialized into a held scratch. A 4|2|1 granule ladder then transfers each
+ * chunk through a register that aliases neither base.
*
- * copy_bytes has two regimes. The common FAST path keeps the src/dst NativeAddr
- * bases as-is and bakes the running byte offset into each load/store immediate
- * (arm_emit_mem dispatches the width + T3/T4 offset encoding); lr is the only
- * scratch and it is RESERVED, so a single copy needs no extra register. When a
- * side's effective offset would leave the inline immediate range (a deep -O0
- * frame slot / wide8 store through a pointer the NDT homed in ip), arm_emit_mem
- * would have to stage the address through a *second* scratch (ip) — which can
- * be the sibling base register, silently corrupting it. The SLOW path below
- * therefore materializes each side's effective address into a stable register
- * once and copies at small offsets, so no per-granule staging happens. */
-/* True if a load/store at signed byte offset `off` from a base fits
- * arm_emit_mem's inline T3/T4 immediate forms (no large-offset staging). */
-static int arm_mem_off_inline(i32 off) {
- return (off >= 0 && off <= 4095) || (off >= -255 && off <= 255);
-}
-
-/* Side-effect-free preview of the per-base byte offset an access would carry,
- * for the copy_bytes range check below. Mirrors arm_addr_base's FRAME/REG
- * offset math WITHOUT emitting the GLOBAL MOVW/MOVT (which arm_addr_base does as
- * a side effect). A GLOBAL base re-materializes its address into lr per access
- * and folds the addend into the reloc, so it never reaches the large-offset
- * lane — report it as in-range. */
-static i32 arm_addr_peek_off(Arm32NativeTarget* a, const NativeAddr* addr) {
- switch (addr->base_kind) {
- case NATIVE_ADDR_BASE_FRAME: {
- NativeFrameSlotEntry* s =
- native_frame_slot_at(&a->frame, addr->base.frame);
- return -(i32)s->off + addr->offset;
- }
- case NATIVE_ADDR_BASE_REG:
- return addr->offset;
- default:
- return 0; /* GLOBAL (in-range by construction) or unsupported */
+ * Register budget, allocated explicitly up front: two reserved scratch (IP, LR);
+ * one callee-save borrowed (pushed/popped) only for the rare case where BOTH
+ * sides must be materialized AND the two bases plus the transfer exceed {IP,LR}.
+ * The borrow is taken before any frame-relative materialization, so r7-relative
+ * offsets (and already-held REG bases) are unaffected by the SP change. */
+
+/* The natural (base register, displacement) of a FRAME/REG addr, no range check. */
+static void arm_addr_natural(Arm32NativeTarget* a, const NativeAddr* addr,
+ u32* base, i32* off) {
+ if (addr->base_kind == NATIVE_ADDR_BASE_FRAME) {
+ NativeFrameSlotEntry* s = native_frame_slot_at(&a->frame, addr->base.frame);
+ *base = ARM_FP;
+ *off = -(i32)s->off + addr->offset;
+ } else {
+ *base = addr->base.reg & 0xfu;
+ *off = addr->offset;
+ }
+}
+
+/* Whether `addr` can be addressed directly for a `span`-byte access (a real base
+ * register + in-range displacement for the whole span). GLOBAL is never direct.
+ * On success sets base+off to the natural (base register, displacement). */
+static int arm_addr_direct(Arm32NativeTarget* a, const NativeAddr* addr,
+ u32 span, u32* base, i32* off) {
+ if (addr->base_kind != NATIVE_ADDR_BASE_FRAME &&
+ addr->base_kind != NATIVE_ADDR_BASE_REG)
+ return 0;
+ arm_addr_natural(a, addr, base, off);
+ return arm_span_inline(*off, span);
+}
+
+/* Take a transfer/held-base register for copy_bytes: a free reserved scratch
+ * (IP then LR) not already in `*used`, else a borrowed callee-save (r4..r6 not in
+ * `*used`, pushed; recorded in `*borrowed`, which starts 0xff and is set once). */
+static u32 arm_copy_take_reg(Arm32NativeTarget* a, u32* used, u32* borrowed) {
+ u32 r;
+ if (!(*used & (1u << ARM_SCRATCH))) r = ARM_SCRATCH;
+ else if (!(*used & (1u << ARM_TMP))) r = ARM_TMP;
+ else {
+ r = (*used & (1u << 4)) ? ((*used & (1u << 5)) ? 6u : 5u) : 4u;
+ *borrowed = r;
+ arm_emit_t32(a->base.mc, arm_push_w(1u << r));
}
+ *used |= 1u << r;
+ return r;
}
static void arm_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
AggregateAccess acc) {
Arm32NativeTarget* a = arm_of(t);
- KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
- NativeLoc tmp = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
- i32 soff = arm_addr_peek_off(a, &src);
- i32 doff = arm_addr_peek_off(a, &dst);
- i32 last = acc.size ? (i32)(acc.size - 1u) : 0;
- u32 off = 0;
- /* Fast path: every per-granule access on BOTH sides stays in the inline
- * immediate range, so arm_emit_mem never needs to stage an offset through a
- * scratch register and cannot collide with the sibling base. */
- if (arm_mem_off_inline(soff) && arm_mem_off_inline(soff + last) &&
- arm_mem_off_inline(doff) && arm_mem_off_inline(doff + last)) {
- /* Widest convenient granule is a word (4); the trailing 1/2/3 bytes fall to
- * halfword/byte ops. The width ladder also keeps each chunk naturally
- * aligned for an aligned aggregate (4|4|...|2|1) and never over-runs
- * acc.size. */
- while (off < acc.size) {
- u32 rem = acc.size - off;
- u32 sz = rem >= 4u ? 4u : rem >= 2u ? 2u : 1u;
- MemAccess mem = acc.mem;
- NativeAddr s = src, d = dst;
- mem.size = sz;
- mem.align = sz;
- s.offset += (i32)off;
- d.offset += (i32)off;
- arm_emit_mem(a, 1, tmp, s, mem); /* LDR/LDRH/LDRB tmp, [src+off] */
- arm_emit_mem(a, 0, tmp, d, mem); /* STR/STRH/STRB tmp, [dst+off] */
- off += sz;
+ u32 size = acc.size, off;
+ u32 sbase, dbase, xfer, used = 0u, borrowed = 0xffu;
+ i32 soff = 0, doff = 0;
+ int sdir, ddir;
+ if (!size) return;
+ if (size > 4096u) arm_panic(a, "aggregate copy > 4096 bytes (needs memcpy)");
+
+ sdir = arm_addr_direct(a, &src, size, &sbase, &soff);
+ ddir = arm_addr_direct(a, &dst, size, &dbase, &doff);
+ if (sdir) used |= 1u << sbase;
+ if (ddir) used |= 1u << dbase;
+
+ /* Allocate held scratch for the non-direct sides, then the transfer reg. */
+ if (!sdir) sbase = arm_copy_take_reg(a, &used, &borrowed);
+ if (!ddir) dbase = arm_copy_take_reg(a, &used, &borrowed);
+ xfer = arm_copy_take_reg(a, &used, &borrowed);
+
+ /* Materialize the non-direct sides (after any borrow-push so a frame-relative
+ * address still reads the unchanged r7). */
+ if (!sdir) {
+ if (src.base_kind == NATIVE_ADDR_BASE_GLOBAL)
+ arm_emit_global_addr(a, sbase, src.base.global.sym,
+ src.base.global.addend + src.offset);
+ else {
+ u32 nb;
+ i32 no;
+ arm_addr_natural(a, &src, &nb, &no);
+ arm_emit_base_off(a, sbase, nb, no);
}
- return;
- }
- /* Slow path: at least one side reaches a deep frame offset (large -O0 frame /
- * wide8 i64 copy). arm_emit_mem's large-offset path stages the address into a
- * scratch register (ip if the transfer reg is lr), which would clobber a
- * sibling base that the NDT legitimately materialized into ip — the wide8
- * `*p = i64` miscompile. Instead, compute each side's full effective address
- * into a stable register ONCE, then access it at small (>=0) offsets so
- * arm_emit_mem stays on its inline T3 path and never stages again.
- *
- * Register budget: the per-granule walk needs three distinct regs — src addr,
- * dst addr, transfer. r7 is the frame pointer; lr (ARM_TMP) is consumed by
- * arm_emit_base_off as its own MOVW/MOVT staging scratch, so it can't also
- * hold a live address during setup. We therefore hold the dst address in a
- * borrowed callee-saved reg (pushed/popped here) and the src address in ip
- * (ARM_SCRATCH), and transfer through lr. The borrowed reg is chosen to alias
- * neither base. r7-relative bases are unaffected by the SP change, and
- * copy_bytes never addresses through SP. */
- {
- MCEmitter* mc = a->base.mc;
- i32 sb_off, db_off;
- u32 sbase, dbase, xfer;
- NativeLoc lrloc = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
- /* GLOBAL bases fold their offset into the reloc addend and always stay in
- * range, so they never reach here; the only out-of-range bases are
- * FRAME/REG, for which arm_addr_base is side-effect-free. */
- if (src.base_kind == NATIVE_ADDR_BASE_GLOBAL ||
- dst.base_kind == NATIVE_ADDR_BASE_GLOBAL)
- arm_panic(a, "large-offset aggregate copy with global base");
- sbase = arm_addr_base(a, &src, &sb_off);
- dbase = arm_addr_base(a, &dst, &db_off);
- /* Per-granule offsets are relative to the freshly-materialized base (off >=
- * 0), so they stay in arm_emit_mem's inline T3 range (0..4095) as long as
- * the aggregate fits; a copy that large would re-stage through ip/lr and
- * corrupt the held addresses — panic cleanly instead of miscompiling. */
- if (acc.size > 4096u) arm_panic(a, "aggregate copy too large for arm32 lane");
- /* Borrow the first callee-saved reg in {r4,r5,r6} that aliases neither base
- * (r7 is the frame pointer; r8 is the spare). Two bases, three candidates,
- * so one is always free. */
- xfer = (dbase != 4u && sbase != 4u) ? 4u
- : (dbase != 5u && sbase != 5u) ? 5u
- : 6u;
- {
- arm_emit_t32(mc, arm_push_w(1u << xfer));
- /* Materialize dst into the borrowed reg FIRST, while its base is still
- * live; then src into ip. arm_emit_base_off reads `base` then may stage
- * |off| through lr, so neither dst-into-xfer nor src-into-ip disturbs the
- * other side's not-yet-read base (xfer aliases no base; ip is only read by
- * its own src materialization). */
- arm_emit_base_off(a, xfer, dbase, db_off);
- arm_emit_base_off(a, ARM_SCRATCH, sbase, sb_off);
- off = 0;
- while (off < acc.size) {
- u32 rem = acc.size - off;
- u32 sz = rem >= 4u ? 4u : rem >= 2u ? 2u : 1u;
- MemAccess mem = acc.mem;
- NativeAddr s, d;
- mem.size = sz;
- mem.align = sz;
- memset(&s, 0, sizeof s);
- memset(&d, 0, sizeof d);
- s.base_kind = NATIVE_ADDR_BASE_REG;
- s.base.reg = ARM_SCRATCH;
- s.offset = (i32)off;
- d.base_kind = NATIVE_ADDR_BASE_REG;
- d.base.reg = xfer;
- d.offset = (i32)off;
- arm_emit_mem(a, 1, lrloc, s, mem); /* LDR/LDRH/LDRB lr, [ip+off] */
- arm_emit_mem(a, 0, lrloc, d, mem); /* STR/STRH/STRB lr, [xfer+off] */
- off += sz;
- }
- arm_emit_t32(mc, arm_pop_w(1u << xfer));
+ soff = 0;
+ }
+ if (!ddir) {
+ if (dst.base_kind == NATIVE_ADDR_BASE_GLOBAL)
+ arm_emit_global_addr(a, dbase, dst.base.global.sym,
+ dst.base.global.addend + dst.offset);
+ else {
+ u32 nb;
+ i32 no;
+ arm_addr_natural(a, &dst, &nb, &no);
+ arm_emit_base_off(a, dbase, nb, no);
}
+ doff = 0;
+ }
+
+ /* Granule ladder: word, then trailing halfword/byte. Each chunk stays in range
+ * (the resolver guaranteed the span fits, and a materialized base has off 0). */
+ for (off = 0; off < size;) {
+ u32 rem = size - off;
+ u32 sz = rem >= 4u ? 4u : rem >= 2u ? 2u : 1u;
+ arm_emit_mem_one(a, 1, xfer, sbase, soff + (i32)off, sz);
+ arm_emit_mem_one(a, 0, xfer, dbase, doff + (i32)off, sz);
+ off += sz;
}
+ if (borrowed != 0xffu) arm_emit_t32(a->base.mc, arm_pop_w(1u << borrowed));
}
/* set_bytes: store the materialized fill byte `v` across acc.size bytes. The
* common case is struct/array zero-init (v == 0); any fill byte is correct. One
