commit 307256a262313dbba445b0e35eb0c331cc395ca3
parent fdc9c73ea5e848d900c1bc154101d3ee41a0d5c3
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 19:16:17 -0700
arm32: fix soft-double helper miscompile (copy_bytes large-offset scratch clobber)
The wide8 (i64/double) store `*p = v` and large-frame aggregate copies
miscompiled when the destination pointer base lived in ip (ARM_SCRATCH) — the
NDT's legitimate operand-materialization register — and the source was a deep
frame slot needing large-offset staging. arm_copy_bytes transfers each granule
through lr; when the source load hit arm_emit_mem's large-offset path it staged
the address into ip (the only other scratch), clobbering the dst pointer that
the very next store needed. The store then targeted the wrong address.
This broke __muldf3 and __divdf3 (and any double helper using the 64x64->128
wideMultiply, whose final `*hi = ...; *lo = ...` are wide8 stores through
pointers from a big frame); __adddf3/__subdf3 happened to dodge it. Single
precision was fine (no wide8 mantissa product).
Fix: give arm_copy_bytes a slow path for out-of-range offsets that materializes
each side's effective address into a stable register once (dst in a borrowed
callee-saved reg chosen to alias neither base, src in ip), then copies at small
inline offsets so arm_emit_mem never stages again. The fast in-range path is
unchanged.
Before (final *hi store, src at r7-304, dst ptr in ip):
ldr.w ip, [r7, #-20] ; ip = *hi pointer
mov.w ip, #304 ; CLOBBERS ip (stage src offset)
sub.w ip, r7, ip ; ip = r7-304
ldr.w lr, [ip]
str.w lr, [ip] ; WRONG: stores to r7-304, not *hi
After:
ldr.w ip, [r7, #-20] ; ip = *hi pointer
push.w {r4}
mov r4, ip ; preserve dst pointer in r4
sub.w ip, r7, #304 ; ip = src address
ldr.w lr, [ip]
str.w lr, [r4] ; correct
...
pop.w {r4}
Repro `double a=1.5,b=2.25; a*b==3.375` (and a/b) goes 1->0 at -O0 and -O1 under
both ld.lld and kit ld against the kit-built rt. make test-smoke-arm32 now 4/0
(soft -O0, soft -O1, kit-ld auto-rt all reach qemu rc=0). test-cg-api green;
arm32_decode 314/0; toy 1392/0.
Diffstat:
1 file changed, 136 insertions(+), 25 deletions(-)
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -1758,38 +1758,149 @@ static void arm_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
arm_emit_global_addr(arm_of(t), loc_reg(dst), sym, addend);
}
/* copy_bytes / set_bytes: aggregate (struct/array, and the wide8 i64/double)
- * memory ops. Both keep the materialized src/dst NativeAddr bases as-is and bake
- * the running byte offset into the load/store immediate (arm_emit_mem dispatches
- * the width and the T3/T4 offset encoding), mirroring the aa64 sibling backend.
+ * memory ops, transferring a word/halfword/byte ladder through ARM_TMP (lr),
+ * mirroring the aa64 sibling backend.
*
- * The only transfer scratch is ARM_TMP (lr): it is RESERVED (never holds an NDT
- * operand or a materialized address base — base regs are r7/r0..r3/r4..r11/ip)
- * and dead in the body, so a single copy needs no extra register. Because the
- * offset is folded into the access, deep frame slots or large aggregates can
- * push the effective offset past the encodable range; arm_emit_mem then panics
- * cleanly (the -O0 frame floor of [r7,#-255] is a pre-existing Phase-1 limit,
- * not widened here). */
+ * copy_bytes has two regimes. The common FAST path keeps the src/dst NativeAddr
+ * bases as-is and bakes the running byte offset into each load/store immediate
+ * (arm_emit_mem dispatches the width + T3/T4 offset encoding); lr is the only
+ * scratch and it is RESERVED, so a single copy needs no extra register. When a
+ * side's effective offset would leave the inline immediate range (a deep -O0
+ * frame slot / wide8 store through a pointer the NDT homed in ip), arm_emit_mem
+ * would have to stage the address through a *second* scratch (ip) — which can
+ * be the sibling base register, silently corrupting it. The SLOW path below
+ * therefore materializes each side's effective address into a stable register
+ * once and copies at small offsets, so no per-granule staging happens. */
+/* True if a load/store at signed byte offset `off` from a base fits
+ * arm_emit_mem's inline T3/T4 immediate forms (no large-offset staging). */
+static int arm_mem_off_inline(i32 off) {
+ return (off >= 0 && off <= 4095) || (off >= -255 && off <= 255);
+}
+
+/* Side-effect-free preview of the per-base byte offset an access would carry,
+ * for the copy_bytes range check below. Mirrors arm_addr_base's FRAME/REG
+ * offset math WITHOUT emitting the GLOBAL MOVW/MOVT (which arm_addr_base does as
+ * a side effect). A GLOBAL base re-materializes its address into lr per access
+ * and folds the addend into the reloc, so it never reaches the large-offset
+ * lane — report it as in-range. */
+static i32 arm_addr_peek_off(Arm32NativeTarget* a, const NativeAddr* addr) {
+ switch (addr->base_kind) {
+ case NATIVE_ADDR_BASE_FRAME: {
+ NativeFrameSlotEntry* s =
+ native_frame_slot_at(&a->frame, addr->base.frame);
+ return -(i32)s->off + addr->offset;
+ }
+ case NATIVE_ADDR_BASE_REG:
+ return addr->offset;
+ default:
+ return 0; /* GLOBAL (in-range by construction) or unsupported */
+ }
+}
+
static void arm_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
AggregateAccess acc) {
Arm32NativeTarget* a = arm_of(t);
KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
NativeLoc tmp = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
+ i32 soff = arm_addr_peek_off(a, &src);
+ i32 doff = arm_addr_peek_off(a, &dst);
+ i32 last = acc.size ? (i32)(acc.size - 1u) : 0;
u32 off = 0;
- /* Widest convenient granule is a word (4); the trailing 1/2/3 bytes fall to
- * halfword/byte ops. The width ladder also keeps each chunk naturally aligned
- * for an aligned aggregate (4|4|...|2|1) and never over-runs acc.size. */
- while (off < acc.size) {
- u32 rem = acc.size - off;
- u32 sz = rem >= 4u ? 4u : rem >= 2u ? 2u : 1u;
- MemAccess mem = acc.mem;
- NativeAddr s = src, d = dst;
- mem.size = sz;
- mem.align = sz;
- s.offset += (i32)off;
- d.offset += (i32)off;
- arm_emit_mem(a, 1, tmp, s, mem); /* LDR/LDRH/LDRB tmp, [src+off] */
- arm_emit_mem(a, 0, tmp, d, mem); /* STR/STRH/STRB tmp, [dst+off] */
- off += sz;
+ /* Fast path: every per-granule access on BOTH sides stays in the inline
+ * immediate range, so arm_emit_mem never needs to stage an offset through a
+ * scratch register and cannot collide with the sibling base. */
+ if (arm_mem_off_inline(soff) && arm_mem_off_inline(soff + last) &&
+ arm_mem_off_inline(doff) && arm_mem_off_inline(doff + last)) {
+ /* Widest convenient granule is a word (4); the trailing 1/2/3 bytes fall to
+ * halfword/byte ops. The width ladder also keeps each chunk naturally
+ * aligned for an aligned aggregate (4|4|...|2|1) and never over-runs
+ * acc.size. */
+ while (off < acc.size) {
+ u32 rem = acc.size - off;
+ u32 sz = rem >= 4u ? 4u : rem >= 2u ? 2u : 1u;
+ MemAccess mem = acc.mem;
+ NativeAddr s = src, d = dst;
+ mem.size = sz;
+ mem.align = sz;
+ s.offset += (i32)off;
+ d.offset += (i32)off;
+ arm_emit_mem(a, 1, tmp, s, mem); /* LDR/LDRH/LDRB tmp, [src+off] */
+ arm_emit_mem(a, 0, tmp, d, mem); /* STR/STRH/STRB tmp, [dst+off] */
+ off += sz;
+ }
+ return;
+ }
+ /* Slow path: at least one side reaches a deep frame offset (large -O0 frame /
+ * wide8 i64 copy). arm_emit_mem's large-offset path stages the address into a
+ * scratch register (ip if the transfer reg is lr), which would clobber a
+ * sibling base that the NDT legitimately materialized into ip — the wide8
+ * `*p = i64` miscompile. Instead, compute each side's full effective address
+ * into a stable register ONCE, then access it at small (>=0) offsets so
+ * arm_emit_mem stays on its inline T3 path and never stages again.
+ *
+ * Register budget: the per-granule walk needs three distinct regs — src addr,
+ * dst addr, transfer. r7 is the frame pointer; lr (ARM_TMP) is consumed by
+ * arm_emit_base_off as its own MOVW/MOVT staging scratch, so it can't also
+ * hold a live address during setup. We therefore hold the dst address in a
+ * borrowed callee-saved reg (pushed/popped here) and the src address in ip
+ * (ARM_SCRATCH), and transfer through lr. The borrowed reg is chosen to alias
+ * neither base. r7-relative bases are unaffected by the SP change, and
+ * copy_bytes never addresses through SP. */
+ {
+ MCEmitter* mc = a->base.mc;
+ i32 sb_off, db_off;
+ u32 sbase, dbase, xfer;
+ NativeLoc lrloc = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
+ /* GLOBAL bases fold their offset into the reloc addend and always stay in
+ * range, so they never reach here; the only out-of-range bases are
+ * FRAME/REG, for which arm_addr_base is side-effect-free. */
+ if (src.base_kind == NATIVE_ADDR_BASE_GLOBAL ||
+ dst.base_kind == NATIVE_ADDR_BASE_GLOBAL)
+ arm_panic(a, "large-offset aggregate copy with global base");
+ sbase = arm_addr_base(a, &src, &sb_off);
+ dbase = arm_addr_base(a, &dst, &db_off);
+ /* Per-granule offsets are relative to the freshly-materialized base (off >=
+ * 0), so they stay in arm_emit_mem's inline T3 range (0..4095) as long as
+ * the aggregate fits; a copy that large would re-stage through ip/lr and
+ * corrupt the held addresses — panic cleanly instead of miscompiling. */
+ if (acc.size > 4096u) arm_panic(a, "aggregate copy too large for arm32 lane");
+ /* Borrow the first callee-saved reg in {r4,r5,r6} that aliases neither base
+ * (r7 is the frame pointer; r8 is the spare). Two bases, three candidates,
+ * so one is always free. */
+ xfer = (dbase != 4u && sbase != 4u) ? 4u
+ : (dbase != 5u && sbase != 5u) ? 5u
+ : 6u;
+ {
+ arm_emit_t32(mc, arm_push_w(1u << xfer));
+ /* Materialize dst into the borrowed reg FIRST, while its base is still
+ * live; then src into ip. arm_emit_base_off reads `base` then may stage
+ * |off| through lr, so neither dst-into-xfer nor src-into-ip disturbs the
+ * other side's not-yet-read base (xfer aliases no base; ip is only read by
+ * its own src materialization). */
+ arm_emit_base_off(a, xfer, dbase, db_off);
+ arm_emit_base_off(a, ARM_SCRATCH, sbase, sb_off);
+ off = 0;
+ while (off < acc.size) {
+ u32 rem = acc.size - off;
+ u32 sz = rem >= 4u ? 4u : rem >= 2u ? 2u : 1u;
+ MemAccess mem = acc.mem;
+ NativeAddr s, d;
+ mem.size = sz;
+ mem.align = sz;
+ memset(&s, 0, sizeof s);
+ memset(&d, 0, sizeof d);
+ s.base_kind = NATIVE_ADDR_BASE_REG;
+ s.base.reg = ARM_SCRATCH;
+ s.offset = (i32)off;
+ d.base_kind = NATIVE_ADDR_BASE_REG;
+ d.base.reg = xfer;
+ d.offset = (i32)off;
+ arm_emit_mem(a, 1, lrloc, s, mem); /* LDR/LDRH/LDRB lr, [ip+off] */
+ arm_emit_mem(a, 0, lrloc, d, mem); /* STR/STRH/STRB lr, [xfer+off] */
+ off += sz;
+ }
+ arm_emit_t32(mc, arm_pop_w(1u << xfer));
+ }
}
}
/* set_bytes: store the materialized fill byte `v` across acc.size bytes. The
