commit f6c08eea644fb990f6c4deaade3b295923dfe392
parent dc176d0cf6aabfefbc2e0b8b68c688d1843b68e9
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Wed, 17 Jun 2026 00:41:41 -0700
cg: NDT span-aware scratch — keep a wide value off a reserved high lane
A wide (8-byte) integer on a 32-bit target occupies a register PAIR (r:r+1).
nd_scratch_acquire handed out a single register regardless of width, so the
backend's wide load/store split (rt, rt+1) could land the high lane on a
reserved register. On arm32 the lone scratch ip(r12) pairs onto r13/sp, so an
i64 switch scrutinee materialized for a comparison clobbered the stack pointer
(123_spec_demo/-O0 lockup). riscv32 avoids it only by luck of three adjacent
scratch regs.
nd_scratch_acquire_span finds a base whose whole register run lies within the
class's acquirable set (scratch + allocable), pinning the run. span==1 — every
narrow value and every 64-bit target — is byte-identical to before.
Diffstat:
1 file changed, 103 insertions(+), 15 deletions(-)
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -175,15 +175,50 @@ static void nd_note_reg_used(NativeDirectTarget* d, NativeAllocClass cls,
d->callee_saved_used[cls] |= 1u << reg;
}
-static Reg nd_scratch_acquire(NativeDirectTarget* d, NativeAllocClass cls) {
+/* Mask of registers the single-pass path may hand out as scratch in `cls` — its
+ * dedicated scratch list plus the allocable pool. A multi-register (wide) value's
+ * whole register run must lie within this set so each lane is a real, writable
+ * GPR the NDT owns: this is what keeps a wide value off a base whose high lane is
+ * a reserved register (the arm32 trap, where the lone scratch ip(r12) would pair
+ * its high lane onto r13/sp). On 64-bit targets every value is one register, so
+ * the run is the base alone and this never narrows the choice. */
+static u32 nd_acquirable_mask(const NativeAllocClassInfo* ci) {
+ u32 m = 0u, i;
+ for (i = 0; i < ci->nscratch; ++i)
+ if (ci->scratch[i] < 32u) m |= 1u << ci->scratch[i];
+ for (i = 0; i < ci->nallocable; ++i)
+ if (ci->allocable[i] < 32u) m |= 1u << ci->allocable[i];
+ return m;
+}
+
+/* A `span`-register value placed at base `r` is legal iff the high lanes
+ * r+1..r+span-1 are all acquirable registers (never sp/pc/fp/lr). The base lane
+ * itself is whatever the caller's scratch/allocable scan already vetted (it may
+ * legitimately be a reserved-from-allocation scratch like ip). */
+static int nd_span_high_acquirable(u32 acq, Reg r, u32 span) {
+ u32 k;
+ for (k = 1u; k < span; ++k)
+ if ((u32)(r + k) >= 32u || !(acq & (1u << (r + k)))) return 0;
+ return 1;
+}
+
+/* Acquire a run of `span` consecutive registers (span==1 is the common case)
+ * whose base is returned; the whole run is pinned. span>1 only arises for a wide
+ * (8-byte) integer value on a 32-bit target, which occupies a register pair. */
+static Reg nd_scratch_acquire_span(NativeDirectTarget* d, NativeAllocClass cls,
+ u32 span) {
const NativeAllocClassInfo* ci = nd_class_info(d, cls);
const Reg* regs = ci->scratch;
u32 nregs = ci->nscratch;
- /* Prefer a register that is neither pinned (scratch_used) nor caching a live
- * local (reg_owner). */
+ u32 acq = span > 1u ? nd_acquirable_mask(ci) : 0u;
+ if (span < 1u) span = 1u;
+ /* Prefer a register (run) that is neither pinned (scratch_used) nor caching a
+ * live local (reg_owner). */
for (u32 pass = 0; pass < 2u; ++pass) {
for (u32 i = 0; i < nregs; ++i) {
Reg r = regs[i];
+ u32 k;
+ int run_ok = 1;
if (r >= 32u) continue;
/* On targets whose register model declares ndt_caller_saved_only, the
* single-pass (-O0) path never takes a callee-saved register as scratch
@@ -196,19 +231,30 @@ static Reg nd_scratch_acquire(NativeDirectTarget* d, NativeAllocClass cls) {
if (d->reg_info && d->reg_info->ndt_caller_saved_only &&
(nd_callee_saved_mask(d, cls) & (1u << r)))
continue;
- if ((d->scratch_used[cls] & (1u << r)) == 0 &&
- d->reg_owner[cls][r] == CG_LOCAL_NONE) {
- d->scratch_used[cls] |= 1u << r;
- nd_note_reg_used(d, cls, r);
- return r;
+ if (span > 1u && !nd_span_high_acquirable(acq, r, span)) continue;
+ /* The whole run must be free: none pinned, none caching a live local. */
+ for (k = 0; k < span; ++k) {
+ Reg h = r + k;
+ if ((d->scratch_used[cls] & (1u << h)) != 0 ||
+ d->reg_owner[cls][h] != CG_LOCAL_NONE) {
+ run_ok = 0;
+ break;
+ }
}
+ if (!run_ok) continue;
+ for (k = 0; k < span; ++k) {
+ d->scratch_used[cls] |= 1u << (r + k);
+ nd_note_reg_used(d, cls, r + k);
+ }
+ return r;
}
regs = ci->allocable;
nregs = ci->nallocable;
}
/* Under pressure, evict the LRU non-pinned cached local (spilling it to its
- * home) and reuse its register as a scratch temporary. */
- {
+ * home) and reuse its register as a scratch temporary. A wide run evicts every
+ * cached local across the run; it never crosses a pinned register. */
+ if (span <= 1u) {
Reg r = nd_pick_cache_victim(d, cls);
if (r != REG_NONE) {
nd_flush_local(d, d->reg_owner[cls][r]);
@@ -216,10 +262,47 @@ static Reg nd_scratch_acquire(NativeDirectTarget* d, NativeAllocClass cls) {
nd_note_reg_used(d, cls, r);
return r;
}
+ } else {
+ const Reg* alloc = ci->allocable;
+ for (u32 i = 0; i < ci->nallocable; ++i) {
+ Reg r = alloc[i];
+ u32 k;
+ int run_ok = 1;
+ if (r >= 32u || !nd_span_high_acquirable(acq, r, span)) continue;
+ for (k = 0; k < span; ++k)
+ if ((d->scratch_used[cls] & (1u << (r + k))) != 0) { run_ok = 0; break; }
+ if (!run_ok) continue;
+ for (k = 0; k < span; ++k)
+ if (d->reg_owner[cls][r + k] != CG_LOCAL_NONE)
+ nd_flush_local(d, d->reg_owner[cls][r + k]);
+ for (k = 0; k < span; ++k) {
+ d->scratch_used[cls] |= 1u << (r + k);
+ nd_note_reg_used(d, cls, r + k);
+ }
+ return r;
+ }
}
nd_panic(d, "out of scratch registers");
}
+static Reg nd_scratch_acquire(NativeDirectTarget* d, NativeAllocClass cls) {
+ return nd_scratch_acquire_span(d, cls, 1u);
+}
+
+/* Number of consecutive registers a value of `type` occupies in class `cls`. On
+ * a 32-bit target an 8-byte integer (i64 / soft-double) is a register pair; a
+ * value that fits the register width — and any value in a non-integer class,
+ * which holds wide data in a single vector register — is one register. */
+static u32 nd_reg_span(NativeDirectTarget* d, NativeAllocClass cls,
+ KitCgTypeId type) {
+ u32 ptr;
+ u64 size;
+ if (cls != NATIVE_REG_INT || type == 0) return 1u;
+ ptr = (u32)d->base.c->target.ptr_size;
+ size = cg_type_size(d->base.c, type);
+ return (ptr && size > (u64)ptr) ? (u32)((size + ptr - 1u) / ptr) : 1u;
+}
+
static void nd_scratch_release(NativeDirectTarget* d, NativeAllocClass cls,
Reg reg) {
if (reg < 32u) d->scratch_used[cls] &= ~(1u << reg);
@@ -837,9 +920,11 @@ static NativeLoc nd_materialize_loc(NativeDirectTarget* d, NativeLoc src,
NativeAllocClass cls, KitCgTypeId type) {
Reg r;
NativeLoc dst;
+ KitCgTypeId dt;
if (src.kind == NATIVE_LOC_REG) return src;
- r = nd_scratch_acquire(d, cls);
- dst = nd_loc_reg(d, type ? type : src.type, cls, r);
+ dt = type ? type : src.type;
+ r = nd_scratch_acquire_span(d, cls, nd_reg_span(d, cls, dt));
+ dst = nd_loc_reg(d, dt, cls, r);
nd_copy_to_reg(d, dst, src);
return dst;
}
@@ -942,8 +1027,11 @@ static void nd_write_loc(NativeDirectTarget* d, NativeLoc dst, NativeLoc src,
}
static void nd_release_materialized(NativeDirectTarget* d, NativeLoc loc) {
- if (loc.kind == NATIVE_LOC_REG)
- nd_scratch_release(d, (NativeAllocClass)loc.cls, loc.v.reg);
+ if (loc.kind == NATIVE_LOC_REG) {
+ u32 span = nd_reg_span(d, (NativeAllocClass)loc.cls, loc.type), k;
+ for (k = 0; k < span; ++k)
+ nd_scratch_release(d, (NativeAllocClass)loc.cls, (Reg)(loc.v.reg + k));
+ }
}
/* Spill cached locals that back an INDIRECT operand's address before it is read
@@ -1014,7 +1102,7 @@ static void nd_materialize_operand_into(NativeDirectTarget* d, NativeLoc want,
static NativeLoc nd_dst_scratch(NativeDirectTarget* d, Operand dst) {
NativeAllocClass cls = nd_class_for_type(d, dst.type);
- Reg r = nd_scratch_acquire(d, cls);
+ Reg r = nd_scratch_acquire_span(d, cls, nd_reg_span(d, cls, dst.type));
return nd_loc_reg(d, dst.type, cls, r);
}
