commit 492813f8d4bc58c72af70327f281a08e832ca959
parent 2e236aa0196bbee9a2c39a5e4c4f9786fd963b99
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Fri, 12 Jun 2026 13:49:07 -0700
perf(cg,arch): light up the -O0 NDT value cache on x64/rv, widen aa64 fp
The single-pass NativeDirectTarget value cache only allocates from a
per-class register pool, and on x64 and riscv that pool was empty: their
NDT allocable lists were callee-saved-only while the cache restricted
itself to caller-saved registers. Every integer local therefore
round-tripped through its stack slot after each op, with only 2-3
transient scratch registers.
Core (native_direct_target.{c,h}): the cache pool is now the caller-saved
subset of allocable on ndt_caller_saved_only targets (aa64, keeping the
deferred prologue free of callee-save spills), else the full allocable
list, with any callee-save covered by the existing reserve_callee_saves
path. Rename caller_alloc -> cache_pool; drop the now-unneeded
caller-saved victim guard.
x64: NDT allocable -> {rsi, rdi, r13, r14, r15}, caller-first (rcx/rax/rdx
excluded -- shift/divide clobber them implicitly; r13-r15 fit the
fixed-size save region).
riscv: NDT allocable -> caller-saved only (int a0-a7; fp ft4-7/ft8-11/
fa0-7) so the cache needs no callee-save spills and the fixed 32-word
single-pass prologue can't overflow on large/variadic frames. Covers
rv64 and rv32 (shared backend).
aa64: widen the fp pool from {v18,v19} with v22-v31 (v16 is the fp emit
scratch, v20/v21 the per-op scratch -- both excluded).
allocable[] is read only by the NDT (the optimizer allocates over phys[]
flags), so each arch's -O0 pool is tunable without touching -O1.
Static -O0 stack mem ops on a header-free compute file:
x86_64 241 -> 159 (-34%), insns 676 -> 551
riscv64 240 -> 138 (-43%), insns 455 -> 353
aarch64 int unchanged; fp bench mem ops 59 -> 51
Verified by execution on all four arches (aa64 native, x64 Rosetta, rv64
podman/qemu) plus toy/cg-api/libc/debug/dwarf/isa suites.
Diffstat:
5 files changed, 218 insertions(+), 65 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -3790,8 +3790,14 @@ static const Reg aa_int_allocable[] = {8u, 11u, 12u, 13u, 14u, 15u, 19u, 20u,
* land. x9/x10/x11 are all caller-saved temporaries reserved out of the
* allocable set below. */
static const Reg aa_int_scratch[] = {9u, 10u, 11u};
-static const Reg aa_fp_allocable[] = {18u, 19u, 8u, 9u, 10u,
- 11u, 12u, 13u, 14u, 15u};
+/* Caller-saved fp temps first (v18/v19, v22..v31) — these form the -O0 value
+ * cache under ndt_caller_saved_only. v16 is the fp emit scratch (the fp AA_TMP0,
+ * used per-op) and v20/v21 are the NDT per-op fp scratch, so all three stay out
+ * of the pool. Callee-saved v8..v15 (AAPCS64) are appended like the int set;
+ * the caller-saved-only NDT never selects them. */
+static const Reg aa_fp_allocable[] = {18u, 19u, 22u, 23u, 24u, 25u, 26u,
+ 27u, 28u, 29u, 30u, 31u, 8u, 9u,
+ 10u, 11u, 12u, 13u, 14u, 15u};
static const Reg aa_fp_scratch[] = {20u, 21u};
#define AA_PHYS_INT_ALLOC(r) \
diff --git a/src/arch/riscv/native.c b/src/arch/riscv/native.c
@@ -44,9 +44,22 @@ enum {
RV_FTMP1 = 1u, /* ft1: emit-internal FP scratch */
RV_FA0 = 10u, /* fa0..fa7 = f10..f17 (FP arg/return registers) */
RV_FA7 = 17u,
- /* Single-pass (-O0) worst-case prologue: sp adjust (3) + far save pair (7)
- * + sret spill (1) + variadic GP spills (8). No callee-saves at -O0. */
- RV_PROLOGUE_WORDS = 32u,
+ /* Single-pass (-O0) tcc-style prologue. The frame-independent entry
+ * `addi sp,sp,-(frame_save_size+va_save) ; sd s0,0(sp) ; sd ra,ptr(sp) ;
+ * addi s0,sp,0`
+ * (RV_NDT_FIXED_ENTRY_WORDS) is emitted live in rv_func_begin: it pre-decrements
+ * sp by just the saved-pair + variadic-GP-save area, stores the s0/ra pair at
+ * the top, and anchors s0 there. The pre-decrement is always small
+ * (<= 16 + 64 = 80 on rv64, 8 + 32 = 40 on rv32), so it always fits imm12 and
+ * needs no lui+add far form. Only the second `sub sp` that grows the rest of
+ * the frame below the pair is deferred (patched once frame_size is final); the
+ * single-pass path uses no callee-saved registers (enforced in the NDT scratch
+ * acquire) and RISC-V has no Windows-style stack probe, so nothing else is
+ * deferred and the reserved region is tiny: worst-case sub = lui(1)+addi(1)+
+ * add(1) = 3 words. The sret + variadic GP spills are s0-relative and emitted
+ * live after the region (rv_emit_entry_save_stores). */
+ RV_NDT_FIXED_ENTRY_WORDS = 4u,
+ RV_NDT_SUB_WORDS = 4u, /* region buffer: worst-case far `sub sp` is 3 (+1 pad) */
/* Known-frame (-O1) prologues are emitted directly, not into the fixed -O0
* NOP region, and additionally save callee-saved registers (up to 11 int + 12
* fp, each up to 4 words for a far s0-relative offset) on top of the header,
@@ -270,6 +283,7 @@ typedef struct RvNativeTarget {
u32 func_start;
u32 prologue_pos;
+ u32 prologue_region_words; /* single-pass: reserved deferred-`sub` region */
MCLabel epilogue_label;
} RvNativeTarget;
@@ -385,11 +399,21 @@ static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0x1fu; }
* `s->arr[i].field`), and the worst case — a 3-operand binop, or such an access
* — has three native-emit scratches live at once. aarch64 reserves three for
* the same reason; x64 needs only two because it folds the index into its
- * addressing mode. s1..s11 (callee-saved) are the allocable set, saved by the
- * optimizer prologue at -O1; rv64 keeps no caller-saved allocable int reg. */
-static const Reg rv_int_allocable[] = {9u, 18u, 19u, 20u, 21u, 22u,
- 23u, 24u, 25u, 26u, 27u};
-static const Reg rv_int_scratch[] = {29u, 30u, 31u}; /* t4, t5, t6 */
+ * addressing mode.
+ *
+ * rv_int_allocable is the NDT (-O0) value-cache / scratch pool and is read only
+ * by NativeDirectTarget; the optimizer allocates over the phys[] ALLOCABLE flags
+ * (s1..s11, callee-saved, saved by its -O1 prologue) instead. The NDT pool is
+ * the caller-saved argument registers a0..a7: the -O0 cache flushes to frame
+ * homes at every call/branch/return, so nothing needs to survive in a register
+ * across a call, and caller-saved regs keep the prologue free of callee-save
+ * spills — which also keeps the single-pass prologue placeholder (a fixed
+ * RV_PROLOGUE_WORDS region) from ever overflowing on large/variadic frames the
+ * way callee-save offsets would. Incoming args are spilled to homes at entry
+ * before any body op, so caching in a0..a7 cannot clobber a live parameter. */
+static const Reg rv_int_allocable[] = {10u, 11u, 12u, 13u, 14u,
+ 15u, 16u, 17u}; /* a0..a7 */
+static const Reg rv_int_scratch[] = {29u, 30u, 31u}; /* t4, t5, t6 */
static const NativePhysRegInfo rv_int_phys[] = {
RV_PHYS_INT_RESERVED(0u), /* zero */
@@ -447,12 +471,18 @@ static const NativePhysRegInfo rv_int_phys[] = {
.spill_cost = 0u, \
.copy_cost = 0u}
-/* Caller-saved allocable first (ft4..ft7, ft8..ft11), then callee (fs0..fs11).
- * ft0/ft1 reserved as emit-internal scratch; ft2/ft3 driver scratch. */
-static const Reg rv_fp_allocable[] = {4u, 5u, 6u, 7u, 28u, 29u, 30u,
- 31u, 8u, 9u, 18u, 19u, 20u, 21u,
- 22u, 23u, 24u, 25u, 26u, 27u};
-static const Reg rv_fp_scratch[] = {2u, 3u}; /* ft2, ft3 */
+/* NDT (-O0) fp value-cache / scratch pool (read only by NativeDirectTarget; the
+ * optimizer uses the phys[] ALLOCABLE flags, which also cover the callee-saved
+ * fs0..fs11). Caller-saved only — ft4..ft7, ft8..ft11, and the fa0..fa7 arg
+ * registers — for the same reasons as the int pool: the -O0 cache flushes at
+ * every call/branch/return, so caller-saved suffices, and keeping callee-saves
+ * out of the pool keeps the fixed single-pass prologue placeholder from
+ * overflowing. ft0/ft1 reserved as emit-internal scratch; ft2/ft3 driver
+ * scratch. */
+static const Reg rv_fp_allocable[] = {4u, 5u, 6u, 7u, 28u, 29u,
+ 30u, 31u, 10u, 11u, 12u, 13u,
+ 14u, 15u, 16u, 17u}; /* ft4-7,ft8-11,fa0-7 */
+static const Reg rv_fp_scratch[] = {2u, 3u}; /* ft2, ft3 */
static const NativePhysRegInfo rv_fp_phys[] = {
RV_PHYS_FP_RESERVED(0u), /* ft0 = FTMP0 */
@@ -1489,6 +1519,12 @@ static void rv_reserve_entry_saves(RvNativeTarget* a) {
}
}
+/* Emit the s0-relative (frame-independent) entry stores live: the sret a0 spill
+ * and, for variadics, the unconsumed-GP register save area. Both are anchored at
+ * s0 (set by the fixed entry in rv_func_begin) and sit at non-negative offsets in
+ * the saved-pair + variadic region the fixed entry already pre-decremented sp
+ * for, so they run correctly after the deferred `sub sp`. Single-pass path only;
+ * the known-frame path emits these inline in rv_build_prologue. */
static void rv_emit_entry_save_stores(RvNativeTarget* a) {
NativeTarget* t = &a->base;
if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
@@ -1502,6 +1538,22 @@ static void rv_emit_entry_save_stores(RvNativeTarget* a) {
rv_emit_mem(a, 0, native_loc_reg(i64t, NATIVE_REG_INT, RV_A0), addr,
native_mem_for_type(t, i64t, ptr));
}
+ /* variadic GP save area: home a0..a7 at [s0 + frame_save_size + i*gp_slot_bytes]
+ * (positive, always within imm12). Emitted live in rv_func_begin, before the
+ * params are bound, so next_param_int is not yet final — home ALL eight GP
+ * registers (the named leading ones are homed too; va_start skips past them via
+ * its next_param_int cursor, so the extra named-slot stores are dead-but-
+ * harmless, mirroring aa64). The save-area offsets are byte-identical to the
+ * historical rv_build_prologue layout. */
+ if (a->is_variadic) {
+ const RiscvVariant* v = a->variant;
+ u32 fsz = v->frame_save_size;
+ u32 gp_slot = v->gp_slot_bytes;
+ u32 i;
+ for (i = 0; i < 8u; ++i)
+ rv64_emit32(t->mc, rv_sd_ptr(v, RV_A0 + i, RV_S0,
+ (i32)fsz + (i32)i * (i32)gp_slot));
+ }
}
/* Collect the callee-saves the body used (none at -O0). */
@@ -1639,13 +1691,74 @@ static u32 rv_build_prologue(RvNativeTarget* a, u32* words, u32 cap,
return wi;
}
+/* Build the deferred portion of the single-pass (-O0) prologue: the `sub sp`
+ * that grows the frame below the saved s0/ra pair the live entry (rv_func_begin)
+ * already saved and anchored s0 at. The fixed entry pre-decremented sp by the
+ * saved-pair + variadic-GP-save area (frame_save_size + va_save_sz), so only
+ * `frame_size - (frame_save_size + va_save_sz)` remains to subtract. RISC-V has
+ * no stack probe, so this is the whole deferred region. Returns the word count
+ * (0 when nothing remains to subtract; <= 3 for the far form). */
+static u32 rv_build_ndt_sub(RvNativeTarget* a, u32 frame_size, u32* words,
+ u32 cap) {
+ const RiscvVariant* v = a->variant;
+ u32 pair_area = v->frame_save_size + rv_va_save_sz(a);
+ u32 sub_bytes = frame_size - pair_area;
+ u32 wi = 0;
+#define ADDI_LO(rd, lo) \
+ (v->has_w_forms ? rv_addiw((rd), (rd), (lo)) : rv_addi((rd), (rd), (lo)))
+#define PUSH(w) \
+ do { \
+ if (wi >= cap) rv_panic(a, "ndt sub placeholder overflow"); \
+ words[wi++] = (w); \
+ } while (0)
+ if (!sub_bytes) return 0;
+ if (fits_i12(-(i32)sub_bytes)) {
+ PUSH(rv_addi(RV_SP, RV_SP, -(i32)sub_bytes));
+ } else {
+ i32 neg = -(i32)sub_bytes;
+ i32 hi = (i32)(((i64)neg + 0x800) >> 12);
+ i32 lo = neg - (i32)((u32)hi << 12);
+ PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
+ if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
+ PUSH(rv_add(RV_SP, RV_SP, RV_TMP0));
+ }
+#undef PUSH
+#undef ADDI_LO
+ return wi;
+}
+
static void rv_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
RvNativeTarget* a = rv_of(t);
MCEmitter* mc = t->mc;
- u32 i;
+ const RiscvVariant* v;
+ u32 pair_area, region, i;
rv_func_begin_common(t, fd);
+ v = a->variant;
+ /* tcc-style prologue. Emit the frame-independent entry live: pre-decrement sp
+ * by just the saved-pair + variadic-GP-save area, store the s0/ra pair at the
+ * top of it, and anchor s0 there. The pair and s0 land at exactly the
+ * addresses every s0-relative offset already assumes (s0 == old_sp -
+ * frame_save_size - va_save_sz, identical to the old `sub frame_size` then
+ * `addi s0,sp,fp_pair_off`), so the frame is byte-identical — only the
+ * instruction sequence changes. pair_area is small (<=80 rv64 / <=40 rv32) and
+ * always fits imm12. */
+ pair_area = v->frame_save_size + rv_va_save_sz(a);
+ rv64_emit32(mc, rv_addi(RV_SP, RV_SP, -(i32)pair_area));
+ rv64_emit32(mc, rv_sd_ptr(v, RV_S0, RV_SP, 0));
+ rv64_emit32(mc, rv_sd_ptr(v, RV_RA, RV_SP, (i32)v->ptr_bytes));
+ rv64_emit32(mc, rv_addi(RV_S0, RV_SP, 0)); /* s0 = sp = saved-pair address */
+ /* Reserve only the deferred `sub sp` region (patched in rv_func_end); record
+ * it for the patch and CFI. RISC-V has no stack probe, so the region is just
+ * the worst-case far `sub`. */
+ region = RV_NDT_SUB_WORDS;
a->prologue_pos = mc->pos(mc);
- for (i = 0; i < RV_PROLOGUE_WORDS; ++i) rv64_emit32(mc, RV_NOP);
+ /* The live fixed entry is exactly RV_NDT_FIXED_ENTRY_WORDS words; the CFI
+ * advance in rv_func_end relies on prologue_pos sitting just past it. */
+ if (a->prologue_pos - a->func_start != RV_NDT_FIXED_ENTRY_WORDS * 4u)
+ rv_panic(a, "fixed prologue entry size drifted");
+ a->prologue_region_words = region;
+ for (i = 0; i < region; ++i) rv64_emit32(mc, RV_NOP);
+ /* sret + variadic GP spills are s0-relative and run live after the region. */
rv_reserve_entry_saves(a);
rv_emit_entry_save_stores(a);
}
@@ -1692,15 +1805,24 @@ static void rv_func_end(NativeTarget* t) {
rv64_emit32(mc, rv_jalr(RV_ZERO, RV_RA, 0));
}
- /* patch prologue */
+ /* patch the deferred-`sub` region. The frame-independent entry (saved pair +
+ * s0 anchor) was already emitted live in rv_func_begin; only the `sub sp` that
+ * grows the frame below the pair is deferred here. Single-pass path only — the
+ * known-frame path emits its prologue final and never patches. */
if (!a->frame.known_frame) {
- u32 words[RV_PROLOGUE_WORDS];
+ u32 region = a->prologue_region_words;
+ u32 words[RV_NDT_SUB_WORDS];
u32 nwords, k;
- for (k = 0; k < RV_PROLOGUE_WORDS; ++k) words[k] = RV_NOP;
- nwords = rv_build_prologue(a, words, RV_PROLOGUE_WORDS, frame_size,
- fp_pair_off, int_regs, n_int, fp_regs, n_fp);
- (void)nwords;
- for (k = 0; k < RV_PROLOGUE_WORDS; ++k)
+ if (region > RV_NDT_SUB_WORDS) rv_panic(a, "prologue region too large");
+ for (k = 0; k < region; ++k) words[k] = RV_NOP;
+ nwords = rv_build_ndt_sub(a, frame_size, words, region);
+ /* If the deferred `sub` is shorter than the reserved region, branch straight
+ * to the entry saves rather than executing the trailing NOPs. */
+ if (nwords < region) {
+ words[nwords] = rv_jal(RV_ZERO, (i32)((region - nwords) * 4u));
+ for (k = nwords + 1u; k < region; ++k) words[k] = RV_NOP;
+ }
+ for (k = 0; k < region; ++k)
rv_patch32(obj, sec, a->prologue_pos + k * 4u, words[k]);
}
/* patch alloca sites: addi dst, sp, max_outgoing */
@@ -1723,9 +1845,15 @@ static void rv_func_end(NativeTarget* t) {
mc->cfi_def_cfa(mc, RV_SP, 0);
} else {
i32 cfa = (i32)frame_size - (i32)fp_pair_off;
+ /* CFI advance to the post-prologue PC. known-frame: prologue_pos +
+ * minimal_prologue_words. single-pass: prologue_pos is past the live fixed
+ * entry, so add the reserved deferred-`sub` region (the saved pair + s0
+ * anchor are already established by the fixed entry; the deferred `sub` is
+ * what completes the frame). The sret/variadic spills after the region do
+ * not move sp/s0, so the CFA state holds from here. */
u32 post = a->prologue_pos + (a->frame.known_frame
? a->minimal_prologue_words * 4u
- : RV_PROLOGUE_WORDS * 4u);
+ : a->prologue_region_words * 4u);
u32 k;
mc->cfi_set_next_pc_offset(mc, post - a->func_start);
mc->cfi_def_cfa(mc, RV_S0, cfa);
diff --git a/src/arch/x64/native.c b/src/arch/x64/native.c
@@ -211,10 +211,17 @@ static void emit_jcc_rel32(MCEmitter* mc, u32 cc, MCLabel l);
.spill_cost = 0u, \
.copy_cost = 0u}
-/* Allocable int pool, opt's spill/reload set. R8/R9 are the driver scratch
- * pool; R10/R11 are emit scratch (reserved); RAX is reserved (return / div-mul,
- * asm-pinnable). */
-static const Reg x64_int_allocable[] = {X64_R13, X64_R14, X64_R15};
+/* The NDT (-O0) value-cache / scratch pool (read only by NativeDirectTarget;
+ * the optimizer allocates over the phys[] ALLOCABLE flags instead, which also
+ * include rcx/rdx/rsi/rdi). Caller-saved RSI/RDI lead so the cache prefers them
+ * (no prologue save) and only spills into callee-saved R13-R15 under pressure
+ * (the fixed-size save region covers them). RSI/RDI are safe to cache: they have
+ * no implicit x86 use — unlike RCX (shift count) and RAX/RDX (div/mul), which
+ * are therefore excluded — and incoming args are spilled to frame homes at entry
+ * before any body op runs. R8/R9 are the driver scratch pool; R10/R11 are emit
+ * scratch (reserved); RAX is reserved (return / div-mul, asm-pinnable). */
+static const Reg x64_int_allocable[] = {X64_RSI, X64_RDI, X64_R13, X64_R14,
+ X64_R15};
static const Reg x64_int_scratch[] = {X64_R8, X64_R9};
static const NativePhysRegInfo x64_int_phys[] = {
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -554,28 +554,28 @@ static Reg nd_pick_cache_victim(NativeDirectTarget* d, NativeAllocClass cls) {
return best;
}
-/* Pick a caller-saved allocable register to cache a local in: a free one, else
- * evict the LRU non-pinned cached local. REG_NONE means use the frame-only
- * path. */
+/* Pick an allocable register to cache a local in: a free one from the cache
+ * pool, else evict the LRU non-pinned cached local. REG_NONE means use the
+ * frame-only path. */
static Reg nd_cache_alloc(NativeDirectTarget* d, NativeAllocClass cls) {
- u32 caller = d->caller_saved[cls];
- const Reg* sub = d->caller_alloc[cls];
- u32 nsub = d->ncaller_alloc[cls];
+ const Reg* pool = d->cache_pool[cls];
+ u32 npool = d->ncache_pool[cls];
Reg victim;
- /* The cache only ever allocates caller-saved registers, so scan the
- * precomputed caller-saved-allocable sublist (allocable order preserved) and
- * skip the always-failing callee-saved candidates. First match is identical
- * to the former full-allocable scan. */
- for (u32 i = 0; i < nsub; ++i) {
- Reg r = sub[i];
+ /* Scan the precomputed cache pool (allocable order preserved; on
+ * ndt_caller_saved_only targets it is the caller-saved subset). */
+ for (u32 i = 0; i < npool; ++i) {
+ Reg r = pool[i];
if (d->reg_owner[cls][r] == CG_LOCAL_NONE &&
(d->scratch_used[cls] & (1u << r)) == 0) {
nd_note_reg_used(d, cls, r);
return r;
}
}
+ /* Under pressure, evict the LRU non-pinned cached local and reuse its
+ * register. Every cached local was allocated from cache_pool, so the victim
+ * is always a valid pool register — no caller/callee guard needed. */
victim = nd_pick_cache_victim(d, cls);
- if (victim != REG_NONE && (caller & (1u << victim))) {
+ if (victim != REG_NONE) {
nd_flush_local(d, d->reg_owner[cls][victim]);
nd_note_reg_used(d, cls, victim);
return victim;
@@ -1954,21 +1954,27 @@ CgTarget* native_direct_target_new(Compiler* c, ObjBuilder* obj,
}
}
- /* Precompute, per class, the caller-saved-allocable sublist (in allocable
- * order) and the live-ABI caller-saved mask, both constant for the program.
- * The local register cache only allocates caller-saved registers, so the
- * free-reg scan iterates this short list and avoids re-querying the mask. */
- for (u32 cls = 0; cls < 3u; ++cls) {
- const NativeAllocClassInfo* ci = d->class_info[cls];
- u32 mask, n = 0;
- if (!ci) continue;
- mask = nd_caller_saved_mask(d, (NativeAllocClass)cls);
- d->caller_saved[cls] = mask;
- for (u32 i = 0; i < ci->nallocable; ++i) {
- Reg r = ci->allocable[i];
- if (r < 32u && (mask & (1u << r))) d->caller_alloc[cls][n++] = r;
+ /* Precompute, per class, the -O0 value-cache register pool (in allocable
+ * order), constant for the program. On ndt_caller_saved_only targets the pool
+ * is the caller-saved subset of allocable (the cache then never forces a
+ * callee-save spill into the deferred prologue); otherwise it is the full
+ * allocable list, with any callee-save in it reported to the backend on use
+ * (nd_note_reg_used) and reserved by reserve_callee_saves. */
+ {
+ int caller_only = d->reg_info && d->reg_info->ndt_caller_saved_only;
+ for (u32 cls = 0; cls < 3u; ++cls) {
+ const NativeAllocClassInfo* ci = d->class_info[cls];
+ u32 mask, n = 0;
+ if (!ci) continue;
+ mask = nd_caller_saved_mask(d, (NativeAllocClass)cls);
+ for (u32 i = 0; i < ci->nallocable; ++i) {
+ Reg r = ci->allocable[i];
+ if (r >= 32u) continue;
+ if (caller_only && !(mask & (1u << r))) continue;
+ d->cache_pool[cls][n++] = r;
+ }
+ d->ncache_pool[cls] = n;
}
- d->ncaller_alloc[cls] = n;
}
d->base.func_begin = nd_func_begin;
diff --git a/src/cg/native_direct_target.h b/src/cg/native_direct_target.h
@@ -120,14 +120,20 @@ struct NativeDirectTarget {
const NativeRegInfo* reg_info;
const NativeAllocClassInfo* class_info[3];
- /* Per-class caller-saved-allocable sublist, precomputed at construction in
- * `allocable[]` order (the cache only ever allocates caller-saved registers,
- * so the free-reg scan iterates this short list instead of all allocable
- * registers, and avoids re-querying the live-ABI caller-saved mask per call).
- * caller_saved[cls] is that same mask cached. */
- Reg caller_alloc[3][32];
- u32 ncaller_alloc[3];
- u32 caller_saved[3];
+ /* Per-class -O0 value-cache register pool, precomputed at construction in
+ * `allocable[]` order. On ndt_caller_saved_only targets (aa64) it is the
+ * caller-saved subset of allocable, so the deferred prologue never has to
+ * spill a callee-save. Other targets take the whole allocable list: their NDT
+ * allocable sets are deliberately hazard-free (no implicit-clobber registers
+ * such as the x86 shift/divide regs) and any callee-save among them is either
+ * covered by the reserve_callee_saves prologue path (x64 r13-r15, whose save
+ * region is fixed-size) or is itself caller-saved (riscv a-regs), so caching
+ * in them is both safe and free of a single-pass prologue-budget blowup.
+ * Note: `allocable[]` is read only by the NDT, never the optimizer (which
+ * allocates over the phys[] ALLOCABLE flags), so each arch tunes it for -O0
+ * register caching independently of -O1. */
+ Reg cache_pool[3][32];
+ u32 ncache_pool[3];
const CGFuncDesc* func;
SrcLoc loc;
