commit 5fe95f6d07bbf02f77cf5801af0aff3ee7f01e45
parent 1e0bc91f3e6f1089e6fa38b438a86b7c38b4b5ab
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 12:41:52 -0700
perf(cg): aa64 -O0 positive-offset far-slot addressing (Lever 1 / Fix B)
A fixed frame slot past stur's signed-9-bit reach (-256) was addressed by
building its address first: sub xN,x29,#off (+ movk when off>4095) ; ldur [xN].
On sqlite that is 29,379 sub + ~4,000 movk just to reach locals.
Instead address such 4/8-byte int/fp slots at a positive *scaled* offset from a
stable bottom anchor: ldr/str [sp, #frame_size-off] reaches 32 KB in one insn,
zero address building. The frame size isn't known mid-body on the single-pass
path, so emit a one-word placeholder ldr/str [sp,#0] and an AA_PATCH_SLOT, then
rewrite the scaled offset in aa_func_end once the frame is final (reusing the
existing deferred-patch machinery). Top-record layout and CFI are unchanged
(fp still anchors the saved pair); only the slot access form changes, so out-
going-stack-arg frames are handled too.
Alloca floats sp, so such functions anchor a callee-saved AA_FRAME_BASE (x28) at
the frame base in the prologue (saved/restored like a callee-save, armed lazily
on the first alloca) and the slot patches address it instead of sp — same one-
word positive scaled form. Byte/half slots and Windows GP-home-area frames keep
the existing fp-relative path (their scaled reach is too small / the layout
shifts). Frames beyond dword scaled reach (32 KB at -O0) are a documented assert.
sqlite3.c -O0 .text: -25,579 insns (sub -22,273, movk -1,989); 1.3215x ->
1.2468x tcc. Gate: determinism + sqlite e2e O0/O1 (golden+vs-clang) + toy/parse/
smoke-x64/smoke-rv64/dwarf/debug all green; alloca (big frame + sp-move) and
far-slot/FP/addr-taken/stack-arg clang-differential probes match clang.
Diffstat:
1 file changed, 141 insertions(+), 1 deletion(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -59,6 +59,12 @@ enum {
aa_copy_bytes (which scratches only x16/x17) never clobbers */
AA_TMP0 = 16u,
AA_TMP1 = 17u,
+ /* Callee-saved frame-base anchor. The single-pass far-slot fast path
+ * (aa_emit_mem / AA_PATCH_SLOT) addresses fixed slots at positive scaled
+ * offsets from a stable bottom anchor: sp itself when the function never moves
+ * sp, else this register, set to the frame base in the prologue and saved /
+ * restored like a callee-save (only when the function calls alloca). */
+ AA_FRAME_BASE = 28u,
AA_FP = 29u,
AA_LR = 30u,
AA_SP = 31u,
@@ -190,14 +196,32 @@ typedef NativeFrameSlotEntry AANativeSlot;
typedef enum AAPatchKind {
AA_PATCH_ALLOCA, /* single instr: add dst, sp, #max_outgoing */
AA_PATCH_TAIL, /* AA_TAIL_WORDS region: callee restores + frame + br/b */
+ AA_PATCH_SLOT, /* single instr: far fixed-slot ldr/str at a positive scaled
+ * offset from the frame base (sp, or AA_FRAME_BASE under
+ * alloca), resolved once frame_size is final. */
} AAPatchKind;
+/* A deferred far fixed-slot access. The body emits a one-word placeholder
+ * `ldr/str [sp,#0]`; aa_apply_patches rewrites the base register and scaled
+ * offset once the frame is final. slot_off + extra is the slot's byte position
+ * (as fed to aa_fp_off_slot); the positive frame-base offset is
+ * aa_sp_off_saved_pair(L) + aa_fp_off_slot(slot)+extra (top-record). */
+typedef struct AASlotPatch {
+ u32 slot_off; /* aa_slot(...)->off */
+ i32 extra; /* addr.offset added to the slot base */
+ u8 sz; /* size_idx: 2 (word) or 3 (dword) */
+ u8 vbit; /* SIMD/FP register */
+ u8 load; /* 1 = ldr, 0 = str */
+ u8 rt; /* transfer register */
+} AASlotPatch;
+
typedef struct AAPatch {
AAPatchKind kind;
u32 pos;
union {
u32 dst_reg; /* AA_PATCH_ALLOCA */
NativeLoc callee; /* AA_PATCH_TAIL */
+ AASlotPatch slot; /* AA_PATCH_SLOT */
} u;
} AAPatch;
@@ -267,6 +291,21 @@ typedef struct AANativeTarget {
* exclusive with slim_prologue (Tier A) and slim_small_frame; gated on
* out_stack==0 && !has_alloca && frame_size <= 504. */
u8 fp_at_bottom;
+
+ /* Single-pass far-slot fast path (Lever 1 / Fix B). When set, fixed-slot
+ * loads/stores whose top-record fp offset falls outside stur's ±256 range are
+ * emitted as a one-word positive scaled `ldr/str [base,#scaled]` placeholder
+ * plus an AA_PATCH_SLOT, instead of the multi-insn `sub xN,x29,#off ; ldur`
+ * address build. The base is a stable bottom anchor: sp (the common case) or
+ * AA_FRAME_BASE when the function moves sp via alloca. Decided at func_begin
+ * (off when a Windows GP home area shifts the layout); the frame-base fallback
+ * is armed lazily on the first alloca. Only 4/8-byte int/fp slots take this
+ * path — byte/half stay on the existing path (their positive scaled reach,
+ * 4 KB/8 KB, is too small to guarantee a one-word patch). */
+ u8 slot_sp_base;
+ u8 uses_frame_base; /* an alloca fired: patch AA_PATCH_SLOT against
+ * AA_FRAME_BASE and set it up in the prologue. */
+ NativeFrameSlot frame_base_slot; /* home for the saved caller AA_FRAME_BASE */
} AANativeTarget;
static AANativeTarget* aa_of(NativeTarget* t) { return (AANativeTarget*)t; }
@@ -763,6 +802,7 @@ static void aa_addr_base(AANativeTarget* a, NativeAddr addr, u32* base_out,
static u32 aa_ldst_q_uimm(int load, u32 rt, u32 rn, u32 byte_off);
static u32 aa_ldst_q_simm9(int load, u32 rt, u32 rn, i32 byte_off);
+static AAPatch* aa_patch_alloc(AANativeTarget* a); /* far-slot deferral */
static void aa_emit_mem_q(AANativeTarget* a, int load, NativeLoc reg,
NativeAddr addr) {
@@ -802,6 +842,33 @@ static void aa_emit_mem(AANativeTarget* a, int load, NativeLoc reg,
return;
}
if (native_loc_is_fp(reg) && sz < 2u) sz = 2u;
+ /* Far fixed-slot fast path (Lever 1 / Fix B). A plain 4/8-byte frame slot
+ * whose top-record fp offset is past stur's ±256 range would otherwise cost
+ * `sub xN,x29,#off (+movk) ; ldur` (2-3 insns). Instead emit a one-word
+ * positive scaled `ldr/str [sp,#0]` placeholder and defer the offset: once the
+ * frame is final, aa_apply_patches rewrites it to `[base, #frame_size-...]`,
+ * base = sp (stable) or AA_FRAME_BASE (alloca). Near slots keep their existing
+ * one-word ldur; byte/half stay on the fp path (scaled reach too small). */
+ if (a->slot_sp_base && addr.base_kind == NATIVE_ADDR_BASE_FRAME &&
+ addr.index_kind == NATIVE_ADDR_INDEX_NONE && (sz == 2u || sz == 3u)) {
+ AANativeSlot* s = aa_slot(a, addr.base.frame);
+ i32 fp_off = aa_fp_off_slot(a, s->off) + addr.offset;
+ if (fp_off < -256) {
+ u32 vbit = native_loc_is_fp(reg) ? 1u : 0u;
+ AAPatch* p = aa_patch_alloc(a);
+ p->kind = AA_PATCH_SLOT;
+ p->pos = mc_pos(mc);
+ p->u.slot.slot_off = s->off;
+ p->u.slot.extra = addr.offset;
+ p->u.slot.sz = (u8)sz;
+ p->u.slot.vbit = (u8)vbit;
+ p->u.slot.load = (u8)(load ? 1u : 0u);
+ p->u.slot.rt = (u8)rt;
+ aa_emit32(mc, load ? aa_ldr_uimm_v(sz, vbit, rt, AA_SP, 0)
+ : aa_str_uimm_v(sz, vbit, rt, AA_SP, 0));
+ return;
+ }
+ }
if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
i64 addend = addr.base.global.addend + (i64)addr.offset;
@@ -1143,6 +1210,9 @@ static void aa_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
a->slim_small_frame = 0;
a->fp_at_bottom = 0;
a->frame_size_final = 0;
+ a->slot_sp_base = 0;
+ a->uses_frame_base = 0;
+ a->frame_base_slot = NATIVE_FRAME_SLOT_NONE;
/* Windows variadic functions reserve a GP register home area at the top of
* the frame (just below the incoming stack args). The plain-pointer va_list
* then walks register-passed then stack-passed varargs as one block. Other
@@ -1183,6 +1253,10 @@ static void aa_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
-(i32)((AA_FRAME_SAVE_SIZE + a->top_home_bytes) /
8u)));
aa_emit32(mc, aa64_add_imm(1, AA_FP, AA_SP, 0, 0)); /* mov x29, sp */
+ /* Arm the far-slot positive-scaled fast path for the common top-record
+ * layout. A Windows GP home area (top_home_bytes) places incoming args/home
+ * above the saved pair and is rare; leave those on the fp-relative path. */
+ a->slot_sp_base = (a->top_home_bytes == 0u);
/* Reserve only the deferred `sub sp` (+ Windows probe), patched in
* aa_func_end. The region starts here; record it for the patch and CFI. */
region = abi_stack_probe_interval(a->base.c->abi) ? AA_NDT_SUB_WORDS
@@ -1386,9 +1460,29 @@ static void aa_words_saved_pair_addr(AANativeTarget* a, u32* words, u32 cap,
words[(*n)++] = aa64_add(1, AA_TMP1, AA_TMP1, AA_TMP0);
}
+/* Positive frame-base offset of the slot homing the caller's AA_FRAME_BASE
+ * (alloca functions only). Same positive-scaled form the far-slot patches use,
+ * so it is one word and fits any frame within dword scaled reach (32 KB). */
+static u32 aa_frame_base_save_off(AANativeTarget* a, const AAFrameLayout* L) {
+ i32 off = (i32)aa_sp_off_saved_pair(L) +
+ aa_fp_off_slot(a, aa_slot(a, a->frame_base_slot)->off);
+ if (off < 0 || ((u32)off >> 3) > 0xfffu)
+ aa_panic(a, "frame-base save slot out of positive scaled range");
+ return (u32)off;
+}
+
static void aa_words_restore_frame(AANativeTarget* a, u32* words, u32 cap,
u32* n, const AAFrameLayout* L) {
if (!L->frame_size) return;
+ /* Restore the caller's AA_FRAME_BASE before tearing the frame down. It still
+ * holds the frame base here (untouched by the body), so the slot is reachable
+ * as a positive scaled offset off itself; the load then overwrites it with
+ * the saved caller value. Alloca single-pass functions only. */
+ if (a->uses_frame_base) {
+ u32 off = aa_frame_base_save_off(a, L);
+ if (*n >= cap) aa_panic(a, "epilogue too small for frame-base restore");
+ words[(*n)++] = aa_ldr_uimm_v(3, 0, AA_FRAME_BASE, AA_FRAME_BASE, off);
+ }
if (a->slim_prologue) {
if (*n + 1u > cap) aa_panic(a, "instruction patch too small");
/* `ldp x29, x30, [sp], #16` — pop saved pair, restore sp. */
@@ -1565,6 +1659,17 @@ static u32 aa_build_ndt_sub_words(AANativeTarget* a, const AAFrameLayout* L,
aa_words_stack_probe(a, words, cap, &n, sub_bytes, interval);
}
aa_words_sub_sp_frame(a, words, cap, &n, sub_bytes);
+ /* Set up the frame-base anchor for alloca functions: sp now points at the
+ * frame base, so save the caller's AA_FRAME_BASE into its slot (positive
+ * scaled, valid before the move) and anchor AA_FRAME_BASE = sp. The body
+ * never touches AA_FRAME_BASE (callee-saved; the single-pass path allocates
+ * caller-saved only), so it stays the frame base across any alloca. */
+ if (a->uses_frame_base) {
+ u32 off = aa_frame_base_save_off(a, L);
+ if (n + 2u > cap) aa_panic(a, "prologue too small for frame-base setup");
+ words[n++] = aa_str_uimm_v(3, 0, AA_FRAME_BASE, AA_SP, off);
+ words[n++] = aa64_add_imm(1, AA_FRAME_BASE, AA_SP, 0, 0);
+ }
return n;
}
@@ -1636,6 +1741,23 @@ static void aa_apply_patches(AANativeTarget* a, const AAFrameLayout* L) {
aa_panic(a, "outgoing area too large for alloca result");
aa_patch32(a->base.obj, sec, p->pos,
aa64_add_imm(1, p->u.dst_reg, AA_SP, imm12, sh));
+ } else if (p->kind == AA_PATCH_SLOT) {
+ /* Resolve a far fixed-slot access to a positive scaled load/store off the
+ * stable bottom anchor. The slot sits at fp_off below fp (top-record);
+ * adding the bytes the prologue subtracted to reach the frame base
+ * (aa_sp_off_saved_pair) gives its offset from sp / AA_FRAME_BASE. */
+ const AASlotPatch* sl = &p->u.slot;
+ i32 fp_off = aa_fp_off_slot(a, sl->slot_off) + sl->extra;
+ i32 base_off = (i32)aa_sp_off_saved_pair(L) + fp_off;
+ u32 base = a->uses_frame_base ? AA_FRAME_BASE : AA_SP;
+ if (base_off < 0 || ((u32)base_off & ((1u << sl->sz) - 1u)) != 0u ||
+ ((u32)base_off >> sl->sz) > 0xfffu)
+ aa_panic(a, "far slot offset out of positive scaled range");
+ aa_patch32(a->base.obj, sec, p->pos,
+ sl->load ? aa_ldr_uimm_v(sl->sz, sl->vbit, sl->rt, base,
+ (u32)base_off)
+ : aa_str_uimm_v(sl->sz, sl->vbit, sl->rt, base,
+ (u32)base_off));
} else { /* AA_PATCH_TAIL */
NativeLoc callee = p->u.callee;
u32 words[AA_TAIL_WORDS];
@@ -2628,7 +2750,25 @@ static void aa_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
aa_panic(a, "outgoing area too large for alloca result");
aa_emit32(t->mc, aa64_add_imm(1, loc_reg(dst), AA_SP, imm12, sh));
} else {
- AAPatch* p = aa_patch_alloc(a);
+ AAPatch* p;
+ /* First alloca arms the frame-base anchor. The far-slot fast path can no
+ * longer key off sp (this `sub sp` floats it), so deferred AA_PATCH_SLOTs —
+ * those already emitted and any still to come — resolve against
+ * AA_FRAME_BASE instead. Reserve a static-frame home for the caller's value;
+ * the prologue saves it and sets AA_FRAME_BASE = frame base (before the body
+ * runs, so every slot access sees a valid anchor), the epilogue restores
+ * it. Only needed when slot_sp_base armed the fast path in the first place. */
+ if (a->slot_sp_base && !a->uses_frame_base) {
+ NativeFrameSlotDesc sd;
+ memset(&sd, 0, sizeof sd);
+ sd.type = builtin_id(KIT_CG_BUILTIN_I64);
+ sd.size = 8;
+ sd.align = 8;
+ sd.kind = NATIVE_FRAME_SLOT_SAVE;
+ a->frame_base_slot = t->frame_slot(t, &sd);
+ a->uses_frame_base = 1;
+ }
+ p = aa_patch_alloc(a);
p->kind = AA_PATCH_ALLOCA;
p->pos = mc_pos(t->mc);
p->u.dst_reg = loc_reg(dst);
