commit b57f569929399c788ffc2078abd167689f73ba5d
parent 0cac2e5ced0729fe4cfe3359319142e8358a615e
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 16:58:23 -0700
perf(cg): -O0 aa64 byte/half far frame slots positive-scaled off sp (Lever 3)
Extend Lever 1's deferred positive-scaled far-slot addressing (4/8-byte
slots resolved to ldr/str [sp,#scaled]) to 1/2-byte accesses. A byte/half
slot past stur's ±256 reach previously cost an address build (sub xN,x29,#off
[+movk] ; ldurb/sturb, 2-4 insns); it now resolves to a single scaled
ldrb/strb/ldrh/strh [base,#scaled] when the final offset fits.
The scaled byte/half reach (imm12*size = 4 KB / 8 KB) is below the largest
sqlite frame (VdbeExec ~6.5 KB), so a one-word patch cannot always fit. A
byte/half slot therefore reserves TWO placeholder words at body time
(scaled load/store + nop). When the resolved offset exceeds the scaled imm12,
aa_apply_patches fills both with an address-build fallback
'add x17,base,#(off & ~0xfff),lsl#12 ; ldrb/strb rt,[x17,#(off & 0xfff)]',
correct for any frame < 16 MB (the high part is a 4096-multiple that fits the
shift-12 addsub imm; the low part is <4096 and, for a half, even, so both
scale-divide cleanly). 4/8-byte slots keep the one-word patch (their
16 KB/32 KB reach covers any frame).
sqlite3.c -c (arm64-macOS): .text 1,495,408 -> 1,494,640 B
(373,852 -> 373,660 insns, -768 B / -192). Histogram: sub -1213, movk -144,
movz -48, sturb -659, sturh -363, ldurb -82, ldurh -157 (all flipped to the
scaled forms + reserved nops, +1210). Net is modest: moderate-far slots are
size-neutral (sub+ldurb -> strb+nop), the win is the big-frame overflow slots
that drop the multi-insn offset materialize. Deterministic, byte-identical
across recompiles.
Gates: test-toy 1392/0, test-cg-api, test-isa, test-opt, test-aa64-inline,
test-debug, test-dwarf all green. Differential probes vs /usr/bin/cc -O0
(kit -O0 and -O1) match, including a large-frame probe exercising the byte
overflow fallback (16 occurrences, offsets to 6854) and the alloca/x28-base
scaled path.
Diffstat:
1 file changed, 61 insertions(+), 17 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -209,8 +209,9 @@ typedef enum AAPatchKind {
typedef struct AASlotPatch {
u32 slot_off; /* aa_slot(...)->off */
i32 extra; /* addr.offset added to the slot base */
- u8 sz; /* size_idx: 2 (word) or 3 (dword) */
- u8 vbit; /* SIMD/FP register */
+ u8 sz; /* size_idx: 0 (byte) / 1 (half) / 2 (word) / 3 (dword). sz<2
+ * reserves a 2nd placeholder word (overflow fallback). */
+ u8 vbit; /* SIMD/FP register (always 0 for sz<2) */
u8 load; /* 1 = ldr, 0 = str */
u8 rt; /* transfer register */
} AASlotPatch;
@@ -875,15 +876,31 @@ static void aa_emit_mem(AANativeTarget* a, int load, NativeLoc reg,
sz <= 1u)
? AA64_LDST_OPC_LDRS_X
: AA64_LDST_OPC_LDR;
- /* Far fixed-slot fast path (Lever 1 / Fix B). A plain 4/8-byte frame slot
- * whose top-record fp offset is past stur's ±256 range would otherwise cost
- * `sub xN,x29,#off (+movk) ; ldur` (2-3 insns). Instead emit a one-word
- * positive scaled `ldr/str [sp,#0]` placeholder and defer the offset: once the
- * frame is final, aa_apply_patches rewrites it to `[base, #frame_size-...]`,
- * base = sp (stable) or AA_FRAME_BASE (alloca). Near slots keep their existing
- * one-word ldur; byte/half stay on the fp path (scaled reach too small). */
+ /* Far fixed-slot fast path (Lever 1/3 / Fix B). A plain frame slot whose
+ * top-record fp offset is past stur's ±256 range would otherwise cost `sub
+ * xN,x29,#off (+movk) ; ldur` (2-4 insns). Instead emit a positive scaled
+ * `ldr/str [sp,#0]` placeholder and defer the offset: once the frame is final,
+ * aa_apply_patches rewrites it to `[base, #frame_size-...]`, base = sp (stable)
+ * or AA_FRAME_BASE (alloca).
+ *
+ * A 4/8-byte slot (sz>=2) always fits the scaled imm12 (reach 16 KB/32 KB >=
+ * any frame), so it is a one-word patch. A 1/2-byte slot (sz<2, Lever 3) has a
+ * smaller scaled reach (4 KB/8 KB) that the largest frames (sqlite VdbeExec
+ * ~6.5 KB) exceed, so it reserves TWO words: when the resolved offset fits it
+ * is `ldrb/strb [base,#scaled] ; nop`, and when it overflows the resolver
+ * emits the address-build fallback `add x17,base,#hi ; ldrb/strb [x17,#lo]`
+ * (correct for any frame). byte/half are always integer (vbit=0; FP was
+ * widened to sz>=2 above).
+ *
+ * Lever 3/4 interaction: a far SIGNED narrow load (ld_opc==LDRS_X) is excluded
+ * from this scaled-slot path — the slot patch records only a plain LDR opcode,
+ * so it would zero-extend while the cg layer dropped the CV_SEXT. Such loads
+ * fall through to the general path below, which honors ld_opc (ldrsb/ldrsh with
+ * an address build). Rare (a far signed char/short slot); correctness over the
+ * one-insn win. */
+ int sext_far = load && ld_opc == AA64_LDST_OPC_LDRS_X;
if (a->slot_sp_base && addr.base_kind == NATIVE_ADDR_BASE_FRAME &&
- addr.index_kind == NATIVE_ADDR_INDEX_NONE && (sz == 2u || sz == 3u)) {
+ addr.index_kind == NATIVE_ADDR_INDEX_NONE && sz <= 3u && !sext_far) {
AANativeSlot* s = aa_slot(a, addr.base.frame);
i32 fp_off = aa_fp_off_slot(a, s->off) + addr.offset;
if (fp_off < -256) {
@@ -899,6 +916,8 @@ static void aa_emit_mem(AANativeTarget* a, int load, NativeLoc reg,
p->u.slot.rt = (u8)rt;
aa_emit32(mc, load ? aa_ldr_uimm_v(sz, vbit, rt, AA_SP, 0)
: aa_str_uimm_v(sz, vbit, rt, AA_SP, 0));
+ /* Reserve the fallback's second word for byte/half (filled at resolve). */
+ if (sz < 2u) aa_emit32(mc, 0xd503201fu /* nop */);
return;
}
}
@@ -1786,14 +1805,39 @@ static void aa_apply_patches(AANativeTarget* a, const AAFrameLayout* L) {
i32 fp_off = aa_fp_off_slot(a, sl->slot_off) + sl->extra;
i32 base_off = (i32)aa_sp_off_saved_pair(L) + fp_off;
u32 base = a->uses_frame_base ? AA_FRAME_BASE : AA_SP;
- if (base_off < 0 || ((u32)base_off & ((1u << sl->sz) - 1u)) != 0u ||
- ((u32)base_off >> sl->sz) > 0xfffu)
+ u32 ubo = (u32)base_off;
+ /* Natural alignment is guaranteed by the slot allocator (and the byte case
+ * is unconditionally aligned); a misalignment would mean a layout bug. */
+ if (base_off < 0 || (ubo & ((1u << sl->sz) - 1u)) != 0u)
+ aa_panic(a, "far slot offset negative or misaligned");
+ if ((ubo >> sl->sz) <= 0xfffu) {
+ /* Scaled immediate fits: one-word ldr/str. For byte/half the reserved
+ * second placeholder word stays the nop emitted at body time. */
+ aa_patch32(a->base.obj, sec, p->pos,
+ sl->load ? aa_ldr_uimm_v(sl->sz, sl->vbit, sl->rt, base, ubo)
+ : aa_str_uimm_v(sl->sz, sl->vbit, sl->rt, base, ubo));
+ } else if (sl->sz < 2u) {
+ /* Lever 3 overflow fallback. byte/half scaled reach (4 KB/8 KB) is below
+ * the largest frames, so build the address into the reserved second word
+ * instead: `add x17,base,#(off & ~0xfff) ; ldrb/strb rt,[x17,#(off &
+ * 0xfff)]`. The high part is a multiple of 4096 (fits the shift-12
+ * addsub imm for any frame < 16 MB); the low part is <4096 and, for a
+ * half, even (base_off is 2-aligned), so both scale-divide cleanly. */
+ u32 hi = ubo & ~0xfffu;
+ u32 lo = ubo & 0xfffu;
+ if ((hi >> 12) > 0xfffu)
+ aa_panic(a, "far slot offset out of address-build range");
+ aa_patch32(a->base.obj, sec, p->pos,
+ aa64_add_imm(1, AA_TMP1, base, hi >> 12, 1));
+ aa_patch32(a->base.obj, sec, p->pos + 4u,
+ sl->load
+ ? aa_ldr_uimm_v(sl->sz, sl->vbit, sl->rt, AA_TMP1, lo)
+ : aa_str_uimm_v(sl->sz, sl->vbit, sl->rt, AA_TMP1, lo));
+ } else {
+ /* 4/8-byte scaled reach (16 KB/32 KB) covers any frame, so this is
+ * unreachable; a one-word patch has no fallback room. */
aa_panic(a, "far slot offset out of positive scaled range");
- aa_patch32(a->base.obj, sec, p->pos,
- sl->load ? aa_ldr_uimm_v(sl->sz, sl->vbit, sl->rt, base,
- (u32)base_off)
- : aa_str_uimm_v(sl->sz, sl->vbit, sl->rt, base,
- (u32)base_off));
+ }
} else { /* AA_PATCH_TAIL */
NativeLoc callee = p->u.callee;
u32 words[AA_TAIL_WORDS];
