commit 1943e48b626acbd09b4728d0abd8a9d95140399d
parent b6c5177e9033c2f2240fe03f2093cb500b033444
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Fri, 12 Jun 2026 11:22:18 -0700
perf(cg): tcc-style -O0 prologue on aa64; caller-saved-only NDT invariant
The single-pass (-O0) NativeDirectTarget path reserved a worst-case 32-word
prologue region per function (sized for a fat frame + callee-save area +
Windows probe), patched the real ~4-word prologue in, and branched over the
rest — ~27 NOP words/function. Measured: 70,836 NOP words across sqlite3.o
(6.0% of text).
Root cause: the region had to cover a *variable* prologue because the frame
size and callee-save set are unknown until the body is emitted. But the -O0
path never actually allocates a callee-saved register (0/2633 functions in
sqlite, like tcc). So:
1. Make that an enforced invariant: nd_scratch_acquire never takes a
callee-saved register (the allocable-pass fallback skipped them; pressure is
absorbed by evicting a cached local instead), and nd_func_end asserts
callee_saved_used==0. The optimizer/known-frame path is untouched — it drives
the NativeTarget directly, never through NDT, and keeps full callee-save
freedom (69% of sqlite functions use callee-saves at -O1).
2. With no callee-saves, only the frame (+ Windows page probe) is
deferred. Restructure the aa64 prologue tcc-style: emit the frame-independent
entry live (stp x29,x30,[sp,#-(16+top_home)]! ; mov x29,sp), then reserve
only a tiny region (5 words, or 12 with a probe) for the deferred sub. The
frame is byte-identical in memory — same fp/sp, same fp-relative offsets,
same epilogue — only the instruction sequence changes; CFI advances by the
live entry plus the reserved region.
sqlite3.o: 70,836 -> 7,793 NOP words (6.0% -> 0.7%, fewer than tcc's 13,757);
text 4.73MB -> 4.47MB. Validated: test-parse 3920/0, test-toy 1392/0,
test-dwarf + debug roundtrip + CFI unit all OK.
Diffstat:
2 files changed, 91 insertions(+), 42 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -69,6 +69,17 @@ enum {
* aa_words_stack_probe). */
AA_PROLOGUE_WORDS = 32u,
AA_TAIL_WORDS = 32u,
+ /* Single-pass (-O0) tcc-style prologue. The frame-independent entry
+ * `stp x29,x30,[sp,#-(16+top_home)]! ; mov x29,sp` (AA_NDT_FIXED_ENTRY_WORDS)
+ * is emitted live in aa_func_begin; only the `sub sp` that grows the rest of
+ * the frame below the saved pair is deferred (patched once frame_size is
+ * final), plus — on stack-probing targets (Windows) — the page probe. The
+ * single-pass path uses no callee-saved registers (enforced in
+ * nd_scratch_acquire), so nothing else is deferred and the reserved region is
+ * tiny: worst-case sub = load_imm(2)+3 = 5; +probe load_imm(2)+5 = 7 → 12. */
+ AA_NDT_FIXED_ENTRY_WORDS = 2u,
+ AA_NDT_SUB_WORDS = 12u, /* region buffer: worst case (probe + sub) */
+ AA_NDT_SUB_WORDS_NOPROBE = 5u, /* reserved region when no stack probe */
};
/* Windows/AArch64 TLS Local-Exec. The TEB pointer lives in the reserved
@@ -226,6 +237,7 @@ typedef struct AANativeTarget {
u32 func_start;
u32 prologue_pos;
+ u32 prologue_region_words; /* single-pass: reserved deferred-`sub` region */
u32 minimal_prologue_words; /* opt path: exact prologue length, else 0 */
MCLabel epilogue_label;
@@ -1147,21 +1159,34 @@ static void aa_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
static void aa_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
AANativeTarget* a = aa_of(t);
MCEmitter* mc = t->mc;
+ u32 region;
aa_func_begin_common(t, fd);
- /* Reserve the worst-case prologue as one bulk emit (32 NOP words) instead of
- * 32 single-word aa_emit32 calls. Replicate the per-instruction -g line rows
- * so debug output is unchanged. */
+ /* tcc-style prologue. Emit the frame-independent entry live: save the fp/lr
+ * pair via a (16 + top_home)-byte pre-decrement and anchor fp at the pair.
+ * top_home (the Windows-variadic GP home area, 0 elsewhere) is known here, so
+ * the pair and fp land at exactly the addresses every fp-relative offset
+ * already assumes — the frame is byte-identical to the old single big-`sub`
+ * prologue, only the instruction sequence changes. */
+ aa_emit32(mc, aa64_stp64_pre(AA_FP, AA_LR, AA_SP,
+ -(i32)((AA_FRAME_SAVE_SIZE + a->top_home_bytes) /
+ 8u)));
+ aa_emit32(mc, aa64_add_imm(1, AA_FP, AA_SP, 0, 0)); /* mov x29, sp */
+ /* Reserve only the deferred `sub sp` (+ Windows probe), patched in
+ * aa_func_end. The region starts here; record it for the patch and CFI. */
+ region = abi_stack_probe_interval(a->base.c->abi) ? AA_NDT_SUB_WORDS
+ : AA_NDT_SUB_WORDS_NOPROBE;
+ a->prologue_pos = mc->pos(mc);
+ a->prologue_region_words = region;
{
- u8 nops[AA_PROLOGUE_WORDS * 4u];
- for (u32 i = 0; i < AA_PROLOGUE_WORDS; ++i)
- wr_u32_le(nops + i * 4u, 0xd503201fu);
+ u8 nops[AA_NDT_SUB_WORDS * 4u];
+ for (u32 i = 0; i < region; ++i) wr_u32_le(nops + i * 4u, 0xd503201fu);
if (mc->debug) {
u32 ofs = obj_pos(mc->obj, mc->section_id);
- mc->emit_bytes(mc, nops, sizeof nops);
- for (u32 i = 0; i < AA_PROLOGUE_WORDS; ++i)
+ mc->emit_bytes(mc, nops, region * 4u);
+ for (u32 i = 0; i < region; ++i)
debug_emit_row(mc->debug, mc->section_id, ofs + i * 4u, mc->loc);
} else {
- mc->emit_bytes(mc, nops, sizeof nops);
+ mc->emit_bytes(mc, nops, region * 4u);
}
}
aa_emit_entry_saves(a);
@@ -1501,31 +1526,45 @@ static u32 aa_build_prologue_words(AANativeTarget* a, const AAFrameLayout* L,
return n;
}
-/* Patch the reserved prologue region (`region` words at prologue_pos) with the
- * real prologue for `L`. Used by the NativeDirectTarget single-pass path,
- * which reserves AA_PROLOGUE_WORDS up front before the frame is known. The
- * optimizer path reserves exactly the words it needs, so `region` equals
- * the real prologue length and no tail remains. */
-static void aa_patch_prologue(AANativeTarget* a, const AAFrameLayout* L,
- u32 region) {
- u32 words[AA_PROLOGUE_WORDS];
+/* Build the deferred portion of the single-pass (-O0) prologue: the optional
+ * Windows page probe and the `sub sp` that grows the frame below the fp/lr pair
+ * the live entry (aa_func_begin) already saved and anchored fp at. Only
+ * `frame_size - 16 - top_home` (= aa_sp_off_saved_pair) remains to subtract. */
+static u32 aa_build_ndt_sub_words(AANativeTarget* a, const AAFrameLayout* L,
+ u32* words, u32 cap) {
+ u32 n = 0;
+ u32 sub_bytes = aa_sp_off_saved_pair(L);
+ if (!sub_bytes) return 0;
+ {
+ u32 interval = abi_stack_probe_interval(a->base.c->abi);
+ if (interval && sub_bytes > interval)
+ aa_words_stack_probe(a, words, cap, &n, sub_bytes, interval);
+ }
+ aa_words_sub_sp_frame(a, words, cap, &n, sub_bytes);
+ return n;
+}
+
+/* Patch the reserved deferred-`sub` region (prologue_region_words at
+ * prologue_pos) once the frame is final. Single-pass (NativeDirectTarget) path
+ * only — the optimizer path emits its prologue final and never patches. The
+ * fixed entry (pair save + fp anchor) was already emitted live, so this writes
+ * just the probe + `sub sp` and branches over any unused tail. */
+static void aa_patch_prologue(AANativeTarget* a, const AAFrameLayout* L) {
+ u32 words[AA_NDT_SUB_WORDS];
+ u32 region = a->prologue_region_words;
u32 n;
ObjSecId sec = a->func->text_section_id;
- if (region > AA_PROLOGUE_WORDS) aa_panic(a, "prologue region too large");
+ if (region > AA_NDT_SUB_WORDS) aa_panic(a, "prologue region too large");
memset(words, 0, sizeof words);
- n = aa_build_prologue_words(a, L, words, region);
- /* If the real prologue is shorter than the reserved region (the worst-case
- * NDT reservation), branch straight to the body rather than leaving the
- * trailing slots as NOPs that fall through and execute on every call. */
+ n = aa_build_ndt_sub_words(a, L, words, region);
+ /* If the deferred prologue is shorter than the reserved region, branch
+ * straight to the entry saves rather than executing the trailing NOPs. */
if (n < region) {
words[n] = aa64_b(region - n);
for (u32 i = n + 1u; i < region; ++i) words[i] = 0xd503201fu;
}
- /* One contiguous patch over the reserved region rather than `region`
- * separate obj_patch calls (each a chunk-directory lookup + small memcpy);
- * mirrors the x64 backend. */
{
- u8 pbytes[AA_PROLOGUE_WORDS * 4u];
+ u8 pbytes[AA_NDT_SUB_WORDS * 4u];
for (u32 i = 0; i < region; ++i) wr_u32_le(pbytes + i * 4u, words[i]);
obj_patch(a->base.obj, sec, a->prologue_pos, pbytes, (size_t)region * 4u);
}
@@ -1603,12 +1642,14 @@ static void aa_func_end(NativeTarget* t) {
a->top_home_bytes);
/* known_frame (optimizer): prologue, allocas, and tail epilogues were emitted
* final and slim eligibility was settled in aa_func_begin_known_frame — there
- * is nothing to patch. Single-pass (NDT): a worst-case prologue region was
- * reserved and the deferred patches recorded; resolve them now that the frame
- * is final. The NDT path always uses the fat prologue/epilogue (slim_* left 0
- * by aa_func_begin_common, since its reserved region is much larger). */
- u32 prologue_region =
- a->frame.known_frame ? a->minimal_prologue_words : AA_PROLOGUE_WORDS;
+ * is nothing to patch. Single-pass (NDT): the frame-independent entry was
+ * emitted live and only the deferred `sub` region + patches remain; resolve
+ * them now that the frame is final. The advance past the prologue (for CFI)
+ * is the live fixed entry plus the reserved deferred region. */
+ u32 prologue_advance_words =
+ a->frame.known_frame
+ ? a->minimal_prologue_words
+ : (AA_NDT_FIXED_ENTRY_WORDS + a->prologue_region_words);
mc->label_place(mc, a->epilogue_label);
aa_emit_callee_restores(a);
aa_emit_restore_frame(a, &L);
@@ -1619,12 +1660,12 @@ static void aa_func_end(NativeTarget* t) {
* the final prologue never saw. */
if (a->npatches != 0) aa_panic(a, "known-frame path left deferred patches");
} else {
- aa_patch_prologue(a, &L, prologue_region);
+ aa_patch_prologue(a, &L);
aa_apply_patches(a, &L);
}
if (mc->cfi_set_next_pc_offset && mc->cfi_def_cfa && mc->cfi_offset) {
i32 cfa = aa_cfa_off(a);
- mc->cfi_set_next_pc_offset(mc, prologue_region * 4u);
+ mc->cfi_set_next_pc_offset(mc, prologue_advance_words * 4u);
/* CFA = caller's sp, an fp-relative offset that depends on the layout:
* fp+16 (top-record) or fp+frame_size (bottom-record). saved fp/lr live at
* [fp]/[fp+8] in both, hence at CFA-cfa / CFA-cfa+8. */
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -163,6 +163,12 @@ static Reg nd_scratch_acquire(NativeDirectTarget* d, NativeAllocClass cls) {
for (u32 i = 0; i < nregs; ++i) {
Reg r = regs[i];
if (r >= 32u) continue;
+ /* The single-pass (-O0) path is caller-saved only: never take a
+ * callee-saved register as scratch (the `allocable` pass would otherwise
+ * offer them under pressure). Pressure is absorbed by evicting a cached
+ * local below instead — keeping callee_saved_used empty so each backend
+ * can reserve only a tiny deferred-prologue region. See nd_func_end. */
+ if (nd_callee_saved_mask(d, cls) & (1u << r)) continue;
if ((d->scratch_used[cls] & (1u << r)) == 0 &&
d->reg_owner[cls][r] == CG_LOCAL_NONE) {
d->scratch_used[cls] |= 1u << r;
@@ -938,17 +944,19 @@ static void nd_func_begin(CgTarget* t, const CGFuncDesc* fd) {
static void nd_func_end(CgTarget* t) {
NativeDirectTarget* d = nd_of(t);
NativeFramePatchState frame;
- u32 ncallee_classes = 0;
memset(&frame, 0, sizeof frame);
frame.max_outgoing = d->max_outgoing;
+ /* Invariant: the single-pass (-O0) path never allocates a callee-saved
+ * register — nd_scratch_acquire and the local cache restrict themselves to
+ * caller-saved regs (see nd_scratch_acquire / nd_cache_alloc). The prologue
+ * therefore never saves/restores any callee-save, which is exactly what lets
+ * each backend reserve only a small fixed deferred-prologue region (just the
+ * frame `sub`, plus a Windows probe) instead of a worst-case callee-save
+ * area. The optimizer/known-frame path keeps full callee-save freedom: it
+ * drives the NativeTarget directly and never routes through here. */
for (u32 cls = 0; cls < 3u; ++cls) {
- if (d->callee_saved_used[cls]) ncallee_classes = cls + 1u;
- }
- if (ncallee_classes) {
- if (!d->native || !d->native->reserve_callee_saves)
- nd_panic(d, "target cannot preserve callee-saved scratch registers");
- d->native->reserve_callee_saves(d->native, d->callee_saved_used,
- ncallee_classes);
+ if (d->callee_saved_used[cls])
+ nd_panic(d, "single-pass (-O0) path must not use callee-saved registers");
}
if (d->native && d->native->note_frame_state)
d->native->note_frame_state(d->native, &frame);
