commit 9a99a0aae81d593f98884d5b2a8a2b94b3e1aecf
parent a69929714a36eaaad30511e1582b14c81e63cab1
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Mon, 15 Jun 2026 23:43:10 -0700
aa64: uniform x29-at-bottom known-frame layout at O1 (O1.md W1.1)
Diffstat:
5 files changed, 427 insertions(+), 133 deletions(-)
diff --git a/mk/test.mk b/mk/test.mk
@@ -859,7 +859,7 @@ test-macho: lib $(TEST_RT_DEP) $(ROUNDTRIP_BIN_MACHO) $(LINK_EXE_RUNNER) $(JIT_R
OPT_TEST_BIN = build/test/cg_ir_lower_test
TINY_INLINE_TEST_BIN = build/test/tiny_inline_test
-test-opt: bin $(OPT_TEST_BIN) test-opt-tiny-inline test-opt-inline test-opt-zero-arg test-opt-static-prune-aa64 test-opt-aa64-tail test-opt-x64-win-tail-sret test-opt-prologue-tier test-opt-whole-program-inline test-opt-lto-phase1 test-opt-redundant-copy-ext test-opt-redundant-frame-sub test-opt-o1-branch-cleanup test-opt-hot-slot-order test-opt-o1-remat
+test-opt: bin $(OPT_TEST_BIN) test-opt-tiny-inline test-opt-inline test-opt-zero-arg test-opt-static-prune-aa64 test-opt-aa64-tail test-opt-x64-win-tail-sret test-opt-prologue-tier test-opt-whole-program-inline test-opt-lto-phase1 test-opt-redundant-copy-ext test-opt-redundant-frame-sub test-opt-o1-branch-cleanup test-opt-hot-slot-order test-opt-o1-remat test-opt-aa64-x29-bottom
$(OPT_TEST_BIN)
@@ -893,6 +893,12 @@ test-opt-hot-slot-order: bin
test-opt-o1-remat: bin
@KIT=$(abspath $(BIN)) bash test/opt/o1_remat.sh
+# Structural disasm check: aa64 x29-at-bottom known-frame layout addresses every
+# spill slot as positive ldr/str [x29,#k] (or add-build) -- no sub xN,x29,# at O1.
+.PHONY: test-opt-aa64-x29-bottom
+test-opt-aa64-x29-bottom: bin
+ @KIT=$(abspath $(BIN)) bash test/opt/aa64_x29_bottom.sh
+
test-opt-tiny-inline: bin $(TINY_INLINE_TEST_BIN)
$(TINY_INLINE_TEST_BIN)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -74,8 +74,13 @@ enum {
* place; the optimizer path reserves exactly what it emits). Sized to hold
* the fat prologue plus the Windows large-frame stack probe (≤7 words, see
* aa_words_stack_probe). */
- AA_PROLOGUE_WORDS = 32u,
- AA_TAIL_WORDS = 32u,
+ /* Worst-case prologue/epilogue word counts. The W1.1 bottom-record general
+ * path can stack: a Windows page probe (≤7), the robust `sub sp` (≤5), the
+ * far saved-pair address build + stp (≤3), the x29 anchor (≤3), a callee-save
+ * base build for huge frames (≤4), and up to 18 callee-save store/restores
+ * (~5 int stp pairs + an int single + 8 fp singles ≈ 14 insns) — round up. */
+ AA_PROLOGUE_WORDS = 48u,
+ AA_TAIL_WORDS = 48u,
/* Single-pass (-O0) tcc-style prologue. The frame-independent entry
* `stp x29,x30,[sp,#-(16+top_home)]! ; mov x29,sp` (AA_NDT_FIXED_ENTRY_WORDS)
* is emitted live in aa_func_begin; only the `sub sp` that grows the rest of
@@ -125,27 +130,40 @@ enum {
* sp --> +------------------------------+
* low addr CFA = fp + 16
*
- * BOTTOM-RECORD (fp_at_bottom — known-frame -O1 small frames with
- * callee-saves/locals and out_stack==0). The record moves to the bottom so the
- * sp adjustment folds into a pre/post-indexed stp/ldp (−2 insns/call). fp = sp;
- * slots/callee-saves stack ABOVE the record at positive offsets. Offsets depend
- * on frame_size (hence known-frame only, where the frame is final before body).
+ * BOTTOM-RECORD (fp_at_bottom — the W1.1 uniform x29-at-bottom layout: every
+ * known frame on the -O1 path EXCEPT slim Tier A and the Windows-variadic
+ * top_home case). The frame record + slots stack ABOVE x29, anchored just above
+ * the outgoing-arg area: x29 = sp + out_stack. Every slot is then a positive
+ * `ldr/str [x29,#k]` — one instruction within the 32 KB scaled reach (add-build
+ * `add x16,x29,#hi ; ldr [x16,#lo]` only past it), replacing the top-record
+ * `sub x17,x29,#k ; ldur` fallback. x29 is the frame pointer (reserved,
+ * alloca-stable, set once) so alloca falls out for free and outgoing args stay
+ * sp-relative (calls after an alloca still address [sp,#k]). Offsets depend on
+ * frame_size (hence known-frame only, where the frame is final before the body).
+ * Let N = frame_size, os = out_stack:
*
* high addr caller's stack frame
* +------------------------------+
- * | incoming stack args | aa_fp_off_in_arg(a,i) = N+i
- * +------------------------------+ <- caller's sp = CFA = fp +
- * N | frame slots (+ align pad) | aa_fp_off_slot(a,off)=N-off |
- * (callee-saves + locals …) | (in [16, N), above record)
+ * | incoming stack args | aa_fp_off_in_arg(a,i)=(N-os)+i
+ * +------------------------------+ <- caller's sp = CFA = x29+(N-os)
+ * | frame slots (+ align pad) | aa_fp_off_slot(a,off)=(N-os)-off
+ * | (callee-saves + locals …) | (in [16, N-os), above record)
* +------------------------------+
- * fp = sp --> | saved x29 (prev fp) | aa_fp_off_saved_fp() = 0
+ * x29 = sp+os->| saved x29 (prev fp) | aa_fp_off_saved_fp() = 0
* | saved x30 (prev lr) | aa_fp_off_saved_lr() = 8
- * low addr +------------------------------+ (N = frame_size;
- * out_stack==0)
+ * +------------------------------+
+ * | outgoing args (os) | aa_sp_off_out_arg(i) ([sp,#i])
+ * sp --> +------------------------------+
+ * low addr
*
* frame_size (N) = align16(AA_FRAME_SAVE_SIZE + slot_bytes + out_stack).
- * Tail calls write outgoing args into the caller's incoming-args window —
- * physically the same address, expressed via aa_fp_off_tail_out_arg.
+ * The saved pair sits AT x29 so [x29]=caller fp, [x29+8]=caller lr — the
+ * frame-pointer chain kit's unwinder / __kit_backtrace walks (uniform
+ * fp[0]/fp[1]); non-negotiable. When os==0 this reduces to fp = sp and the
+ * earlier folded `stp [sp,#-N]! ; mov x29,sp` encoding (the original
+ * fp_at_bottom fast path). Tail calls write outgoing args into the caller's
+ * incoming-args window — physically the same address, expressed via
+ * aa_fp_off_tail_out_arg.
* ========================================================================== */
typedef struct AAFrameLayout {
@@ -243,6 +261,13 @@ typedef struct AANativeTarget {
* layout (where slot/incoming-arg offsets depend on frame_size); meaningless
* and unread on the single-pass path, which never sets fp_at_bottom. */
u32 frame_size_final;
+ /* Final outgoing-arg area (= max_outgoing), set with frame_size_final in
+ * aa_func_begin_known_frame. In the bottom-record layout x29 is anchored
+ * `out_stack` bytes above sp, so every fp-relative offset is measured from
+ * `frame_size - out_stack` (the bytes above the anchor). Zero on the
+ * single-pass path (top-record) and on os==0 known frames (where the formula
+ * reduces to the original fp_at_bottom one). */
+ u32 out_stack_final;
u32 incoming_stack_size;
/* Windows-variadic GP register home area size (gp_reg_count * gp_slot_size,
* 64 today; 0 on every other ABI). When nonzero the function takes the fat
@@ -274,25 +299,18 @@ typedef struct AANativeTarget {
* emit a 2-insn `stp x29,x30,[sp,#-16]! ; mov x29,sp` and matching `ldp
* x29,x30,[sp],#16 ; ret` instead of the fat 4+3-insn FP-frame form. */
u8 slim_prologue;
- /* Set at func_end when frame_size - 16 fits stp's signed 7-bit scaled
- * immediate (frame_size <= 520). Skips the `add x17, sp, #(N-16)` scratch
- * materialization in the prologue (stp x29,x30,[sp,#N-16] instead) and
- * the matching `add x10, fp, #0` in the epilogue (ldp x29,x30,[sp,#N-16]
- * + add sp,sp,#N). Mutually exclusive with `slim_prologue` (Tier A wins
- * when both would apply) and `fp_at_bottom` (which wins for out_stack==0).
- * Now only reached for small frames with outgoing stack args (out_stack>0),
- * where the record cannot move to the bottom. Keeps the top-record layout. */
- u8 slim_small_frame;
- /* Set by aa_func_begin_known_frame for a small frame with callee-saves/locals
- * and no outgoing stack args: the frame record moves to the bottom of the
- * frame (fp = sp, `mov x29,sp`) so the sp adjustment folds into a pre-indexed
- * `stp x29,x30,[sp,#-N]!` entry and post-indexed `ldp x29,x30,[sp],#N` exit
- * (−2 insns/call vs slim_small_frame). Slots and callee-saves stack ABOVE the
- * record at positive fp offsets; incoming args sit at fp+frame_size; CFA =
- * fp+frame_size. The frame-size-dependent offsets are the reason this is only
- * available on the known-frame path (frame final before the body). Mutually
- * exclusive with slim_prologue (Tier A) and slim_small_frame; gated on
- * out_stack==0 && !has_alloca && frame_size <= 504. */
+ /* Set by aa_func_begin_known_frame for the W1.1 uniform x29-at-bottom layout:
+ * every known frame except slim Tier A and the Windows-variadic top_home
+ * case. x29 is anchored out_stack bytes above sp with the saved pair AT x29;
+ * every slot is a positive `ldr/str [x29,#k]`, slots/callee-saves stack ABOVE
+ * the record, incoming args at x29 + (frame_size - out_stack) = CFA. Covers
+ * out_stack>0, alloca (x29 is stable across the floated sp), and arbitrarily
+ * large frames. Two prologue/epilogue encodings: a folded `stp [sp,#-N]!`
+ * fast path when out_stack==0 && frame_size<=504 && !alloca, and the general
+ * `sub sp ; stp [sp,#os] ; add x29,sp,#os` / `mov x16,x29 ; ldp [x16] ;
+ * add sp,x16,#(N-os)` form otherwise. The frame-size-dependent offsets are
+ * the reason this is only available on the known-frame path (frame final
+ * before the body). Mutually exclusive with slim_prologue (Tier A). */
u8 fp_at_bottom;
/* Single-pass far-slot fast path (Lever 1 / Fix B). When set, fixed-slot
@@ -318,20 +336,24 @@ static AANativeTarget* aa_of(NativeTarget* t) { return (AANativeTarget*)t; }
*
* top-record (default): record near the top, fp anchored at the saved pair.
* incoming args at fp+16+b, slots below fp at -off. CFA = fp+16.
- * bottom-record (fp_at_bottom): record at the bottom, fp = sp.
- * incoming args at fp+frame_size+b, slots above the record at
- * frame_size-off (in [16, frame_size), never overlapping the 16-byte
- * record since frame_size = align16(16+cum_off) >= 16+cum_off).
- * CFA = fp+frame_size. */
+ * bottom-record (fp_at_bottom): x29 = sp + out_stack, the saved pair AT x29,
+ * slots above the record at (frame_size - out_stack) - off, incoming args
+ * at x29 + (frame_size - out_stack). CFA = x29 + (frame_size - out_stack).
+ * The "bytes above the anchor" is bsz = frame_size - out_stack = align16(16
+ * + slot_bytes) padding aside, so slots land in [16, bsz) and never overlap
+ * the 16-byte record. When out_stack==0, bsz = frame_size (fp = sp). */
+static inline u32 aa_fp_bottom_above(const AANativeTarget* a) {
+ return a->frame_size_final - a->out_stack_final;
+}
static inline i32 aa_fp_off_in_arg(const AANativeTarget* a, u32 byte_off) {
/* top-record incoming args sit above the saved pair and the (usually empty)
* Windows-variadic GP home area; bottom-record never carries a home area. */
- u32 base = a->fp_at_bottom ? a->frame_size_final
+ u32 base = a->fp_at_bottom ? aa_fp_bottom_above(a)
: AA_FRAME_SAVE_SIZE + a->top_home_bytes;
return (i32)(base + byte_off);
}
static inline i32 aa_fp_off_slot(const AANativeTarget* a, u32 slot_off) {
- return a->fp_at_bottom ? (i32)a->frame_size_final - (i32)slot_off
+ return a->fp_at_bottom ? (i32)aa_fp_bottom_above(a) - (i32)slot_off
: -(i32)slot_off;
}
/* Outgoing stack args on a tail call land in the caller's incoming-arg window —
@@ -342,10 +364,10 @@ static inline i32 aa_fp_off_tail_out_arg(const AANativeTarget* a,
return aa_fp_off_in_arg(a, byte_off);
}
/* CFA = caller's sp, expressed as an fp-relative offset (fp+16 top-record,
- * fp+frame_size bottom-record). Named so the CFI emit site stays layout-blind.
- */
+ * x29 + (frame_size - out_stack) bottom-record). Named so the CFI emit site
+ * stays layout-blind. */
static inline i32 aa_cfa_off(const AANativeTarget* a) {
- return a->fp_at_bottom ? (i32)a->frame_size_final
+ return a->fp_at_bottom ? (i32)aa_fp_bottom_above(a)
: (i32)(AA_FRAME_SAVE_SIZE + a->top_home_bytes);
}
@@ -1299,9 +1321,9 @@ static void aa_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
a->npatches = 0;
a->nalloca = 0;
a->slim_prologue = 0;
- a->slim_small_frame = 0;
a->fp_at_bottom = 0;
a->frame_size_final = 0;
+ a->out_stack_final = 0;
a->slot_sp_base = 0;
a->uses_frame_base = 0;
a->frame_base_slot = NATIVE_FRAME_SLOT_NONE;
@@ -1582,27 +1604,43 @@ static void aa_words_restore_frame(AANativeTarget* a, u32* words, u32 cap,
return;
}
if (a->fp_at_bottom) {
- /* Bottom-record fold: `ldp x29,x30,[sp],#N` reloads the pair from the
- * bottom AND releases the whole frame in one insn. Callee-saves were
- * already restored by aa_emit_callee_restores. -1 insn vs slim_small_frame
- * (which needs a separate `add sp`). N <= 504 holds the post-index imm. */
- if (*n + 1u > cap) aa_panic(a, "instruction patch too small");
- words[(*n)++] =
- aa64_ldp64_post(AA_FP, AA_LR, AA_SP, (i32)(L->frame_size / 8u));
- return;
- }
- if (a->slim_small_frame) {
- /* `ldp x29,x30,[sp,#saved_pair] ; add sp,sp,#frame_size` — load through
- * sp avoids the fat path's `add x10, fp, #0` scratch, and the subsequent
- * `add sp` unwinds without depending on the (now-clobbered) old fp. */
- u32 save_off = aa_sp_off_saved_pair(L);
- u32 imm12, sh;
- if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
- words[(*n)++] = aa64_ldp64_soff(AA_FP, AA_LR, AA_SP, (i32)(save_off / 8u));
- if (!aa64_addsub_imm_fits(L->frame_size, &imm12, &sh))
- aa_panic(a, "slim_small_frame: frame_size out of addsub imm range");
- words[(*n)++] = aa64_add_imm(1, AA_SP, AA_SP, imm12, sh);
- return;
+ u32 os = L->out_stack;
+ if (os == 0u && !a->frame.has_alloca && L->frame_size <= 504u) {
+ /* Folded fast path (matches the os==0 prologue fold): the saved pair sits
+ * at the very bottom (= sp, since no alloca floated it), so
+ * `ldp x29,x30,[sp],#N` reloads the pair AND releases the whole frame in
+ * one insn. Callee-saves were already restored by aa_emit_callee_restores.
+ * N <= 504 holds the post-index imm. */
+ if (*n + 1u > cap) aa_panic(a, "instruction patch too small");
+ words[(*n)++] =
+ aa64_ldp64_post(AA_FP, AA_LR, AA_SP, (i32)(L->frame_size / 8u));
+ return;
+ }
+ /* General bottom-record teardown, correct with OR without alloca (the
+ * post-indexed `ldp [sp],#N` can't be used once alloca floated sp): recover
+ * sp from the stable anchor x29, not from sp. `mov x16,x29 ; ldp x29,x30,
+ * [x16] ; add sp,x16,#(fs-os)` — x16 holds the anchor across the pair reload
+ * so the final add restores sp = x29 + (fs-os) = caller's sp = CFA. */
+ {
+ u32 bsz = L->frame_size - os; /* = aa_cfa_off in this layout */
+ u32 imm12, sh;
+ if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
+ words[(*n)++] = aa64_add_imm(1, AA_TMP0, AA_FP, 0, 0); /* mov x16, x29 */
+ words[(*n)++] = aa64_ldp64_soff(AA_FP, AA_LR, AA_TMP0, 0);
+ if (aa64_addsub_imm_fits(bsz, &imm12, &sh)) {
+ if (*n >= cap) aa_panic(a, "instruction patch too small");
+ words[(*n)++] = aa64_add_imm(1, AA_SP, AA_TMP0, imm12, sh);
+ } else {
+ /* bsz exceeds the addsub-imm window. The 3-register add cannot target
+ * SP (Rd=31 means XZR there), so sum into x16 (a normal register) then
+ * copy to sp via the immediate `add sp, x16, #0` form (Rd=31 == SP). */
+ aa_words_load_imm(a, words, cap, n, AA_TMP1, bsz);
+ if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
+ words[(*n)++] = aa64_add(1, AA_TMP0, AA_TMP0, AA_TMP1);
+ words[(*n)++] = aa64_add_imm(1, AA_SP, AA_TMP0, 0, 0);
+ }
+ return;
+ }
}
if (aa_cfa_off(a) == AA_FRAME_SAVE_SIZE) {
/* Common top-record case (CFA == 16: no Windows-variadic GP home area). The
@@ -1631,16 +1669,48 @@ static void aa_words_restore_frame(AANativeTarget* a, u32* words, u32 cap,
* reserve_callee_saves allocates consecutive 8-byte slots in order, so
* callee_saves[i] sits 8 bytes above callee_saves[i+1]; for an int pair the
* lower-addressed reg[i+1] is the stp's Rt and reg[i] is Rt2. FP registers (and
- * an unpaired trailing int) use the single-register stur/ldur form. */
+ * an unpaired trailing int) use the single-register stur/ldur form.
+ *
+ * Layout reach. Top-record callee-saves sit nearest fp at small NEGATIVE offsets
+ * (reserved first), so they always fit stur/stp directly. Bottom-record (W1.1)
+ * anchors x29 at the bottom, so callee-saves stack near the TOP at large
+ * POSITIVE offsets — past stp's +504 reach in any frame above ~512 bytes (and
+ * past the scaled str reach of 32 KB in a huge frame). When the highest
+ * callee-save offset exceeds stp's reach we materialize a base register
+ * `x16 = x29 + adj` once (adj 16-aligned so it stays addsub-encodable) and
+ * address the whole callee-save block relative to it at small offsets — x16 is
+ * dead here on both the prologue (after the x29 anchor) and epilogue (callee
+ * restores precede the teardown that reuses x16) paths. */
static void aa_words_callee_saves(AANativeTarget* a, int save, u32* words,
u32 cap, u32* n) {
+ u32 base = AA_FP;
+ i32 adj = 0;
+ if (a->fp_at_bottom && a->frame.ncallee_saves) {
+ /* The first reserved callee-save (smallest slot.off) gets the largest fp
+ * offset; that bound decides whether a base register is needed. */
+ i32 hi = aa_fp_off_slot(a, aa_slot(a, a->frame.callee_saves[0].slot)->off);
+ if (hi > 504) {
+ u32 imm12, sh;
+ adj = (i32)((u32)(hi - 504) & ~15u); /* 16-aligned, leaves hi-adj <= 504 */
+ if (!aa64_addsub_imm_fits((u32)adj, &imm12, &sh)) {
+ /* adj is a multiple of 4096 worth of frame; build it via x16 load_imm. */
+ aa_words_load_imm(a, words, cap, n, AA_TMP0, adj);
+ if (*n + 1u > cap) aa_panic(a, "prologue too large");
+ words[(*n)++] = aa64_add(1, AA_TMP0, AA_FP, AA_TMP0);
+ } else {
+ if (*n >= cap) aa_panic(a, "prologue too large");
+ words[(*n)++] = aa64_add_imm(1, AA_TMP0, AA_FP, imm12, sh);
+ }
+ base = AA_TMP0;
+ }
+ }
for (u32 i = 0; i < a->frame.ncallee_saves;) {
const AACalleeSave* cs = &a->frame.callee_saves[i];
- i32 off = aa_fp_off_slot(a, aa_slot(a, cs->slot)->off);
+ i32 off = aa_fp_off_slot(a, aa_slot(a, cs->slot)->off) - adj;
if (i + 1u < a->frame.ncallee_saves && cs->cls == (u8)NATIVE_REG_INT &&
a->frame.callee_saves[i + 1u].cls == (u8)NATIVE_REG_INT) {
const AACalleeSave* cs2 = &a->frame.callee_saves[i + 1u];
- i32 off2 = aa_fp_off_slot(a, aa_slot(a, cs2->slot)->off);
+ i32 off2 = aa_fp_off_slot(a, aa_slot(a, cs2->slot)->off) - adj;
/* cs2 is reserved after cs (larger slot.off), so it is the lower address
* in both layouts (off2 = off - 8): stp's Rt = cs2, Rt2 = cs, base off2.
* stp/ldp's signed-7-bit scaled immediate reaches ±504. */
@@ -1648,25 +1718,25 @@ static void aa_words_callee_saves(AANativeTarget* a, int save, u32* words,
aa_panic(a, "callee-save pair offset out of prologue range");
if (*n >= cap) aa_panic(a, "prologue too large");
words[(*n)++] = save
- ? aa64_stp64_soff(cs2->reg, cs->reg, AA_FP, off2 / 8)
- : aa64_ldp64_soff(cs2->reg, cs->reg, AA_FP, off2 / 8);
+ ? aa64_stp64_soff(cs2->reg, cs->reg, base, off2 / 8)
+ : aa64_ldp64_soff(cs2->reg, cs->reg, base, off2 / 8);
i += 2u;
} else {
u32 v = cs->cls == (u8)NATIVE_REG_FP ? 1u : 0u;
if (*n >= cap) aa_panic(a, "prologue too large");
if (a->fp_at_bottom) {
- /* Positive, 8-aligned offset above the record (up to frame_size-8 ≤
- * 496): the unscaled stur (±256) can't reach it, so use the scaled
- * unsigned-imm str/ldr. */
+ /* Positive, 8-aligned offset (after the optional base adjustment): the
+ * unscaled stur (±256) can't reach it, so use the scaled unsigned-imm
+ * str/ldr (reach 32 KB; adj keeps offsets in range for huge frames). */
if (off < 0 || (u32)off > 0x7ff8u)
aa_panic(a, "callee-save offset out of prologue range");
- words[(*n)++] = save ? aa_str_uimm_v(3, v, cs->reg, AA_FP, (u32)off)
- : aa_ldr_uimm_v(3, v, cs->reg, AA_FP, (u32)off);
+ words[(*n)++] = save ? aa_str_uimm_v(3, v, cs->reg, base, (u32)off)
+ : aa_ldr_uimm_v(3, v, cs->reg, base, (u32)off);
} else {
if (off < -256 || off > 255)
aa_panic(a, "callee-save offset out of prologue range");
- words[(*n)++] = save ? aa_stur_v(3, v, cs->reg, AA_FP, off)
- : aa_ldur_v(3, v, cs->reg, AA_FP, off);
+ words[(*n)++] = save ? aa_stur_v(3, v, cs->reg, base, off)
+ : aa_ldur_v(3, v, cs->reg, base, off);
}
i += 1u;
}
@@ -1696,14 +1766,64 @@ static u32 aa_build_prologue_words(AANativeTarget* a, const AAFrameLayout* L,
return n;
}
if (a->fp_at_bottom) {
- /* Bottom-record fold: `stp x29,x30,[sp,#-N]!` decrements sp by the whole
- * frame AND saves the pair at the new bottom in one insn; `mov x29,sp`
- * (add #0) anchors fp there. Callee-saves then stack above the record at
- * positive offsets. -2 insns/call vs the top-record slim_small_frame. */
- if (n + 2u > cap) aa_panic(a, "prologue too large");
- words[n++] =
- aa64_stp64_pre(AA_FP, AA_LR, AA_SP, -(i32)(L->frame_size / 8u));
- words[n++] = aa64_add_imm(1, AA_FP, AA_SP, 0, 0);
+ /* W1.1 uniform x29-at-bottom: anchor x29 = sp + out_stack, saved pair AT
+ * x29, slots above it. Two encodings: */
+ u32 os = L->out_stack;
+ u32 imm12, sh;
+ if (os == 0u && L->frame_size <= 504u) {
+ /* Folded fast path (the original fp_at_bottom): the saved pair is at the
+ * very bottom, so `stp x29,x30,[sp,#-N]!` decrements sp by the whole frame
+ * AND saves the pair in one insn; `mov x29,sp` anchors fp there. The
+ * post-index ldp on exit needs N <= 504. */
+ if (n + 2u > cap) aa_panic(a, "prologue too large");
+ words[n++] =
+ aa64_stp64_pre(AA_FP, AA_LR, AA_SP, -(i32)(L->frame_size / 8u));
+ words[n++] = aa64_add_imm(1, AA_FP, AA_SP, 0, 0);
+ aa_words_callee_saves(a, 1, words, cap, &n);
+ return n;
+ }
+ /* General bottom-record. `sub sp,sp,#fs` (robust / Windows-probed); save the
+ * pair at [sp,#os] and anchor x29 = sp + os; callee-saves at positive x29
+ * offsets. Outgoing args occupy [sp, sp+os); the body addresses them
+ * sp-relative, so anchoring x29 above them keeps every slot positive while
+ * leaving the arg area reachable from the (possibly alloca-floated) sp. */
+ {
+ u32 interval = abi_stack_probe_interval(a->base.c->abi);
+ if (interval && L->frame_size > interval)
+ aa_words_stack_probe(a, words, cap, &n, L->frame_size, interval);
+ }
+ aa_words_sub_sp_frame(a, words, cap, &n, L->frame_size);
+ if (os <= 504u) {
+ /* `stp x29,x30,[sp,#os]` — os fits stp's signed-7-bit scaled imm. */
+ if (n >= cap) aa_panic(a, "prologue too large");
+ words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_SP, (i32)(os / 8u));
+ } else {
+ /* Far outgoing area: build the saved-pair address (sp + os) into x17. The
+ * 3-register add cannot read SP (Rn=31 means XZR), so copy sp into x17 via
+ * the immediate `add x17, sp, #0` form first, then add os. */
+ if (aa64_addsub_imm_fits(os, &imm12, &sh)) {
+ if (n >= cap) aa_panic(a, "prologue too large");
+ words[n++] = aa64_add_imm(1, AA_TMP1, AA_SP, imm12, sh);
+ } else {
+ aa_words_load_imm(a, words, cap, &n, AA_TMP0, os);
+ if (n + 2u > cap) aa_panic(a, "prologue too large");
+ words[n++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0);
+ words[n++] = aa64_add(1, AA_TMP1, AA_TMP1, AA_TMP0);
+ }
+ if (n >= cap) aa_panic(a, "prologue too large");
+ words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_TMP1, 0);
+ }
+ /* x29 = sp + os (the anchor). Same SP-as-Rn restriction: build via immediate
+ * copy + register add for a far os. */
+ if (aa64_addsub_imm_fits(os, &imm12, &sh)) {
+ if (n >= cap) aa_panic(a, "prologue too large");
+ words[n++] = aa64_add_imm(1, AA_FP, AA_SP, imm12, sh);
+ } else {
+ aa_words_load_imm(a, words, cap, &n, AA_TMP0, os);
+ if (n + 2u > cap) aa_panic(a, "prologue too large");
+ words[n++] = aa64_add_imm(1, AA_FP, AA_SP, 0, 0);
+ words[n++] = aa64_add(1, AA_FP, AA_FP, AA_TMP0);
+ }
aa_words_callee_saves(a, 1, words, cap, &n);
return n;
}
@@ -1717,18 +1837,13 @@ static u32 aa_build_prologue_words(AANativeTarget* a, const AAFrameLayout* L,
aa_words_stack_probe(a, words, cap, &n, L->frame_size, interval);
}
aa_words_sub_sp_frame(a, words, cap, &n, L->frame_size);
- if (a->slim_small_frame) {
- /* `stp x29, x30, [sp, #saved_pair_off]` — skip the `add x17, sp, #...`
- * scratch the fat path needs. Valid when the offset fits stp's
- * signed-7-bit scaled immediate (saved_pair_off <= 504). */
- u32 save_off = aa_sp_off_saved_pair(L);
- if (n >= cap) aa_panic(a, "prologue too large");
- words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_SP, (i32)(save_off / 8u));
- } else {
- aa_words_saved_pair_addr(a, words, cap, &n, L);
- if (n >= cap) aa_panic(a, "prologue too large");
- words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_TMP1, 0); /* fp,lr @ [x17] */
- }
+ /* Fat top-record prologue: build the saved-pair address (sp + saved_pair_off)
+ * into x17, store the pair there, then anchor fp. Reached only by the
+ * single-pass (-O0) path and the Windows-variadic top_home case — the known-
+ * frame -O1 path takes slim_prologue or the fp_at_bottom layout above. */
+ aa_words_saved_pair_addr(a, words, cap, &n, L);
+ if (n >= cap) aa_panic(a, "prologue too large");
+ words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_TMP1, 0); /* fp,lr @ [x17] */
aa_words_frame_ptr_from_sp(a, words, cap, &n, L);
/* Save callee-saved registers the allocator used (fp-relative; their slots
* were reserved first by aa_reserve_callee_saves so offsets fit stur). */
@@ -2063,31 +2178,27 @@ static void aa_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
}
/* Frame is final: slot_bytes (cum_off) and out_stack (max_outgoing) are both
* known, so the prologue immediates and slim-form choice are settled here.
- * frame_size_final must be set before aa_build_prologue_words / entry saves,
- * since the bottom-record offset helpers read it. */
+ * frame_size_final + out_stack_final must be set before aa_build_prologue_words
+ * / entry saves, since the bottom-record offset helpers read both. */
L = aa_build_layout(a->frame.cum_off, a->frame.max_outgoing,
a->top_home_bytes);
a->frame_size_final = L.frame_size;
- /* Slim Tier A: no callee-saves, no alloca, no body slots, no outgoing stack
- * args — the whole frame is the 16-byte record. fp_at_bottom: a small frame
- * with callee-saves/locals and no outgoing stack args; the record moves to
- * the bottom (fp = sp) so sp adjustment folds into the pre/post-indexed
- * stp/ldp (frame_size <= 504 keeps the post-index ldp imm in range).
- * Otherwise slim_small_frame keeps the top-record layout but skips the
- * x17/x10 scratch (out_stack>0 small frames land here). A Windows-variadic
- * home area forces the fat top-record layout: it lives above the saved pair,
- * which neither the slim forms (saved pair at the very top) nor the
- * bottom-record (saved pair at the very bottom) leave room for. (See
- * aa_func_end for the single-pass path, which never takes any slim form.) */
+ a->out_stack_final = L.out_stack;
+ /* Layout choice (W1.1). Slim Tier A: no callee-saves, no alloca, no body
+ * slots, no outgoing stack args — the whole frame is the 16-byte record, with
+ * its own folded `stp [sp,#-16]!` encoding. Otherwise the uniform x29-at-bottom
+ * (fp_at_bottom) layout: x29 anchored out_stack bytes above sp, every slot a
+ * positive [x29,#k], for ALL remaining known frames (out_stack>0, alloca, and
+ * arbitrarily large frames included). The only exception is a Windows-variadic
+ * GP home area, which must sit ABOVE the saved pair (the plain-pointer va_list
+ * walks register then stack varargs as one contiguous block) — bottom-record
+ * has no room for it, so that case keeps the fat top-record layout.
+ * (See aa_func_end for the single-pass path, which never takes any slim/bottom
+ * form.) */
a->slim_prologue = a->frame.ncallee_saves == 0 && !a->frame.has_alloca &&
L.slot_bytes == 0 && L.out_stack == 0 &&
!a->top_home_bytes;
- a->fp_at_bottom = !a->slim_prologue && !a->frame.has_alloca &&
- L.out_stack == 0 && L.frame_size <= 504u &&
- !a->top_home_bytes;
- a->slim_small_frame = !a->slim_prologue && !a->fp_at_bottom &&
- !a->frame.has_alloca && !a->top_home_bytes &&
- aa_sp_off_saved_pair(&L) <= 504u;
+ a->fp_at_bottom = !a->slim_prologue && !a->top_home_bytes;
n = aa_build_prologue_words(a, &L, words, AA_PROLOGUE_WORDS);
for (u32 i = 0; i < n; ++i) aa_emit32(t->mc, words[i]);
a->minimal_prologue_words = n;
diff --git a/test/opt/aa64_tail_call.sh b/test/opt/aa64_tail_call.sh
@@ -50,13 +50,36 @@ if ! grep -Eq '\bb\b.*target.*AARCH64_JUMP26' "$WORK/direct_stack.caller.dis"; t
fail 'direct stack-arg tail missing b target / AARCH64_JUMP26' \
"$WORK/direct_stack.caller.dis"
fi
-if ! grep -Eq '\bstr\b.*\[x29, #16\]' "$WORK/direct_stack.caller.dis" ||
- ! grep -Eq '\bstr\b.*\[x29, #24\]' "$WORK/direct_stack.caller.dis"; then
+# The tail's two stack-arg stores must land in the caller's incoming stack-arg
+# window -- the same [x29,#K] / [x29,#K+8] addresses the caller LOADS its own
+# 9th/10th incoming args from (physically the caller's CFA window). With the
+# W1.1 uniform x29-at-bottom layout that window is at the CFA-relative offset
+# x29 + (frame_size - out_stack), not the old top-record fixed [x29,#16]/[#24];
+# pinning the literal offset would be brittle, so assert that some [x29,#K] is
+# both an incoming-arg load and a tail-arg store, with [x29,#K+8] the same.
+if ! awk '
+ function off(s, t) {
+ if (!match(s, /#[0-9]+\]/)) return -1
+ t = substr(s, RSTART + 1, RLENGTH - 2)
+ return t + 0
+ }
+ /ldr[[:space:]]+x[0-9]+, \[x29, #[0-9]+\]/ { loaded[off($0)] = 1 }
+ /str[[:space:]]+x[0-9]+, \[x29, #[0-9]+\]/ { stored[off($0)] = 1 }
+ END {
+ for (k in stored)
+ if (loaded[k] && stored[k+8] && loaded[k+8]) { ok = 1 }
+ exit(ok ? 0 : 1)
+ }
+' "$WORK/direct_stack.caller.dis"; then
fail 'direct stack-arg tail did not write caller incoming stack window' \
"$WORK/direct_stack.caller.dis"
fi
+# The caller frame must be torn down before the sibling branch. The W1.1 general
+# bottom-record teardown recovers sp from the x29 anchor (`add sp, x16, #...`);
+# the os==0 small-frame fold uses a post-indexed `ldp ...,[sp],#N`. Accept either.
if ! awk '
- /add[[:space:]]+sp, sp,/ { restored = 1 }
+ /add[[:space:]]+sp, (sp|x1[0-9]),/ { restored = 1 }
+ /ldp[[:space:]]+x29, x30, \[sp\], #/ { restored = 1 }
/[[:space:]]b[[:space:]]+.*target.*AARCH64_JUMP26/ {
found = 1; ok = restored; exit
}
diff --git a/test/opt/aa64_x29_bottom.sh b/test/opt/aa64_x29_bottom.sh
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+# Structural checks for the -O1 aa64 uniform x29-at-bottom known-frame layout
+# (doc/plan/O1.md W1.1).
+#
+# At -O1 the frame is fully known before the body emits, so aa64 anchors x29 at
+# the BOTTOM of the static slots (just above the outgoing-arg area). Every spill
+# / local slot is then a one-instruction POSITIVE `ldr/str [x29,#k]` within the
+# 32 KB scaled reach (`add x16,x29,#hi ; ldr [x16,#lo]` only past it). This
+# replaces the old `sub x17,x29,#k ; ldur` two-instruction fallback that was 36%
+# of lvm / 12% of sqlite instructions.
+#
+# The saved x29/x30 pair sits AT [x29]/[x29+8] (the frame-pointer chain kit's
+# unwinder / __kit_backtrace walks) and outgoing args stay sp-relative
+# ([sp,#k]) so calls after an alloca still address their arg area at current sp.
+#
+# These checks pin the resulting disassembly on aarch64 (the reference backend):
+# 1. a big-frame, high-pressure function spills via `ldr/str [x29,#k]` and
+# emits NO `sub xN,x29,#k` for spill addressing;
+# 2. an alloca function (sp floats) still addresses spills via [x29,#k] and
+# never recomputes a slot address with `sub xN,x29,#k`;
+# 3. a >32 KB-frame function uses the add-build (`add x16,x29,#... ; ldr/str`)
+# and never `sub xN,x29,#`;
+# 4. a signed-narrow-load function never uses `sub xN,x29,#` for spills.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+KIT="${KIT:-$ROOT/build/kit}"
+WORK="$ROOT/build/test/opt/aa64_x29_bottom"
+rm -rf "$WORK"
+mkdir -p "$WORK"
+
+fail() {
+ printf 'aa64-x29-bottom check FAILED: %s\n' "$1" >&2
+ if [ -n "${2:-}" ] && [ -f "$2" ]; then
+ sed 's/^/ | /' "$2" >&2
+ fi
+ exit 1
+}
+
+slice_func() {
+ local src="$1" func="$2" out="$3"
+ awk -v name="$func" '
+ $0 ~ "^[0-9a-f]+ <" name ">:" { in_fn = 1; print; next }
+ /^[0-9a-f]+ </ { in_fn = 0 }
+ in_fn { print }
+ ' "$src" > "$out"
+}
+
+compile_case() {
+ local name="$1" src="$2"
+ "$KIT" cc -target aarch64-linux-gnu -O1 -std=c11 -c "$src" \
+ -o "$WORK/$name.o" > "$WORK/$name.cc.out" 2> "$WORK/$name.cc.err" ||
+ { cat "$WORK/$name.cc.err" >&2; fail "compile of $name failed"; }
+ "$KIT" objdump -d "$WORK/$name.o" \
+ > "$WORK/$name.dis" 2> "$WORK/$name.objdump.err"
+}
+
+# A spill never recomputes its address with `sub xN, x29, #k` on the known-frame
+# path -- that is exactly the 2-insn pattern W1.1 removes.
+no_frame_sub() { # $1 = sliced fn file, $2 = label
+ if grep -Eq 'sub[[:space:]]+x[0-9]+, x29, #' "$1"; then
+ fail "$2 recomputes a frame-slot address with 'sub xN, x29, #' (W1.1 should address [x29,#k] positively)" "$1"
+ fi
+}
+
+# ---- fixtures ----
+
+# 1. Big frame (>256-byte spill area) + outgoing stack args (>8-arg calls). This
+# is the case W1.1 targets: a large frame means spill offsets escape `ldur`'s
+# +-256 range, and out_stack>0 disqualifies the old folded bottom-record. The
+# baseline emits `sub x17,x29,#k ; ldur` for each such spill; x29-at-bottom
+# must address every one as a positive [x29,#k] with no frame-address `sub`.
+cat > "$WORK/pressure.c" <<'EOF'
+extern long sink(long,long,long,long,long,long,long,long,long,long);
+long pressure(long *p) {
+ long v[48];
+ for (int i = 0; i < 48; i++) v[i] = p[i] + i;
+ long s = 0;
+ s += sink(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7],v[8],v[9]);
+ s += sink(v[10],v[11],v[12],v[13],v[14],v[15],v[16],v[17],v[18],v[19]);
+ s += sink(v[20],v[21],v[22],v[23],v[24],v[25],v[26],v[27],v[28],v[29]);
+ s += sink(v[30],v[31],v[32],v[33],v[34],v[35],v[36],v[37],v[38],v[39]);
+ for (int i = 0; i < 48; i++) s += v[i];
+ return s;
+}
+EOF
+compile_case pressure "$WORK/pressure.c"
+slice_func "$WORK/pressure.dis" pressure "$WORK/pressure.fn"
+[ -s "$WORK/pressure.fn" ] || fail "pressure not found in disassembly" "$WORK/pressure.dis"
+no_frame_sub "$WORK/pressure.fn" pressure
+grep -Eq '(ldr|str)[[:space:]]+[wx][0-9]+, \[x29, #[0-9]+\]' "$WORK/pressure.fn" ||
+ fail "pressure has no positive [x29,#k] spill access (expected x29-at-bottom)" "$WORK/pressure.fn"
+# The prologue must anchor x29 = sp + os (saved fp/lr pair at [x29]/[x29+8]).
+grep -Eq 'add[[:space:]]+x29, sp' "$WORK/pressure.fn" ||
+ fail "pressure prologue did not anchor x29 = sp + os" "$WORK/pressure.fn"
+
+# 2. alloca: sp floats during the body, but x29 stays put, so spills are still
+# [x29,#k] and outgoing args are [sp,#k]. (The call below takes stack args.)
+cat > "$WORK/alloca_fn.c" <<'EOF'
+extern long use(long *, long, long, long, long, long, long, long, long, long);
+long alloca_fn(long n, long x) {
+ long *p = __builtin_alloca(n * sizeof(long));
+ for (long i = 0; i < n; i++) p[i] = x + i;
+ long r = use(p, x, x+1, x+2, x+3, x+4, x+5, x+6, x+7, x+8);
+ return r + p[0];
+}
+EOF
+compile_case alloca_fn "$WORK/alloca_fn.c"
+slice_func "$WORK/alloca_fn.dis" alloca_fn "$WORK/alloca_fn.fn"
+[ -s "$WORK/alloca_fn.fn" ] || fail "alloca_fn not found in disassembly" "$WORK/alloca_fn.dis"
+no_frame_sub "$WORK/alloca_fn.fn" alloca_fn
+grep -Eq 'add[[:space:]]+x29, sp' "$WORK/alloca_fn.fn" ||
+ fail "alloca_fn prologue did not anchor x29 = sp + os" "$WORK/alloca_fn.fn"
+
+# 3. >32 KB frame: the scaled `ldr/str [x29,#k]` reach is exceeded, so the slot
+# address is built with `add x16/x17, x29, #hi` -- never `sub xN, x29, #k`.
+cat > "$WORK/huge.c" <<'EOF'
+extern void use_buf(volatile char *, volatile char *);
+long huge(long x) {
+ volatile char a[40000];
+ volatile char b[8];
+ a[0] = (char)x;
+ b[0] = (char)(x + 1);
+ use_buf(a, b);
+ return a[0] + b[0];
+}
+EOF
+compile_case huge "$WORK/huge.c"
+slice_func "$WORK/huge.dis" huge "$WORK/huge.fn"
+[ -s "$WORK/huge.fn" ] || fail "huge not found in disassembly" "$WORK/huge.dis"
+no_frame_sub "$WORK/huge.fn" huge
+grep -Eq 'add[[:space:]]+x1[67], x29, #' "$WORK/huge.fn" ||
+ fail "huge >32KB frame did not build a far slot address with 'add x16/x17, x29, #'" "$WORK/huge.fn"
+
+# 4. signed narrow loads in a big frame: spilled narrow values still address
+# [x29,#k] (ldrsb when reloaded as signed), never a sub-x29 recompute.
+cat > "$WORK/snl.c" <<'EOF'
+extern long sink2(long,long,long,long,long,long,long,long,long,long);
+long snl(signed char *p) {
+ signed char v[400];
+ for (int i = 0; i < 400; i++) v[i] = p[i];
+ long s = sink2(v[0],v[40],v[80],v[120],v[160],v[200],v[240],v[280],v[320],v[360]);
+ for (int i = 0; i < 400; i += 17) s += v[i];
+ return s;
+}
+EOF
+compile_case snl "$WORK/snl.c"
+slice_func "$WORK/snl.dis" snl "$WORK/snl.fn"
+[ -s "$WORK/snl.fn" ] || fail "snl not found in disassembly" "$WORK/snl.dis"
+no_frame_sub "$WORK/snl.fn" snl
+
+printf 'aa64-x29-bottom: ok (pressure, alloca, >32KB add-build, signed narrow)\n'
diff --git a/test/opt/redundant_frame_sub.sh b/test/opt/redundant_frame_sub.sh
@@ -2,15 +2,17 @@
# Structural check for the -O1 local frame-address `sub`-CSE (O1.md W1a).
#
# An `IR_ADDR_OF` of a frame-local materializes a frame-slot address into a
-# register; on aarch64 that lowers to `sub xN, x29, #k` (the offset built off the
-# frame base). Two back-to-back addr-of's of the *same* local recompute the
-# identical `sub xN, x29, #k`. The W1a peephole in src/opt/pass_combine.c
-# (mir_combine) tracks the still-live earlier producer and rewrites the
-# recompute into a register move (`mov xN, xM`), so the second `sub` vanishes.
+# register; under the W1.1 x29-at-bottom layout aarch64 builds it as a positive
+# `add xN, x29, #k` off the frame base. Two back-to-back addr-of's of the *same*
+# local recompute the identical `add xN, x29, #k`. The W1a peephole in
+# src/opt/pass_combine.c (mir_combine) tracks the still-live earlier producer and
+# rewrites the recompute into a register move (`mov xN, xM`), so the second
+# address build vanishes. (Before W1.1 the build was a negative `sub xN,x29,#k`;
+# W1.1 made frame addressing positive, but the W1a CSE still applies to the add.)
#
# This pins the resulting disassembly on aarch64 (the reference backend), where
# the pattern has a stable mnemonic. Each local is addressed twice as adjacent
-# call arguments; with the peephole only ONE `sub xN, x29, #k` survives per
+# call arguments; with the peephole only ONE `add xN, x29, #k` survives per
# distinct constant K, and the duplicate becomes a `mov`.
set -euo pipefail
@@ -55,16 +57,16 @@ B="$(fn_body _g)"
[ -n "$B" ] || B="$(fn_body g)"
[ -n "$B" ] || fail "g not found in disassembly"
-# For each distinct constant K appearing in `sub xN, x29, #K`, the address is
+# For each distinct constant K appearing in `add xN, x29, #K`, the address is
# built exactly once; a back-to-back recompute of the same K must NOT survive.
# Collect the K constants and assert no duplicates.
-KS="$(printf '%s\n' "$B" | grep -oE 'sub[[:space:]]+x[0-9]+, x29, #[0-9]+' \
+KS="$(printf '%s\n' "$B" | grep -oE 'add[[:space:]]+x[0-9]+, x29, #[0-9]+' \
| grep -oE '#[0-9]+$' | sort)"
-[ -n "$KS" ] || fail "expected at least one 'sub xN, x29, #k' frame-address build"
+[ -n "$KS" ] || fail "expected at least one 'add xN, x29, #k' frame-address build"
DUP="$(printf '%s\n' "$KS" | uniq -d || true)"
if [ -n "$DUP" ]; then
- fail "duplicate frame-address build survived (sub xN, x29, #k recomputed for: $DUP)"
+ fail "duplicate frame-address build survived (add xN, x29, #k recomputed for: $DUP)"
fi
# Sanity: the de-duplicated address must be reused via a register move, so the
@@ -74,4 +76,4 @@ if ! printf '%s\n' "$B" | grep -Eq 'mov[[:space:]]+x[0-9]+, x[0-9]+'; then
fail "expected a 'mov xN, xM' reusing the CSE'd frame address"
fi
-printf 'redundant_frame_sub: OK (no duplicate sub xN,x29,#k; address reused via mov)\n'
+printf 'redundant_frame_sub: OK (no duplicate add xN,x29,#k; address reused via mov)\n'
