commit 77852d159a7e48ca536dbc560544eaa5114a1a84
parent edc69e06f3adb3e28162278281e22517db860d06
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 08:04:27 -0700
opt(aa64): widen AGG_SET zero-init via byte-splat + wide stores (O1-PATTERNS L3)
aa_set_bytes emitted one strb per byte even for an aligned whole-struct/array
zero (cjson had a 64-long strb run). Splat the fill byte across a 64-bit
scratch (and #0xff + ORR-shift doublings) once, then store the widest aligned
chunk covering the remaining run (str x -> str w -> strh -> strb tail),
mirroring the aa_copy_bytes_dir width ladder. Size-gated (>=8 bytes) so tiny
runs keep the cheaper per-byte loop. Correct for any fill byte, unaligned
base/size, and exact count -- pure per-call expansion, removes instructions.
A 64-byte zero now: 4-insn splat + 8 str x (was 64 strb + movz). Guard
test/opt/o1p_aa64.sh (L3 section).
Diffstat:
| M | src/arch/aa64/native.c | | | 83 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- |
| A | test/opt/o1p_aa64.sh | | | 124 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
2 files changed, 205 insertions(+), 2 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -2574,6 +2574,21 @@ static void aa_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
aa_copy_bytes_dir(t, dst, src, access, 0);
}
+/* L3: widen the AGG_SET (whole-struct/array zero-init, memset) expander. The
+ * old form emitted one `strb` per byte even for an aligned whole-struct zero
+ * (cjson had a 64-long strb run). Instead, splat the fill byte across a 64-bit
+ * register once and store the widest aligned chunk covering the remaining run
+ * (`str x` -> `str w` -> `strh` -> `strb` tail), exactly mirroring the
+ * aa_copy_bytes_dir width ladder. This is a pure per-call expansion (it only
+ * removes emitted instructions); correctness holds for any fill byte (the splat
+ * replicates it to all 8 bytes), for unaligned bases/sizes (the ladder narrows
+ * the chunk to the remaining run, which the store offset never over-runs), and
+ * for the exact byte count.
+ *
+ * The splat costs a few up-front instructions, so for tiny runs the per-byte
+ * loop is still cheaper; gate the widening on a size threshold. */
+#define AA_SET_BYTES_WIDEN_MIN 8u
+
static void aa_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc byte_value,
AggregateAccess access) {
KitCgTypeId i8 = builtin_id(KIT_CG_BUILTIN_I8);
@@ -2583,8 +2598,72 @@ static void aa_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc byte_value,
mem.size = 1u;
mem.align = 1u;
byte.type = i8;
- for (u32 off = 0; off < access.size; ++off)
- aa_store_native(t, aa_addr_plus(dst, off), byte, mem);
+ if (access.size < AA_SET_BYTES_WIDEN_MIN || native_loc_is_fp(byte_value)) {
+ /* Small run (or a non-integer fill value): the original per-byte loop. */
+ for (u32 off = 0; off < access.size; ++off)
+ aa_store_native(t, aa_addr_plus(dst, off), byte, mem);
+ return;
+ }
+ /* Splat the fill byte to every byte of a 64-bit scratch (AA_TMP0), distinct
+ * from the address base/index (IR_AGG_SET materializes the value clear of
+ * them). `and w,b,#0xff` clears any high bits, then ORR-shift doublings build
+ * bb -> bbbb -> bbbbbbbb. For a zero fill these collapse to zeros, but stay
+ * correct; the win is replacing N byte stores with N/8 dword stores. */
+ {
+ MCEmitter* mc = t->mc;
+ KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
+ KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
+ KitCgTypeId i16 = builtin_id(KIT_CG_BUILTIN_I16);
+ u32 b = loc_reg(byte_value);
+ NativeLoc val = aa_tmp_loc(i64, AA_TMP0);
+ u32 immr = 0, imms = 0, N = 0;
+ u32 off = 0;
+ (void)aa64_logimm_encode(0xffu, 0u, &N, &immr, &imms);
+ aa_emit32(mc, aa64_and_imm(0u, AA_TMP0, b, N, immr, imms)); /* w16 = b&0xff */
+ aa_emit32(mc, aa64_logsr_pack((AA64LogSR){.sf = 0u,
+ .opc = AA64_LOG_ORR_OPC,
+ .Rm = AA_TMP0,
+ .imm6 = 8u,
+ .Rn = AA_TMP0,
+ .Rd = AA_TMP0})); /* orr w,w,w<<8 */
+ aa_emit32(mc, aa64_logsr_pack((AA64LogSR){
+ .sf = 0u,
+ .opc = AA64_LOG_ORR_OPC,
+ .Rm = AA_TMP0,
+ .imm6 = 16u,
+ .Rn = AA_TMP0,
+ .Rd = AA_TMP0})); /* orr w,w,w<<16 -> low 32 = bbbb */
+ aa_emit32(mc, aa64_logsr_pack((AA64LogSR){
+ .sf = 1u,
+ .opc = AA64_LOG_ORR_OPC,
+ .Rm = AA_TMP0,
+ .imm6 = 32u,
+ .Rn = AA_TMP0,
+ .Rd = AA_TMP0})); /* orr x,x,x<<32 -> all 8 bytes = b */
+ while (off < access.size) {
+ u32 rem = access.size - off;
+ if (rem >= 8u) {
+ mem.type = i64;
+ mem.size = 8u;
+ val.type = i64;
+ } else if (rem >= 4u) {
+ mem.type = i32;
+ mem.size = 4u;
+ val.type = i32;
+ } else if (rem >= 2u) {
+ mem.type = i16;
+ mem.size = 2u;
+ val.type = i16;
+ } else {
+ mem.type = i8;
+ mem.size = 1u;
+ val.type = i8;
+ }
+ mem.align = mem.size;
+ aa_store_native(t, aa_addr_plus(dst, off), val, mem);
+ off += mem.size;
+ }
+ }
}
static void aa_lsl_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh);
diff --git a/test/opt/o1p_aa64.sh b/test/opt/o1p_aa64.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Structural checks for two aa64 -O1 code-size follow-ups
+# (doc/plan/O1-PATTERNS.md L3 + L10).
+#
+# L3 -- widen aggregate / memset zero-init. The aa64 AGG_SET expander
+# (aa_set_bytes) used to emit one `strb` per byte for a whole-struct / array
+# zero (cjson has a 64-long strb run). It now emits the widest aligned store
+# covering the remaining run (`str xzr`/`str x`/`str w`/`strh`/`strb` tail),
+# so a 64-byte zero-init must NOT produce a long `strb` run and MUST use wider
+# (8/4-byte) stores.
+#
+# L10 -- frame elision on no-spill leaf functions. kit used to emit a full
+# `stp x29,x30,[sp,#-16]!; add x29,sp,#0 ... ldp` frame even for a leaf that
+# spills nothing and never touches the frame. Such a function must now emit NO
+# `stp x29,x30` / `add x29,sp` frame, while a frame-needing function (call /
+# alloca / spill / address-taken local) MUST still keep its frame.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+KIT="${KIT:-$ROOT/build/kit}"
+WORK="$ROOT/build/test/opt/o1p_aa64"
+rm -rf "$WORK"
+mkdir -p "$WORK"
+
+fail() {
+ printf 'o1p-aa64 check FAILED: %s\n' "$1" >&2
+ if [ -n "${2:-}" ] && [ -f "$2" ]; then
+ sed 's/^/ | /' "$2" >&2
+ fi
+ exit 1
+}
+
+slice_func() {
+ local src="$1" func="$2" out="$3"
+ awk -v name="$func" '
+ $0 ~ "^[0-9a-f]+ <" name ">:" { in_fn = 1; print; next }
+ /^[0-9a-f]+ </ { in_fn = 0 }
+ in_fn { print }
+ ' "$src" > "$out"
+}
+
+compile_case() {
+ local name="$1" src="$2"
+ "$KIT" cc -target aarch64-linux-gnu -O1 -std=c11 -c "$src" \
+ -o "$WORK/$name.o" > "$WORK/$name.cc.out" 2> "$WORK/$name.cc.err" ||
+ { cat "$WORK/$name.cc.err" >&2; fail "compile of $name failed"; }
+ "$KIT" objdump -d "$WORK/$name.o" \
+ > "$WORK/$name.dis" 2> "$WORK/$name.objdump.err"
+}
+
+# Count `strb` instructions in a sliced function (the mnemonic is after the
+# second tab in objdump output).
+count_strb() { grep -cE $'\t''strb' "$1" || true; }
+
+# ============================================================================
+# L3: widen aggregate zero-init
+# ============================================================================
+# A 64-byte aggregate zeroed via `= {0}` -- the cjson PrintBuffered shape. The
+# baseline emits ~64 consecutive `strb`; the widened expander must emit far
+# fewer strb (only an unaligned tail at most) and use 8/4-byte stores.
+cat > "$WORK/zero64.c" <<'EOF'
+struct Big { long a,b,c,d,e,f,g,h; }; /* 64 bytes, 8-aligned */
+void zero64(struct Big *out) {
+ struct Big z = {0};
+ *out = z;
+}
+char buf64[64];
+void zerobuf(void) {
+ char local[64] = {0};
+ __builtin_memcpy(buf64, local, 64);
+}
+EOF
+compile_case zero64 "$WORK/zero64.c"
+slice_func "$WORK/zero64.dis" zero64 "$WORK/zero64.fn"
+[ -s "$WORK/zero64.fn" ] || fail "zero64 not found in disassembly" "$WORK/zero64.dis"
+nstrb=$(count_strb "$WORK/zero64.fn")
+# An 8-aligned 64-byte zero should produce ZERO per-byte stores after widening.
+if [ "$nstrb" -gt 8 ]; then
+ fail "zero64 still emits a long strb run ($nstrb strb); expected wide stores (L3)" "$WORK/zero64.fn"
+fi
+# It must use wider stores (8-byte `str x..`, or a 4-byte `str w..`).
+grep -Eq $'\t''str[[:space:]]+x[0-9]+' "$WORK/zero64.fn" ||
+ fail "zero64 did not widen to 8-byte stores (str xN) (L3)" "$WORK/zero64.fn"
+
+# ============================================================================
+# L10: frame elision on no-spill leaf functions
+# ============================================================================
+# A one-op leaf accessor: no call, no spill, no address-taken local, not
+# varargs. It must NOT establish a frame.
+cat > "$WORK/leaf.c" <<'EOF'
+double add(double a, double b) { return a + b; }
+long getfield(long *p) { return p[3]; }
+int sum3(int a, int b, int c) { return a + b + c; }
+EOF
+compile_case leaf "$WORK/leaf.c"
+for fn in add getfield sum3; do
+ slice_func "$WORK/leaf.dis" "$fn" "$WORK/leaf.$fn.fn"
+ [ -s "$WORK/leaf.$fn.fn" ] || fail "$fn not found in disassembly" "$WORK/leaf.dis"
+ if grep -Eq $'\t''stp[[:space:]]+x29, x30' "$WORK/leaf.$fn.fn"; then
+ fail "$fn establishes a frame (stp x29,x30) but is a no-spill leaf (L10)" "$WORK/leaf.$fn.fn"
+ fi
+ if grep -Eq $'\t''add[[:space:]]+x29, sp' "$WORK/leaf.$fn.fn"; then
+ fail "$fn anchors x29 but is a no-spill leaf (L10)" "$WORK/leaf.$fn.fn"
+ fi
+ grep -Eq $'\t''ret' "$WORK/leaf.$fn.fn" ||
+ fail "$fn has no ret" "$WORK/leaf.$fn.fn"
+done
+
+# Frame-NEEDING functions must STILL establish a frame.
+cat > "$WORK/framed.c" <<'EOF'
+extern long ext(long);
+long calls(long x) { return ext(x) + 1; } /* non-leaf: keeps frame */
+long takesaddr(long x) { long v = x; long *p = &v; return *p + ext(x); }
+long recurse(long n) { return n <= 1 ? 1 : n * recurse(n - 1); }
+EOF
+compile_case framed "$WORK/framed.c"
+for fn in calls takesaddr recurse; do
+ slice_func "$WORK/framed.dis" "$fn" "$WORK/framed.$fn.fn"
+ [ -s "$WORK/framed.$fn.fn" ] || fail "$fn not found in disassembly" "$WORK/framed.dis"
+ grep -Eq $'\t''stp[[:space:]]+x29, x30' "$WORK/framed.$fn.fn" ||
+ fail "$fn dropped its frame but needs one (call/addr/recursion) (L10)" "$WORK/framed.$fn.fn"
+done
+
+printf 'o1p-aa64: ok (L3 wide zero-init, L10 leaf frame elision + framed kept)\n'
