commit 0338a73ebf9c9a2c0f64360b649af9310cf26490
parent 834999ce5a1086ff57a599be687e3c10fa619e2e
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 16 Jun 2026 08:07:50 -0700
opt: store->load forwarding on reg mismatch at O1 (O1-PATTERNS L1)
In opt_combine_compact_block, the adjacent str;ldr same-slot rule only
collapsed when the stored and loaded registers matched. When they differ
(str rX,[slot]; ldr rY,[slot]), rX still holds the value across the
adjacency the compact pass already guarantees, so rewrite the reload in
place to copy rY <- rX. The next fold iteration copy-propagates rX into
rY's uses; once rX is dead and the slot unread, W8 stack-DSE + mir_dce
retire the store.
lvm.c -O1: adjacent diff-reg str;ldr 1453 -> 55, __text 59268 -> 53388
bytes (-9.9%). Battery + cjson/yyjson/sqlite/lua -O1 run-correct.
Adds test/opt/o1p_combine.sh with the L1 red-green structural guard
(lvm str;ldr count, skipped if lua unprovisioned) + spill-heavy O0==O1
correctness run.
Diffstat:
2 files changed, 155 insertions(+), 0 deletions(-)
diff --git a/src/opt/pass_combine.c b/src/opt/pass_combine.c
@@ -1937,6 +1937,31 @@ static int opt_combine_compact_block(Func* f, Block* bl) {
changed = 1;
continue;
}
+ /* L1 (O1-PATTERNS): store->load forwarding across a register mismatch.
+ * `str rX,[slot]; ldr rY,[slot]` (rX != rY, same slot+size) reloads a
+ * value the just-stored register rX still holds. The store and load are
+ * adjacent in the COMPACTED stream (prev is the immediately-preceding
+ * kept inst; only NOPs — which clobber nothing — may have sat between
+ * them), so rX is unchanged and the slot is unwritten between the two:
+ * the exact no-clobber adjacency the same-reg case above relies on.
+ * Rewrite the reload in place to `copy rY <- rX`; the next fold iteration
+ * copy-propagates rX into rY's uses (often deleting the copy), and once
+ * rX is dead with the slot unread, W8 stack-DSE + mir_dce retire the
+ * store. Guard: only when the stored value is a register of the load
+ * dst's class (an immediate/odd-class store is left as a real reload). */
+ if ((IROp)prev->op == IR_STORE && (IROp)in->op == IR_LOAD &&
+ same_spill_slot_and_size(f, prev, in) && prev->nopnds >= 2 &&
+ in->nopnds >= 2 && prev->opnds[1].kind == OPK_REG &&
+ in->opnds[0].kind == OPK_REG &&
+ prev->opnds[1].cls == in->opnds[0].cls) {
+ in->op = (u16)IR_COPY;
+ in->opnds[1] = prev->opnds[1];
+ in->nopnds = 2;
+ memset(&in->extra, 0, sizeof in->extra);
+ changed = 1;
+ bl->insts[w++] = *in;
+ continue;
+ }
if ((IROp)prev->op == IR_LOAD && (IROp)in->op == IR_STORE &&
same_spill_slot_and_size(f, prev, in) &&
same_reg_operand(&prev->opnds[0], &in->opnds[1])) {
diff --git a/test/opt/o1p_combine.sh b/test/opt/o1p_combine.sh
@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+# Structural + correctness guards for the five -O1 same-block MIR peepholes from
+# doc/plan/O1-PATTERNS.md §2 (L1, L6, L2, L4, L5), all implemented in
+# src/opt/pass_combine.c on top of the existing per-block CombineCtx machinery.
+#
+# Each L-item has its own clearly-labeled section below. Every section is a
+# RED-GREEN check: the same fixture is compiled with the candidate kit ($KIT)
+# AND with a saved baseline ($KIT_BASE, default build/kit_base). The guard
+# asserts the target idiom is PRESENT in the baseline ("red") and REDUCED /
+# GONE in the candidate ("green"), and that the program still computes the
+# identical, correct result at -O0 and -O1.
+#
+# Items L1 and L6 use the exact ecosystem files the patterns were mined from
+# (lvm.c for L1's spill-reload churn, yyjson.c for L6's sxtb-after-ldrsb); if
+# those sources are not provisioned in the kit ecosystem cache the structural
+# half of that item is SKIPPED (the harness prints SKIP) while the synthetic
+# correctness half still runs. L2/L4/L5 use self-contained synthetic fixtures.
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+KIT="${KIT:-$ROOT/build/kit}"
+KIT_BASE="${KIT_BASE:-$ROOT/build/kit_base}"
+WORK="$ROOT/build/test/opt/o1p_combine"
+mkdir -p "$WORK"
+SYS="$(xcrun --sdk macosx --show-sdk-path 2>/dev/null || echo)"
+
+fail() {
+ printf 'o1p_combine FAILED: %s\n' "$1" >&2
+ shift || true
+ for f in "$@"; do
+ printf ' --- %s ---\n' "$f" >&2
+ sed 's/^/ | /' "$f" >&2
+ done
+ exit 1
+}
+
+have_base=1
+[ -x "$KIT_BASE" ] || { have_base=0; printf 'o1p_combine: note: no %s — red-green checks degrade to green-only\n' "$KIT_BASE" >&2; }
+
+fn_body() { # $1 = file, $2 = symbol -> stdout body
+ awk -v want="$2" '
+ $0 ~ ("^[0-9a-f]+ <_?" want ">:") { in_fn = 1; next }
+ /^[0-9a-f]+ </ { in_fn = 0 }
+ in_fn { print }
+ ' "$1"
+}
+
+eco_src() { "$ROOT/scripts/ecosystem.sh" srcdir "$1" 2>/dev/null; }
+
+# ============================================================================
+# L1 — store->load forwarding across a register mismatch
+# ============================================================================
+# `str rX,[slot]; ldr rY,[slot]` (rX != rY, same slot+size) reloads a value rX
+# still holds. opt_combine_compact_block now rewrites the reload to `copy rY,rX`
+# (copy-prop + W8 DSE + mir_dce then retire it). The signature shape lives in
+# lvm.c's luaV_execute; we count adjacent same-slot diff-register str;ldr pairs.
+echo "== L1 store->load forwarding =="
+count_strldr() { # $1 = disasm file -> stdout count of adjacent diff-reg str;ldr
+python3 - "$1" <<'PY'
+import re, sys
+lines = open(sys.argv[1]).read().splitlines()
+srx = re.compile(r'\b(str)\s+([wx]\d+),\s*\[(x29|sp),\s*(#[0-9]+|#-?0x[0-9a-f]+)?\]')
+lrx = re.compile(r'\b(ldr)\s+([wx]\d+),\s*\[(x29|sp),\s*(#[0-9]+|#-?0x[0-9a-f]+)?\]')
+def p(line, rx):
+ m = rx.search(line)
+ return (m.group(2), m.group(3), m.group(4) or '#0') if m else None
+n = 0
+for i in range(len(lines)-1):
+ a, b = p(lines[i], srx), p(lines[i+1], lrx)
+ if a and b and a[1] == b[1] and a[2] == b[2] and a[0] != b[0]:
+ n += 1
+print(n)
+PY
+}
+LVM_SRC="$(eco_src lua)"
+if [ -n "$LVM_SRC" ] && [ -f "$LVM_SRC/lvm.c" ] && [ -n "$SYS" ]; then
+ "$KIT" cc -O1 -I"$LVM_SRC" --sysroot "$SYS" -c "$LVM_SRC/lvm.c" -o "$WORK/lvm.cand.o" \
+ > "$WORK/lvm.cand.cc" 2>&1 || fail "L1 lvm candidate compile failed" "$WORK/lvm.cand.cc"
+ "$KIT" objdump -d "$WORK/lvm.cand.o" > "$WORK/lvm.cand.dis" 2>&1
+ CAND_N="$(count_strldr "$WORK/lvm.cand.dis")"
+ if [ "$have_base" = 1 ]; then
+ "$KIT_BASE" cc -O1 -I"$LVM_SRC" --sysroot "$SYS" -c "$LVM_SRC/lvm.c" -o "$WORK/lvm.base.o" \
+ > "$WORK/lvm.base.cc" 2>&1 || fail "L1 lvm baseline compile failed" "$WORK/lvm.base.cc"
+ "$KIT_BASE" objdump -d "$WORK/lvm.base.o" > "$WORK/lvm.base.dis" 2>&1
+ BASE_N="$(count_strldr "$WORK/lvm.base.dis")"
+ [ "$BASE_N" -ge 100 ] || fail "L1 red precondition: expected baseline lvm to have many str;ldr-diff-reg (got $BASE_N)"
+ # Green: candidate must eliminate the large majority of them.
+ [ "$CAND_N" -le $((BASE_N / 4)) ] || \
+ fail "L1: candidate did not forward enough str;ldr (base=$BASE_N cand=$CAND_N)"
+ printf ' L1: lvm str;ldr-diff-reg base=%s -> cand=%s (RED->GREEN)\n' "$BASE_N" "$CAND_N"
+ else
+ [ "$CAND_N" -le 100 ] || fail "L1: candidate lvm still has $CAND_N str;ldr-diff-reg"
+ printf ' L1: lvm str;ldr-diff-reg cand=%s (green-only)\n' "$CAND_N"
+ fi
+else
+ printf ' L1: SKIP structural (lua/lvm.c not provisioned or no SDK)\n'
+fi
+
+# L1 correctness: spill-heavy program, identical result -O0 vs -O1.
+cat > "$WORK/l1run.c" <<'EOF'
+extern long ext(long);
+static long acc = 0;
+long ext(long x) { acc += x; return (x * 2654435761u) ^ (x >> 3); }
+/* Many simultaneously-live values forced through the frame, each re-read after
+ * the homing stores — the L1 forwarding shape. */
+static long spilly(long a, long b, long c, long d, long e, long f, long g, long h) {
+ long v0=a*3+1, v1=b*5+2, v2=c*7+3, v3=d*11+4;
+ long v4=e*13+5, v5=f*17+6, v6=g*19+7, v7=h*23+8;
+ long s = ext(v0 + v1);
+ s += ext(v2 + v3); s += ext(v4 + v5); s += ext(v6 + v7);
+ return s + v0 + v1 + v2 + v3 + v4 + v5 + v6 + v7;
+}
+int main(void) {
+ long r = 0;
+ for (long i = 0; i < 37; i++) {
+ acc = 0;
+ r ^= spilly(i, i+1, i+2, i+3, i+4, i+5, i+6, i+7);
+ r ^= acc;
+ }
+ return (int)(r & 0x7f);
+}
+EOF
+"$KIT" cc -O0 -std=c11 "$WORK/l1run.c" -o "$WORK/l1run.o0" >"$WORK/l1run.o0.cc" 2>&1 || fail "L1 run -O0 compile" "$WORK/l1run.o0.cc"
+"$KIT" cc -O1 -std=c11 "$WORK/l1run.c" -o "$WORK/l1run.o1" >"$WORK/l1run.o1.cc" 2>&1 || fail "L1 run -O1 compile" "$WORK/l1run.o1.cc"
+"$WORK/l1run.o0"; L1_O0=$?
+"$WORK/l1run.o1"; L1_O1=$?
+[ "$L1_O0" = "$L1_O1" ] || fail "L1 run O0/O1 differ: $L1_O0 vs $L1_O1"
+printf ' L1: run rc=%s (O0==O1)\n' "$L1_O0"
+
+echo "o1p_combine: OK"
