commit 1812cce424a193d2044059895dde09565734344c
parent cd46fd6eb67337c8e1c94cd0ac0adc42f9897d23
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 17:38:54 -0700
perf(cg): -O0 args into ABI arg regs (PERF-O0-CODESIZE Lever 1, aa64) — -3.8% .text
Front the ABI argument/return registers in the aa64 -O0 NDT value-cache pool
(x0..x7 in aa_int_allocable, v0..v7 in aa_fp_allocable) AND materialize call
arguments arg0-first, so an argument expression computes straight into its ABI
register and the per-call parallel-copy collapses to no-ops — the tcc
get_reg(0..N) + eager-left-to-right model. rv64 already pools a0..a7; aa64 was
the lone holdout.
The two halves are both required. The pool fronting alone slightly regressed:
kit materializes computed args lazily, right-to-left, so with arg regs in the
pool they landed in the *reversed* arg registers and the shuffle had to permute
them (f(c,a,b)-shaped cycles). api_pack_call_args_in_order reverses the on-stack
arg order so the LIFO pop yields arg0 first, then pops+materializes forward —
preserving api_temp_dead semantics exactly (not-yet-materialized args stay on the
stack, so a temp shared between args is coalesced at its last use, never killed
early). Matches tcc; on a 4-computed-arg call, 4/6 movs -> 0.
Also drop x11 from aa_int_allocable (it is int emit scratch, was double-listed)
and refresh the now-stale "ret reg excluded from the cache pool" comment in
nd_call (x0 is an ordinary -O0 pool member now; the evict-prior-owner guard is
the real backstop).
sqlite3.c -c, arm64-macOS, vs HEAD baseline (same SDK):
.text 1,486,660 -> 1,430,316 B (-3.79%)
insns 371,537 -> 357,707 (-13,830)
mov 48,578 -> 36,188 (-12,390, -25.5%)
Gate (codesize-track, not byte-identity): determinism (compile x2 + cmp);
test-toy 1392/0, test-cg-api, test-opt, test-smoke-x64/-rv64, test-parse-ok/-err,
test-dwarf, test-debug all green; sqlite e2e at -O0 and -O1 matches clang;
shared-operand / nested-call / arg-eval-order functional check matches clang.
Diffstat:
3 files changed, 63 insertions(+), 26 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -4069,27 +4069,39 @@ static void aa_set_loc(NativeTarget* t, SrcLoc loc) {
static void aa_bind_native_param(NativeTarget* t, const CGParamDesc* p,
NativeLoc dst);
-/* Caller-saved allocables come first so the allocator prefers them (lower
- * spill_cost); callee-saved x19..x28 / v8..v15 are appended and only chosen
- * under register pressure, after which the prologue saves/restores them. */
-static const Reg aa_int_allocable[] = {8u, 11u, 12u, 13u, 14u, 15u, 19u, 20u,
- 21u, 22u, 23u, 24u, 25u, 26u, 27u, 28u};
+/* ABI argument/return registers (x0..x7) come FIRST so the -O0 value cache
+ * prefers them: an argument expression then computes straight into its ABI arg
+ * register and the per-call parallel-copy (native_arg_shuffle) collapses to
+ * no-ops — the tcc get_reg(0..N) trick (mirrors rv_int_allocable a0..a7 and
+ * x64's leading rsi/rdi). They are caller-saved, so the deferred prologue stays
+ * callee-save-free. Incoming params are spilled to frame homes at entry before
+ * any body op (aa_bind_native_param), so caching in x0..x7 cannot clobber a
+ * live parameter. Next come the remaining caller-saved temps x8/x12..x15;
+ * callee-saved x19..x28 are appended and only chosen under register pressure,
+ * after which the prologue saves/restores them. x9/x10/x11 are the int emit
+ * scratch (aa_int_scratch) and stay out of this pool. */
+static const Reg aa_int_allocable[] = {0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u,
+ 8u, 12u, 13u, 14u, 15u, 19u, 20u, 21u,
+ 22u, 23u, 24u, 25u, 26u, 27u, 28u};
/* Three int scratch registers, not two: a 3-operand op (e.g. `binop dst, a, b`
* or `store [base+index], value`) whose dst/sources all spill needs three
* distinct scratch regs at emit time — the IR spill-rewrite round-robins
* operands across this pool and the native emitter materializes each into one.
* With only two, an immediate operand of an all-spilled binop had nowhere to
* land. x9/x10/x11 are all caller-saved temporaries reserved out of the
- * allocable set below. */
+ * allocable set above. */
static const Reg aa_int_scratch[] = {9u, 10u, 11u};
-/* Caller-saved fp temps first (v18/v19, v22..v31) — these form the -O0 value
- * cache under ndt_caller_saved_only. v16 is the fp emit scratch (the fp AA_TMP0,
- * used per-op) and v20/v21 are the NDT per-op fp scratch, so all three stay out
- * of the pool. Callee-saved v8..v15 (AAPCS64) are appended like the int set;
- * the caller-saved-only NDT never selects them. */
-static const Reg aa_fp_allocable[] = {18u, 19u, 22u, 23u, 24u, 25u, 26u,
- 27u, 28u, 29u, 30u, 31u, 8u, 9u,
- 10u, 11u, 12u, 13u, 14u, 15u};
+/* ABI fp argument/return registers (v0..v7) come FIRST (same rationale as the
+ * int pool: fp args land in place, the shuffle is a no-op). Then the caller-
+ * saved fp temps v18/v19, v22..v31 — the rest of the -O0 value cache under
+ * ndt_caller_saved_only. v16 is the fp emit scratch (the fp AA_TMP0, also the
+ * arg-shuffle fp cycle scratch) and v20/v21 are the NDT per-op fp scratch, so
+ * all three stay out of the pool. Callee-saved v8..v15 (AAPCS64) are appended
+ * like the int set; the caller-saved-only NDT never selects them. */
+static const Reg aa_fp_allocable[] = {0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u,
+ 18u, 19u, 22u, 23u, 24u, 25u, 26u, 27u,
+ 28u, 29u, 30u, 31u, 8u, 9u, 10u, 11u,
+ 12u, 13u, 14u, 15u};
static const Reg aa_fp_scratch[] = {20u, 21u};
#define AA_PHYS_INT_ALLOC(r) \
diff --git a/src/cg/call.c b/src/cg/call.c
@@ -82,6 +82,35 @@ void api_pack_call_arg(KitCg* g, CGLocal* out, KitCgTypeId fty, u32 idx) {
*out = api_materialize_call_local(g, &arg, aty);
}
+/* Pack the NARGS top-of-stack call arguments into ARGS[0..nargs) IN ARG ORDER,
+ * materializing each (emitting any pending producer) arg0-first. The arguments
+ * sit on the value stack as `arg0 .. arg(n-1)` (arg(n-1) on top); reverse that
+ * sub-array so the LIFO pop yields arg0 first, then pop+materialize forward.
+ *
+ * Why forward: an argument materialized into a register lands in the first free
+ * -O0 cache register, and the cache pool front-loads the ABI argument registers
+ * (x0,x1,.. — see aa_int_allocable). Producing arg0 first therefore lands it in
+ * x0, arg1 in x1, .. so the per-call parallel-copy collapses to no-ops, instead
+ * of the reverse order leaving each value in the *wrong* arg register and
+ * forcing a permutation shuffle. Matches tcc's eager left-to-right argument
+ * emission.
+ *
+ * Reversing the on-stack order (rather than popping all up front) preserves the
+ * api_temp_dead semantics exactly: the not-yet-materialized arguments stay on
+ * the stack, so a temp shared between arguments (e.g. f(t*2, t*3)) is scanned as
+ * live until its last (highest-index) use and is coalesced there, never killed
+ * prematurely. */
+static void api_pack_call_args_in_order(KitCg* g, CGLocal* args, u32 nargs,
+ KitCgTypeId fty) {
+ u32 base = g->sp - nargs; /* arg0's stack slot (callee, if any, sits below) */
+ for (u32 i = 0; i < nargs / 2u; ++i) {
+ ApiSValue tmp = g->stack[base + i];
+ g->stack[base + i] = g->stack[base + nargs - 1u - i];
+ g->stack[base + nargs - 1u - i] = tmp;
+ }
+ for (u32 i = 0; i < nargs; ++i) api_pack_call_arg(g, &args[i], fty, i);
+}
+
CGLocal api_alloc_call_result(KitCg* g, KitCgTypeId ret_ty) {
return api_alloc_temp_local(g, ret_ty);
}
@@ -178,10 +207,7 @@ void kit_cg_call(KitCg* g, uint32_t nargs, KitCgTypeId fn_type,
}
args = api_alloc_call_args(g, nargs);
- for (u32 i = 0; i < nargs; ++i) {
- u32 idx = nargs - 1u - i;
- api_pack_call_arg(g, &args[idx], fty, idx);
- }
+ api_pack_call_args_in_order(g, args, nargs, fty);
callee = api_pop(g);
api_ensure_local(g, &callee);
@@ -236,10 +262,7 @@ void api_call_symbol_common(KitCg* g, KitCgSym sym, uint32_t nargs,
return;
}
args = api_alloc_call_args(g, nargs);
- for (u32 i = 0; i < nargs; ++i) {
- u32 idx = nargs - 1u - i;
- api_pack_call_arg(g, &args[idx], fty, idx);
- }
+ api_pack_call_args_in_order(g, args, nargs, fty);
callee_op = api_op_global((ObjSymId)sym, 0, cg_type_ptr_to(g->c, fty));
inline_policy = attrs.inline_policy;
if (inline_policy == KIT_CG_INLINE_DEFAULT) {
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -2160,10 +2160,12 @@ static void nd_call(CgTarget* t, const CGCallDesc* desc) {
nd_invalidate_local(d, desc->args[i]);
}
/* Stable-result-register path: cache the result directly in the ABI result
- * register the call left it in (plan.rets[0].src) and skip its home store —
- * that register is excluded from the general cache pool, so it is free here
- * (the dead args were just invalidated); the next call's flush spills it if
- * still live, or drops it if dead. */
+ * register the call left it in (plan.rets[0].src) and skip its home store.
+ * That register may now be an ordinary cache-pool member (x0 on aa64 is -O0
+ * allocable), but it is free here regardless: the live-across flush + the
+ * dead-arg invalidation above cleared it, and the guard below evicts any
+ * residual owner before claiming it. The next call's flush spills it if still
+ * live, or drops it if dead. */
u32 ret_start = 0;
if (result_in_abi_reg) {
NativeDirectLocal* rl = nd_local(d, desc->result);
