commit b4b5067ac11a2340ed56164c3e0fa60ec5a55651
parent f409e1485107d1a9127abf984f5df548417b68b3
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 13:26:10 -0700
fix(cg): gate -O0 result-in-result-register caching to safe arches (L4 Phase 1)
L4 Phase 1 cached a scalar call result in the ABI result register on every
target. That is correct only where the result register is clobbered solely at
call boundaries (which flush the cache). On x86-64 RAX is an implicit operand of
div/mul, so a result left in RAX was silently clobbered by a following division
before its consumer — e.g. `int r=f(); return r + a/b;` returned garbage.
Add a NativeRegInfo capability ndt_result_reg_stable; only when set does nd_call
keep the result in the ABI result register. aarch64 sets it (x0; div/mul take
explicit operands). x86-64 and riscv64 leave it clear and keep the prior path:
pre-claim a general cache register and move the ABI result into it after the
call. (riscv64's a0 is in fact call-only-clobbered too; left off pending an
on-target check.)
aarch64 sqlite -O0 unchanged (1.1730x tcc, the P1+P2 win retained); x86-64
result-after-divide now matches clang. Gate: full suite green + new p1-x64
clang-differential probe (call result survives a RAX-clobbering divide).
Diffstat:
3 files changed, 64 insertions(+), 31 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -4225,6 +4225,9 @@ static const NativeRegInfo aa_reg_info = {
* needs a callee-saved register — enabling the tcc-style deferred-`sub`
* prologue (see aa_func_begin / AA_NDT_SUB_WORDS). */
.ndt_caller_saved_only = 1u,
+ /* x0 is clobbered only by calls (div/mul take explicit operands), so a
+ * scalar call result can stay cached in it — see L4 Phase 1 in nd_call. */
+ .ndt_result_reg_stable = 1u,
.resolve_name = aa_resolve_name,
.asm_operand_reg_ok = aa_asm_operand_reg_ok,
.asm_constraint_reg = aa_asm_constraint_reg,
diff --git a/src/arch/native_target.h b/src/arch/native_target.h
@@ -174,6 +174,15 @@ struct NativeRegInfo {
* has full callee-save freedom. */
u8 ndt_caller_saved_only;
+ /* True when the ABI scalar-result register is clobbered ONLY at call
+ * boundaries (where the -O0 cache already flushes), so a scalar call result
+ * can be left cached in it across following ops instead of being moved into a
+ * general cache register (saving the post-call `mov cachereg, retreg`). Set on
+ * aarch64 (x0) and riscv64 (a0), whose div/mul/etc. take explicit operands.
+ * Clear on x86-64, where RAX is an implicit div/mul operand and a result left
+ * in it would be silently clobbered before its consumer. */
+ u8 ndt_result_reg_stable;
+
/* Map a register name to its (Reg, class). `name` is the raw spelling
* ("rax", "x8", "a7"); the caller resolves any Sym to its bytes first so this
* stays pool-free. Returns 0 on success, non-zero for a non-register name. */
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -1964,17 +1964,37 @@ static void nd_call(CgTarget* t, const CGCallDesc* desc) {
if (desc->callee.kind == OPK_LOCAL) nd_flush_local(d, desc->callee.v.local);
/* Spill the live-across set: everything cached except the kept (dead) args. */
nd_flush_all_except_kept_args(d, desc);
- /* Place the scalar result. A cacheable result is cached in the ABI result
- * register the call leaves it in — done after the call below, with no mov and
- * no home store. Feed the plan a placeholder register dst here (never applied;
- * the post-call step always skips rets[0] for cacheable results) so no home
- * slot is allocated for a result consumed before the next flush. Non-cacheable
- * results (aggregate / multi-part / sret) keep the home path. */
+ /* Place the scalar result. With ndt_result_reg_stable (aa64/rv64) the result
+ * is cached directly in the ABI result register after the call — no mov, no
+ * home store; feed the plan a placeholder dst (never applied, the post-call
+ * step always skips rets[0]) so no home slot is allocated for a result
+ * consumed before the next flush. Without it (x86-64: RAX is an implicit
+ * div/mul operand, so a result left there is clobbered before its consumer)
+ * claim a general cache register now and let the post-call move write the
+ * result into it. A non-cacheable result (aggregate / sret) keeps the home. */
+ int result_in_abi_reg = 0;
if (nresults) {
NativeDirectLocal* rl = nd_local(d, desc->result);
- results[0] = nd_local_cacheable(d, rl)
- ? nd_loc_reg(d, rl->type, (NativeAllocClass)rl->cls, 0)
- : nd_loc_frame(d, desc->result, 0);
+ if (!nd_local_cacheable(d, rl)) {
+ results[0] = nd_loc_frame(d, desc->result, 0);
+ } else if (d->reg_info && d->reg_info->ndt_result_reg_stable) {
+ result_in_abi_reg = 1;
+ results[0] = nd_loc_reg(d, rl->type, (NativeAllocClass)rl->cls, 0);
+ } else {
+ Reg r = nd_cache_alloc(d, (NativeAllocClass)rl->cls);
+ if (r != REG_NONE) {
+ d->reg_owner[rl->cls][r] = desc->result;
+ rl->reg = r;
+ rl->dirty = 1;
+ nd_cache_link(d, desc->result);
+ /* Pin until the post-call move so the callee materialize cannot evict
+ * this not-yet-valid entry as a scratch victim. */
+ d->scratch_used[rl->cls] |= 1u << r;
+ results[0] = nd_loc_reg(d, rl->type, (NativeAllocClass)rl->cls, r);
+ } else {
+ results[0] = nd_loc_frame(d, desc->result, 0);
+ }
+ }
}
nd_barrier(d, NATIVE_DIRECT_BARRIER_CALL | NATIVE_DIRECT_BARRIER_MEMORY);
nd.fn_type = desc->fn_type;
@@ -2017,33 +2037,34 @@ static void nd_call(CgTarget* t, const CGCallDesc* desc) {
d->scratch_used[args[i].cls] &= ~(1u << args[i].v.reg);
nd_invalidate_local(d, desc->args[i]);
}
- /* Cache a cacheable scalar result directly in the ABI result register the call
- * left it in (x0 / v0 — reg 0 of its class on aa64, a0 on rv64, etc.), instead
- * of moving it to a general cache register or storing it home. That register
- * is excluded from the general cache pool, so it is free here (the dead args
- * were just invalidated); the next call's flush spills it if it is still live,
- * or it is dropped if it turns out dead. The home store for rets[0] is then
- * skipped — the value lives in its cache register. */
+ /* Stable-result-register path: cache the result directly in the ABI result
+ * register the call left it in (plan.rets[0].src) and skip its home store —
+ * that register is excluded from the general cache pool, so it is free here
+ * (the dead args were just invalidated); the next call's flush spills it if
+ * still live, or drops it if dead. */
u32 ret_start = 0;
- if (nresults) {
+ if (result_in_abi_reg) {
NativeDirectLocal* rl = nd_local(d, desc->result);
- if (nd_local_cacheable(d, rl)) {
- NativeLoc src = plan.rets[0].src;
- if (plan.nrets != 1u || src.kind != NATIVE_LOC_REG ||
- (NativeAllocClass)src.cls != (NativeAllocClass)rl->cls)
- nd_panic(d, "cacheable scalar result not returned in one register");
- if (d->reg_owner[rl->cls][src.v.reg] != CG_LOCAL_NONE)
- nd_invalidate_local(d, d->reg_owner[rl->cls][src.v.reg]);
- d->reg_owner[rl->cls][src.v.reg] = desc->result;
- rl->reg = src.v.reg;
- rl->dirty = 1;
- nd_cache_link(d, desc->result);
- nd_touch_local(d, rl);
- ret_start = 1u;
- }
+ NativeLoc src = plan.rets[0].src;
+ if (plan.nrets != 1u || src.kind != NATIVE_LOC_REG ||
+ (NativeAllocClass)src.cls != (NativeAllocClass)rl->cls)
+ nd_panic(d, "cacheable scalar result not returned in one register");
+ if (d->reg_owner[rl->cls][src.v.reg] != CG_LOCAL_NONE)
+ nd_invalidate_local(d, d->reg_owner[rl->cls][src.v.reg]);
+ d->reg_owner[rl->cls][src.v.reg] = desc->result;
+ rl->reg = src.v.reg;
+ rl->dirty = 1;
+ nd_cache_link(d, desc->result);
+ nd_touch_local(d, rl);
+ ret_start = 1u;
}
for (u32 i = ret_start; i < plan.nrets; ++i)
nd_write_loc(d, plan.rets[i].dst, plan.rets[i].src, plan.rets[i].mem);
+ /* Non-stable path: the result was pre-claimed in a general cache register and
+ * the move above wrote the ABI reg into it; unpin it so it behaves as an
+ * ordinary write-back entry. */
+ if (nresults && !result_in_abi_reg && results[0].kind == NATIVE_LOC_REG)
+ d->scratch_used[results[0].cls] &= ~(1u << results[0].v.reg);
if (release_callee_tmp)
nd_scratch_release(d, (NativeAllocClass)callee_tmp.cls, callee_tmp.v.reg);
}
