commit d953be125bc6d503b325833cce8d7cafb0e7fb47
parent 3e0e8146073711b21af37610484a9ca06dbeffb6
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 10:10:08 -0700
perf(cg): narrow register-only NativeTarget hooks — slim the -O0 NativeLoc crossing
Diffstat:
5 files changed, 199 insertions(+), 10 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -2571,6 +2571,31 @@ static void aa_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
}
}
+/* §E.3 narrow register-only entry points. The NDT crosses 16 B NativeRegLoc
+ * here; these reconstruct the NativeLoc the fat hook expects and delegate, so
+ * the emitted bytes are identical. Used only on the -O0 NDT path; the opt
+ * replay path calls aa_binop/aa_move/aa_cmp/aa_convert directly. */
+static void aa_binop_rr(NativeTarget* t, BinOp op, NativeRegLoc dst,
+ NativeRegLoc a, NativeRegLoc b) {
+ aa_binop(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
+ native_loc_from_reg(b));
+}
+
+static void aa_move_rr(NativeTarget* t, NativeRegLoc dst, NativeRegLoc src) {
+ aa_move(t, native_loc_from_reg(dst), native_loc_from_reg(src));
+}
+
+static void aa_cmp_rr(NativeTarget* t, CmpOp op, NativeRegLoc dst,
+ NativeRegLoc a, NativeRegLoc b) {
+ aa_cmp(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
+ native_loc_from_reg(b));
+}
+
+static void aa_convert_rr(NativeTarget* t, ConvKind op, NativeRegLoc dst,
+ NativeRegLoc src) {
+ aa_convert(t, op, native_loc_from_reg(dst), native_loc_from_reg(src));
+}
+
static void aa_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
u32 align) {
AANativeTarget* a = aa_of(t);
@@ -4114,6 +4139,10 @@ NativeTarget* aa64_native_target_new(Compiler* c, ObjBuilder* obj,
t->unop = aa_unop;
t->cmp = aa_cmp;
t->convert = aa_convert;
+ t->binop_rr = aa_binop_rr;
+ t->move_rr = aa_move_rr;
+ t->cmp_rr = aa_cmp_rr;
+ t->convert_rr = aa_convert_rr;
t->alloca_ = aa_alloca;
t->spill = aa_spill;
t->reload = aa_reload;
diff --git a/src/arch/native_target.h b/src/arch/native_target.h
@@ -289,6 +289,30 @@ typedef struct NativeLoc {
} v;
} NativeLoc;
+/* Narrow register-only operand descriptor for the -O0 NativeDirectTarget hot
+ * path. NDT resolves every post-materialization binop/move/cmp/convert operand
+ * to a hard register (or, for an arithmetic/compare RHS, a target-legal
+ * immediate), then crosses into the per-arch NativeTarget. The fat 48 B
+ * NativeLoc carries a union wide enough for memory/global/addr forms that these
+ * ops never receive; passing it by value copies 48 B per operand (144 B/op)
+ * only for the arch to unwrap a register number + the cached width. NativeRegLoc
+ * carries exactly the fields the arch reads — register/immediate, class, the
+ * §E.2 cached width descriptor, and the type for the cold width-fallback — in
+ * 16 B. The narrow *_rr hooks below take it; they are byte-identical to the fat
+ * hooks because native_loc_from_reg reconstructs the same NativeLoc the fat path
+ * would have received. */
+typedef struct NativeRegLoc {
+ u8 cls; /* NativeAllocClass for the register/value */
+ u8 szinfo; /* cached scalar-width descriptor, same encoding as NativeLoc */
+ u8 is_imm; /* 1 => v.imm holds a target-legal immediate (RHS only); else reg */
+ u8 pad;
+ KitCgTypeId type;
+ union {
+ Reg reg;
+ i64 imm;
+ } v;
+} NativeRegLoc;
+
typedef struct NativeInst NativeInst;
typedef enum NativePatchKind {
@@ -522,6 +546,19 @@ struct NativeTarget {
NativeLoc b_reg_or_imm);
void (*convert)(NativeTarget*, ConvKind, NativeLoc dst_reg,
NativeLoc src_reg);
+ /* Narrow register-only entry points for the -O0 NativeDirectTarget hot path
+ * (§E.3). Same semantics and byte output as the fat binop/move/cmp/convert,
+ * but the operands cross as 16 B NativeRegLoc instead of 48 B NativeLoc. NDT
+ * calls these when installed (always, for the native arch backends); the
+ * fat hooks above remain for the opt/-O1 replay path, which is unchanged.
+ * Optional: a backend that leaves them NULL keeps the fat path. */
+ void (*binop_rr)(NativeTarget*, BinOp, NativeRegLoc dst, NativeRegLoc a,
+ NativeRegLoc b_reg_or_imm);
+ void (*move_rr)(NativeTarget*, NativeRegLoc dst, NativeRegLoc src);
+ void (*cmp_rr)(NativeTarget*, CmpOp, NativeRegLoc dst, NativeRegLoc a,
+ NativeRegLoc b_reg_or_imm);
+ void (*convert_rr)(NativeTarget*, ConvKind, NativeRegLoc dst,
+ NativeRegLoc src);
void (*alloca_)(NativeTarget*, NativeLoc dst_reg, NativeLoc size_reg,
u32 align);
@@ -630,6 +667,42 @@ static inline int native_loc_is_fp(NativeLoc loc) {
return (NativeAllocClass)loc.cls == NATIVE_REG_FP;
}
+/* Narrow a register-or-immediate NativeLoc to the 16 B NativeRegLoc the §E.3
+ * *_rr hooks take. Only NATIVE_LOC_REG / NATIVE_LOC_IMM forms reach these hooks
+ * (the post-materialization contract), so this carries the register/immediate,
+ * class, the cached width descriptor, and the type for the cold fallback. */
+static inline NativeRegLoc native_reg_loc_of(NativeLoc loc) {
+ NativeRegLoc r;
+ r.cls = loc.cls;
+ r.szinfo = loc.szinfo;
+ r.is_imm = (u8)(loc.kind == NATIVE_LOC_IMM);
+ r.pad = 0;
+ r.type = loc.type;
+ if (loc.kind == NATIVE_LOC_IMM)
+ r.v.imm = loc.v.imm;
+ else
+ r.v.reg = loc.v.reg;
+ return r;
+}
+
+/* Reconstruct the full NativeLoc a fat binop/move/cmp/convert hook would have
+ * received from a NativeRegLoc. Byte-identical by construction: the arch reads
+ * only kind / cls / szinfo / type / v.{reg,imm}, all preserved here, and the
+ * rest of the 48 B union is never consulted on this path. */
+static inline NativeLoc native_loc_from_reg(NativeRegLoc r) {
+ NativeLoc loc;
+ loc.kind = (u8)(r.is_imm ? NATIVE_LOC_IMM : NATIVE_LOC_REG);
+ loc.cls = r.cls;
+ loc.szinfo = r.szinfo;
+ loc.pad[0] = 0;
+ loc.type = r.type;
+ if (r.is_imm)
+ loc.v.imm = r.v.imm;
+ else
+ loc.v.reg = r.v.reg;
+ return loc;
+}
+
/* NativeLoc.szinfo layout (see the field comment on NativeLoc). */
#define NATIVE_SZINFO_VALID 0x01u
#define NATIVE_SZINFO_SIZE_LOG2_SHIFT 1u
diff --git a/src/arch/riscv/native.c b/src/arch/riscv/native.c
@@ -1359,6 +1359,31 @@ static void rv_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
}
}
+/* §E.3 narrow register-only entry points. The NDT crosses 16 B NativeRegLoc
+ * here; these reconstruct the NativeLoc the fat hook expects and delegate, so
+ * the emitted bytes are identical. Used only on the -O0 NDT path; the opt
+ * replay path calls rv_binop/rv_move/rv_cmp/rv_convert directly. */
+static void rv_binop_rr(NativeTarget* t, BinOp op, NativeRegLoc dst,
+ NativeRegLoc a, NativeRegLoc b) {
+ rv_binop(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
+ native_loc_from_reg(b));
+}
+
+static void rv_move_rr(NativeTarget* t, NativeRegLoc dst, NativeRegLoc src) {
+ rv_move(t, native_loc_from_reg(dst), native_loc_from_reg(src));
+}
+
+static void rv_cmp_rr(NativeTarget* t, CmpOp op, NativeRegLoc dst,
+ NativeRegLoc a, NativeRegLoc b) {
+ rv_cmp(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
+ native_loc_from_reg(b));
+}
+
+static void rv_convert_rr(NativeTarget* t, ConvKind op, NativeRegLoc dst,
+ NativeRegLoc src) {
+ rv_convert(t, op, native_loc_from_reg(dst), native_loc_from_reg(src));
+}
+
/* ============================ spill / reload ============================ */
static void rv_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
@@ -4070,6 +4095,10 @@ NativeTarget* rv64_native_target_new(Compiler* c, ObjBuilder* obj,
t->unop = rv_unop;
t->cmp = rv_cmp;
t->convert = rv_convert;
+ t->binop_rr = rv_binop_rr;
+ t->move_rr = rv_move_rr;
+ t->cmp_rr = rv_cmp_rr;
+ t->convert_rr = rv_convert_rr;
t->alloca_ = rv_alloca;
t->spill = rv_spill;
t->reload = rv_reload;
diff --git a/src/arch/x64/native.c b/src/arch/x64/native.c
@@ -1462,6 +1462,31 @@ static void x64_convert(NativeTarget* t, ConvKind k, NativeLoc dst,
}
}
+/* §E.3 narrow register-only entry points. The NDT crosses 16 B NativeRegLoc
+ * here; these reconstruct the NativeLoc the fat hook expects and delegate, so
+ * the emitted bytes are identical. Used only on the -O0 NDT path; the opt
+ * replay path calls x64_binop/x64_move/x64_cmp/x64_convert directly. */
+static void x64_binop_rr(NativeTarget* t, BinOp op, NativeRegLoc dst,
+ NativeRegLoc a, NativeRegLoc b) {
+ x64_binop(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
+ native_loc_from_reg(b));
+}
+
+static void x64_move_rr(NativeTarget* t, NativeRegLoc dst, NativeRegLoc src) {
+ x64_move(t, native_loc_from_reg(dst), native_loc_from_reg(src));
+}
+
+static void x64_cmp_rr(NativeTarget* t, CmpOp op, NativeRegLoc dst,
+ NativeRegLoc a, NativeRegLoc b) {
+ x64_cmp(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
+ native_loc_from_reg(b));
+}
+
+static void x64_convert_rr(NativeTarget* t, ConvKind op, NativeRegLoc dst,
+ NativeRegLoc src) {
+ x64_convert(t, op, native_loc_from_reg(dst), native_loc_from_reg(src));
+}
+
/* ============================ spill / reload ============================ */
static void x64_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
@@ -4334,6 +4359,10 @@ NativeTarget* x64_native_target_new(Compiler* c, ObjBuilder* obj,
t->unop = x64_unop;
t->cmp = x64_cmp;
t->convert = x64_convert;
+ t->binop_rr = x64_binop_rr;
+ t->move_rr = x64_move_rr;
+ t->cmp_rr = x64_cmp_rr;
+ t->convert_rr = x64_convert_rr;
t->alloca_ = x64_alloca;
t->spill = x64_spill;
t->reload = x64_reload;
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -846,8 +846,13 @@ static void nd_copy_to_reg(NativeDirectTarget* d, NativeLoc dst,
switch ((NativeLocKind)src.kind) {
case NATIVE_LOC_REG:
if (src.v.reg != dst.v.reg || src.cls != dst.cls) {
- ND_REQUIRE_NATIVE(d, move, "target does not emit register moves");
- d->native->move(d->native, dst, src);
+ if (d->native->move_rr)
+ d->native->move_rr(d->native, native_reg_loc_of(dst),
+ native_reg_loc_of(src));
+ else {
+ ND_REQUIRE_NATIVE(d, move, "target does not emit register moves");
+ d->native->move(d->native, dst, src);
+ }
}
break;
case NATIVE_LOC_FRAME:
@@ -1771,8 +1776,17 @@ static void nd_binop(CgTarget* t, BinOp op, Operand dst, Operand a, Operand b) {
ar = nd_materialize_operand(d, a);
br = nd_rhs_imm_or_reg(d, NATIVE_IMM_BINOP, (u32)op, b);
dr = nd_dst_reg(d, dst);
- ND_REQUIRE_NATIVE(d, binop, "target does not emit binary ops");
- d->native->binop(d->native, op, dr, ar, br);
+ /* §E.3: cross the post-materialization operands as 16 B NativeRegLoc when the
+ * backend installs the narrow hook (every native arch does); fall back to the
+ * fat 48 B hook otherwise. Byte-identical — the arch reconstructs the same
+ * NativeLoc. */
+ if (d->native->binop_rr)
+ d->native->binop_rr(d->native, op, native_reg_loc_of(dr),
+ native_reg_loc_of(ar), native_reg_loc_of(br));
+ else {
+ ND_REQUIRE_NATIVE(d, binop, "target does not emit binary ops");
+ d->native->binop(d->native, op, dr, ar, br);
+ }
nd_dst_writeback(d, dst, dr);
nd_release_materialized(d, br);
nd_release_materialized(d, ar);
@@ -1823,8 +1837,13 @@ static void nd_cmp(CgTarget* t, CmpOp op, Operand dst, Operand a, Operand b) {
ar = nd_materialize_operand(d, a);
br = nd_rhs_imm_or_reg(d, NATIVE_IMM_CMP, (u32)op, b);
dr = nd_dst_reg(d, dst);
- ND_REQUIRE_NATIVE(d, cmp, "target does not emit compares");
- d->native->cmp(d->native, op, dr, ar, br);
+ if (d->native->cmp_rr)
+ d->native->cmp_rr(d->native, op, native_reg_loc_of(dr),
+ native_reg_loc_of(ar), native_reg_loc_of(br));
+ else {
+ ND_REQUIRE_NATIVE(d, cmp, "target does not emit compares");
+ d->native->cmp(d->native, op, dr, ar, br);
+ }
nd_dst_writeback(d, dst, dr);
nd_release_materialized(d, br);
nd_release_materialized(d, ar);
@@ -1861,16 +1880,26 @@ static void nd_convert(CgTarget* t, ConvKind op, Operand dst, Operand src) {
NativeAllocClass cls = nd_class_for_type(d, dst.type);
sr = nd_loc_reg(d, src.type, cls, r);
dr = nd_loc_reg(d, dst.type, cls, r);
- ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
- d->native->convert(d->native, op, dr, sr);
+ if (d->native->convert_rr)
+ d->native->convert_rr(d->native, op, native_reg_loc_of(dr),
+ native_reg_loc_of(sr));
+ else {
+ ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
+ d->native->convert(d->native, op, dr, sr);
+ }
nd_dst_writeback(d, dst, dr);
return;
}
}
sr = nd_materialize_operand(d, src);
dr = nd_dst_reg(d, dst);
- ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
- d->native->convert(d->native, op, dr, sr);
+ if (d->native->convert_rr)
+ d->native->convert_rr(d->native, op, native_reg_loc_of(dr),
+ native_reg_loc_of(sr));
+ else {
+ ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
+ d->native->convert(d->native, op, dr, sr);
+ }
nd_dst_writeback(d, dst, dr);
nd_release_materialized(d, sr);
}
