commit bcdbeba368da56abbac093222d97d19298491466
parent f616cb096a6817069948ce912baf045bfd0b42ef
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 09:45:00 -0700
perf(cg): cache scalar width on the NativeLoc pad — kill arch type re-query
The -O0 NDT arch backends re-derived a scalar's width on every binop/move/
convert/cmp by calling cg_type_size (and cg_type_is_ptr) on loc.type — a
type-bridge crossing the NDT had already paid when it built the NativeLoc.
Stamp the width once at the single register-NativeLoc producer. NativeLoc's free
pad now carries a szinfo byte {VALID, size_log2}; nd_loc_reg (a thin wrapper
over native_loc_reg used at all 14 NDT register-loc sites) fills it via
native_loc_stamp_size, computing cg_type_size exactly once. The arch hot path
reads the byte through new loc_size32()/loc_is_64() helpers (aarch64, riscv64,
x86-64) instead of re-querying; both fall back to the live type query when the
loc is unstamped (immediates, memory, cold producers), so partial adoption stays
byte-identical.
Only the byte SIZE is cached, never a baked is64 class: each arch derives its
own is64 from the cached size plus its pointer/xlen rule, preserving the rv32
4-byte-pointer and i128 distinctions. Only exact power-of-two scalar widths
(1/2/4/8/16) are stamped, so the cached value is bit-for-bit what cg_type_size
would return.
Byte-identical: perf identity gate 60/60 PASS (objects -O0/-O1, -g DWARF, -S
asm, linked exe) + cross-arch byte-identity diff over aarch64/riscv64/x86-64
(linux+macos) and riscv32, general + atomic/fp/conversion-heavy sources, at
-O0/-O1/-g.
Diffstat:
5 files changed, 205 insertions(+), 101 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -370,8 +370,21 @@ static u32 size_idx(u32 n) {
static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0x1fu; }
+/* Scalar byte width of a register operand. Reads the NDT-stamped szinfo
+ * descriptor (one byte) when present, else falls back to the live type query —
+ * so partial adoption stays byte-identical. */
+static u32 loc_size32(NativeTarget* t, NativeLoc loc) {
+ if (loc.szinfo & NATIVE_SZINFO_VALID) return native_szinfo_size(loc.szinfo);
+ return type_size32(t, loc.type);
+}
+
+/* The original predicate exactly, but sourcing the width from the cached
+ * descriptor when stamped: `size == 8 || is_ptr`. is_ptr is short-circuited
+ * away whenever the cached size is already 8 (the pointer case on this arch),
+ * so the common path becomes a single byte test. Byte-identical to the old
+ * type_size32+cg_type_is_ptr form. */
static int loc_is_64(NativeTarget* t, NativeLoc loc) {
- return type_size32(t, loc.type) == 8u || cg_type_is_ptr(t->c, loc.type);
+ return loc_size32(t, loc) == 8u || cg_type_is_ptr(t->c, loc.type);
}
/* native_loc_is_fp is shared in native_target.h. */
@@ -1943,10 +1956,10 @@ static void aa_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
native_loc_is_fp(dst) == native_loc_is_fp(src) && dst.v.reg == src.v.reg)
return;
if (native_loc_is_fp(dst) && native_loc_is_fp(src)) {
- if (type_size32(t, dst.type) == 16u)
+ if (loc_size32(t, dst) == 16u)
aa_emit32(t->mc, aa_mov_vec16(loc_reg(dst), loc_reg(src)));
else
- aa_emit32(t->mc, aa_fmov_fp(type_size32(t, dst.type) == 8u, loc_reg(dst),
+ aa_emit32(t->mc, aa_fmov_fp(loc_size32(t, dst) == 8u, loc_reg(dst),
loc_reg(src)));
} else if (native_loc_is_fp(dst)) {
aa_emit32(t->mc,
@@ -2301,7 +2314,7 @@ static void aa_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc lhs,
u32 sf = loc_is_64(t, dst) ? 1u : 0u;
u32 rd = loc_reg(dst), rn = loc_reg(lhs), rm = loc_reg(rhs);
if (native_loc_is_fp(dst)) {
- u32 d = type_size32(t, dst.type) == 8u;
+ u32 d = loc_size32(t, dst) == 8u;
switch (op) {
case BO_FADD:
aa_emit32(t->mc, aa_fp_bin(0x002800u, d, rd, rn, rm));
@@ -2416,7 +2429,7 @@ static void aa_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
switch (op) {
case UO_FNEG:
case UO_NEG:
- aa_emit32(t->mc, aa_fneg(type_size32(t, dst.type) == 8u, loc_reg(dst),
+ aa_emit32(t->mc, aa_fneg(loc_size32(t, dst) == 8u, loc_reg(dst),
loc_reg(src)));
return;
default:
@@ -2442,7 +2455,7 @@ static void aa_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
static void aa_emit_cmp_to_flags(NativeTarget* t, NativeLoc lhs,
NativeLoc rhs) {
if (native_loc_is_fp(lhs)) {
- aa_emit32(t->mc, aa_fcmp(type_size32(t, lhs.type) == 8u, loc_reg(lhs),
+ aa_emit32(t->mc, aa_fcmp(loc_size32(t, lhs) == 8u, loc_reg(lhs),
loc_reg(rhs)));
return;
}
@@ -2494,8 +2507,8 @@ static void aa_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
aa_move(t, dst, src);
return;
case CV_ZEXT: {
- u32 src_bits = type_size32(t, src.type) * 8u;
- u32 dst_bits = type_size32(t, dst.type) * 8u;
+ u32 src_bits = loc_size32(t, src) * 8u;
+ u32 dst_bits = loc_size32(t, dst) * 8u;
u32 sf = dst_bits > 32u;
if (src_bits >= dst_bits) {
aa_move(t, dst, src);
@@ -2514,8 +2527,8 @@ static void aa_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
return;
}
case CV_SEXT: {
- u32 src_bits = type_size32(t, src.type) * 8u;
- u32 dst_bits = type_size32(t, dst.type) * 8u;
+ u32 src_bits = loc_size32(t, src) * 8u;
+ u32 dst_bits = loc_size32(t, dst) * 8u;
u32 sf = dst_bits > 32u;
if (src_bits >= dst_bits) {
aa_move(t, dst, src);
@@ -2526,22 +2539,20 @@ static void aa_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
return;
}
case CV_ITOF_S:
- aa_emit32(t->mc, aa_scvtf(type_size32(t, dst.type) == 8u,
- loc_is_64(t, src), loc_reg(dst), loc_reg(src)));
+ aa_emit32(t->mc, aa_scvtf(loc_size32(t, dst) == 8u, loc_is_64(t, src),
+ loc_reg(dst), loc_reg(src)));
return;
case CV_ITOF_U:
- aa_emit32(t->mc, aa_ucvtf(type_size32(t, dst.type) == 8u,
- loc_is_64(t, src), loc_reg(dst), loc_reg(src)));
+ aa_emit32(t->mc, aa_ucvtf(loc_size32(t, dst) == 8u, loc_is_64(t, src),
+ loc_reg(dst), loc_reg(src)));
return;
case CV_FTOI_S:
- aa_emit32(t->mc,
- aa_fcvtzs(loc_is_64(t, dst), type_size32(t, src.type) == 8u,
- loc_reg(dst), loc_reg(src)));
+ aa_emit32(t->mc, aa_fcvtzs(loc_is_64(t, dst), loc_size32(t, src) == 8u,
+ loc_reg(dst), loc_reg(src)));
return;
case CV_FTOI_U:
- aa_emit32(t->mc,
- aa_fcvtzu(loc_is_64(t, dst), type_size32(t, src.type) == 8u,
- loc_reg(dst), loc_reg(src)));
+ aa_emit32(t->mc, aa_fcvtzu(loc_is_64(t, dst), loc_size32(t, src) == 8u,
+ loc_reg(dst), loc_reg(src)));
return;
case CV_FEXT:
if (dst_fp && src_fp)
@@ -3350,7 +3361,7 @@ static void aa_saved_tmp_restore(AANativeTarget* a, u32 reg) {
static void aa_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
MemAccess mem, KitCgMemOrder order) {
u32 base = AA_TMP0;
- u32 sz = size_idx(mem.size ? mem.size : type_size32(t, dst.type));
+ u32 sz = size_idx(mem.size ? mem.size : loc_size32(t, dst));
aa_atomic_addr_reg(t, addr, base);
aa_emit32(t->mc, aa_order_acquire(order)
? aa_ldar(sz, loc_reg(dst), base)
@@ -3362,7 +3373,7 @@ static void aa_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
static void aa_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
MemAccess mem, KitCgMemOrder order) {
u32 base = AA_TMP0;
- u32 sz = size_idx(mem.size ? mem.size : type_size32(t, src.type));
+ u32 sz = size_idx(mem.size ? mem.size : loc_size32(t, src));
if (order == KIT_CG_MO_SEQ_CST)
aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
aa_atomic_addr_reg(t, addr, base);
@@ -3382,7 +3393,7 @@ static void aa_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
u32 status = aa_saved_tmp_pick(loc_reg(dst), loc_reg(val), base);
NativeLoc next = aa_tmp_loc(dst.type, next_reg);
MCLabel retry = mc_label_new(t->mc);
- u32 sz = size_idx(mem.size ? mem.size : type_size32(t, dst.type));
+ u32 sz = size_idx(mem.size ? mem.size : loc_size32(t, dst));
if (order == KIT_CG_MO_SEQ_CST)
aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
aa_saved_tmp_spill(a, status);
@@ -3432,7 +3443,7 @@ static void aa_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
KitCgMemOrder success, KitCgMemOrder failure) {
u32 base = AA_TMP0;
u32 status = AA_TMP1;
- u32 sz = size_idx(mem.size ? mem.size : type_size32(t, prior.type));
+ u32 sz = size_idx(mem.size ? mem.size : loc_size32(t, prior));
u32 sf = sz == 3u;
int acquire = aa_order_acquire(success) || aa_order_acquire(failure);
int release = aa_order_release(success);
@@ -4929,7 +4940,7 @@ static void aa_asm_load_loc_to_reg(AANativeTarget* a, SrcLoc loc, NativeLoc src,
return;
}
aa_emit_mem(a, 1, dst, aa_asm_loc_to_addr(a, loc, src),
- aa_mem_for_type(t, dst.type, type_size32(t, dst.type)));
+ aa_mem_for_type(t, dst.type, loc_size32(t, dst)));
}
static void aa_asm_store_reg_to_loc(AANativeTarget* a, SrcLoc loc,
@@ -4940,7 +4951,7 @@ static void aa_asm_store_reg_to_loc(AANativeTarget* a, SrcLoc loc,
return;
}
aa_emit_mem(a, 0, src, aa_asm_loc_to_addr(a, loc, dst),
- aa_mem_for_type(t, src.type, type_size32(t, src.type)));
+ aa_mem_for_type(t, src.type, loc_size32(t, src)));
}
static void aa_asm_bind_native(AANativeTarget* a, SrcLoc loc, Operand* out,
diff --git a/src/arch/native_target.h b/src/arch/native_target.h
@@ -258,7 +258,20 @@ typedef struct NativeAddr {
typedef struct NativeLoc {
u8 kind; /* NativeLocKind */
u8 cls; /* NativeAllocClass for register-like locations */
- u8 pad[2];
+ /* Cached scalar-width descriptor, stamped once by the NDT at the
+ * register-NativeLoc choke point (nd_loc_reg) so the per-arch binop/move/
+ * convert/cmp hot path reads a byte instead of re-crossing the type bridge
+ * via cg_type_size on `type`. Layout:
+ * bit0 SZINFO_VALID — set => the size field is authoritative
+ * bits 1..3 SZINFO_SIZE_LOG2 — log2(byte_size) clamped: 0=1,1=2,2=4,3=8,4=16
+ * Each arch derives its own `is64` predicate from this cached size plus its
+ * pointer/xlen rule (so the rv32 4-byte-pointer and i128 distinctions stay
+ * correct per-arch) — the descriptor caches only the size, never a baked
+ * width class. Left 0 (invalid) for cold locs (immediates, memory, unstamped
+ * producers); the arch then falls back to the live cg_type_size query, so
+ * partial adoption stays byte-identical. */
+ u8 szinfo;
+ u8 pad[1];
KitCgTypeId type;
union {
Reg reg;
@@ -617,6 +630,48 @@ static inline int native_loc_is_fp(NativeLoc loc) {
return (NativeAllocClass)loc.cls == NATIVE_REG_FP;
}
+/* NativeLoc.szinfo layout (see the field comment on NativeLoc). */
+#define NATIVE_SZINFO_VALID 0x01u
+#define NATIVE_SZINFO_SIZE_LOG2_SHIFT 1u
+#define NATIVE_SZINFO_SIZE_LOG2_MASK 0x07u
+
+/* size (bytes, 1..16) -> log2 bucket (0=1,1=2,2=4,3=8,4=16). */
+static inline u8 native_szinfo_log2(u32 size) {
+ if (size <= 1u) return 0u;
+ if (size <= 2u) return 1u;
+ if (size <= 4u) return 2u;
+ if (size <= 8u) return 3u;
+ return 4u;
+}
+
+/* Stamp a register NativeLoc's cached width descriptor from its type, computing
+ * cg_type_size exactly once. Called by the NDT at its register-loc choke point;
+ * the arch backends then read loc.szinfo instead of re-querying. Only meaningful
+ * for scalar (<=16B) reg locs; left invalid for a 0 type or an over-wide size so
+ * the arch's own (panicking) query path runs. The mapped size is byte-identical
+ * to native_type_size / aarch64's type_size32 for in-range scalars: a 0 type
+ * maps to 8, a 0 size to 8 (the register-sized default both helpers apply). */
+static inline void native_loc_stamp_size(NativeTarget* t, NativeLoc* loc) {
+ u64 n = loc->type ? cg_type_size(t->c, loc->type) : 8u;
+ if (n == 0) n = 8u;
+ /* Cache only exact-power-of-two scalar widths (1/2/4/8/16): the szinfo bucket
+ * recovers the size exactly, so the cached value is bit-for-bit what
+ * cg_type_size / native_type_size would return. Anything else (over-wide, or a
+ * non-power-of-two aggregate that should never reach a register loc) is left
+ * invalid and falls back to the live query. */
+ if (n != 1u && n != 2u && n != 4u && n != 8u && n != 16u) return;
+ loc->szinfo = (u8)(NATIVE_SZINFO_VALID | ((u32)native_szinfo_log2((u32)n)
+ << NATIVE_SZINFO_SIZE_LOG2_SHIFT));
+}
+
+/* Byte size (1/2/4/8/16) recovered from a stamped szinfo descriptor. Only valid
+ * when (szinfo & NATIVE_SZINFO_VALID). */
+static inline u32 native_szinfo_size(u8 szinfo) {
+ u32 lg =
+ ((u32)szinfo >> NATIVE_SZINFO_SIZE_LOG2_SHIFT) & NATIVE_SZINFO_SIZE_LOG2_MASK;
+ return 1u << lg;
+}
+
/* Scalar size/align, clamped to a usable register-sized default. Shared by the
* backends whose scalars are at most pointer-width (x64, rv64); aa64 keeps its
* own size query because it asserts on over-wide scalars. */
diff --git a/src/arch/riscv/native.c b/src/arch/riscv/native.c
@@ -361,6 +361,24 @@ static int rv_is_64(NativeTarget* t, KitCgTypeId type) {
(v->xlen == 64u && cg_type_is_ptr(t->c, type));
}
+/* Scalar byte width of a register operand. Reads the NDT-stamped szinfo
+ * descriptor (one byte) when present, else falls back to the live type query —
+ * byte-identical to native_type_size for in-range scalars. */
+static u32 loc_size32(NativeTarget* t, NativeLoc loc) {
+ if (loc.szinfo & NATIVE_SZINFO_VALID) return native_szinfo_size(loc.szinfo);
+ return native_type_size(t, loc.type);
+}
+
+/* The original rv_is_64 predicate exactly, but sourcing the width from the
+ * cached descriptor: `size >= 8 || (xlen==64 && is_ptr)`. The is_ptr clause
+ * keeps the rv32 4-byte-pointer distinction (a stamped 4-byte pointer caches
+ * size 4, so >= 8 is false and the xlen guard decides). Byte-identical. */
+static int loc_is_64(NativeTarget* t, NativeLoc loc) {
+ const RiscvVariant* v = rv_of(t)->variant;
+ return loc_size32(t, loc) >= 8u ||
+ (v->xlen == 64u && cg_type_is_ptr(t->c, loc.type));
+}
+
static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0x1fu; }
/* ============================ register tables ============================ */
@@ -794,7 +812,7 @@ static void rv_emit_mem(RvNativeTarget* a, int is_load, NativeLoc reg,
MCEmitter* mc = t->mc;
u32 r = loc_reg(reg);
int fp = native_loc_is_fp(reg);
- u32 sz = mem.size ? mem.size : native_type_size(t, reg.type);
+ u32 sz = mem.size ? mem.size : loc_size32(t, reg);
u32 base;
i32 off;
@@ -816,18 +834,18 @@ static void rv_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
int dfp = native_loc_is_fp(dst), sfp = native_loc_is_fp(src);
u32 rd = loc_reg(dst), rs = loc_reg(src);
if (dfp && sfp) {
- u32 fmt = native_type_size(t, dst.type) == 8u ? RV_FMT_D : RV_FMT_S;
+ u32 fmt = loc_size32(t, dst) == 8u ? RV_FMT_D : RV_FMT_S;
if (rd == rs) return;
rv64_emit32(mc, rv_fsgnj(fmt, rd, rs, rs));
return;
}
if (!dfp && sfp) {
- u32 sz = native_type_size(t, src.type);
+ u32 sz = loc_size32(t, src);
rv64_emit32(mc, sz == 8u ? rv_fmv_x_d(rd, rs) : rv_fmv_x_w(rd, rs));
return;
}
if (dfp && !sfp) {
- u32 sz = native_type_size(t, dst.type);
+ u32 sz = loc_size32(t, dst);
rv64_emit32(mc, sz == 8u ? rv_fmv_d_x(rd, rs) : rv_fmv_w_x(rd, rs));
return;
}
@@ -836,7 +854,7 @@ static void rv_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
}
static void rv_load_imm(NativeTarget* t, NativeLoc dst, i64 imm) {
- rv_emit_load_imm(rv_of(t)->variant, t->mc, rv_is_64(t, dst.type) ? 1u : 0u,
+ rv_emit_load_imm(rv_of(t)->variant, t->mc, loc_is_64(t, dst) ? 1u : 0u,
loc_reg(dst), imm);
}
@@ -971,7 +989,7 @@ static void rv_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc aop,
MCEmitter* mc = t->mc;
u32 rd = loc_reg(dst);
u32 ra = loc_reg(aop);
- int sf = rv_is_64(t, dst.type);
+ int sf = loc_is_64(t, dst);
/* The W-form ops (ADDW/SUBW/MULW/SLLW/...) are RV64-only and act on a 32-bit
* value held in a 64-bit register. They are emitted only for a narrow value
* on rv64; on rv32 the BASE ops ARE the 32-bit ops, so `w` is always 0 and we
@@ -989,7 +1007,7 @@ static void rv_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc aop,
case BO_FSUB:
case BO_FMUL:
case BO_FDIV: {
- u32 fmt = native_type_size(t, dst.type) == 8u ? RV_FMT_D : RV_FMT_S;
+ u32 fmt = loc_size32(t, dst) == 8u ? RV_FMT_D : RV_FMT_S;
switch (op) {
case BO_FADD:
rv64_emit32(mc, rv_fadd(fmt, rd, ra, rb));
@@ -1076,14 +1094,14 @@ static void rv_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
const RiscvVariant* v = rv_of(t)->variant;
MCEmitter* mc = t->mc;
u32 rd = loc_reg(dst), rs = loc_reg(src);
- int sf = rv_is_64(t, dst.type);
+ int sf = loc_is_64(t, dst);
int w = !sf && v->has_w_forms; /* SUBW is RV64-only; base SUB on rv32 */
switch (op) {
case UO_NEG:
rv64_emit32(mc, w ? rv_subw(rd, RV_ZERO, rs) : rv_sub(rd, RV_ZERO, rs));
return;
case UO_FNEG: {
- u32 fmt = native_type_size(t, dst.type) == 8u ? RV_FMT_D : RV_FMT_S;
+ u32 fmt = loc_size32(t, dst) == 8u ? RV_FMT_D : RV_FMT_S;
rv64_emit32(mc, rv_fsgnjn(fmt, rd, rs, rs));
return;
}
@@ -1107,7 +1125,7 @@ static u32 rv_cmp_ext(NativeTarget* t, int is_signed, NativeLoc op, u32 tmp) {
/* On rv32 a 32-bit operand already fills the whole register — there is no
* wider container to canonicalize into, so the extension is a no-op. */
if (v->xlen == 32u) return r;
- if (rv_is_64(t, op.type)) return r;
+ if (loc_is_64(t, op)) return r;
if (is_signed) {
rv64_emit32(mc, rv_addiw(tmp, r, 0)); /* sign-extend low 32 */
} else {
@@ -1197,7 +1215,7 @@ static void rv_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc aop,
* primitive and OR the two strict relations (a<b | a>b) via scratch RV_TMP2
* (x7, reserved & never allocable, so it can't alias rd). */
if (op >= CMP_OEQ_F) {
- u32 fmt = native_type_size(t, aop.type) == 8u ? RV_FMT_D : RV_FMT_S;
+ u32 fmt = loc_size32(t, aop) == 8u ? RV_FMT_D : RV_FMT_S;
u32 ra = loc_reg(aop), rb = loc_reg(bop);
switch (op) {
case CMP_OEQ_F:
@@ -1263,8 +1281,8 @@ static void rv_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
const RiscvVariant* v = rv_of(t)->variant;
MCEmitter* mc = t->mc;
u32 rd = loc_reg(dst), rs = loc_reg(src);
- u32 src_sz = native_type_size(t, src.type);
- u32 dst_sz = native_type_size(t, dst.type);
+ u32 src_sz = loc_size32(t, src);
+ u32 dst_sz = loc_size32(t, dst);
/* `il` (int-side wide): the 64-bit-integer fcvt L-forms are RV64-only; on
* rv32 only the w/wu forms exist and a 64-bit int<->fp is legalized to a
* libcall before reaching here. */
@@ -1296,7 +1314,7 @@ static void rv_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
rv64_emit32(mc, rv_addi(rd, rs, 0)); /* low bits; users re-narrow */
return;
case CV_ITOF_S:
- if (native_type_size(t, dst.type) == 8u)
+ if (dst_sz == 8u)
rv64_emit32(
mc, il && src_sz == 8u ? rv_fcvt_d_l(rd, rs) : rv_fcvt_d_w(rd, rs));
else
@@ -1304,7 +1322,7 @@ static void rv_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
mc, il && src_sz == 8u ? rv_fcvt_s_l(rd, rs) : rv_fcvt_s_w(rd, rs));
return;
case CV_ITOF_U:
- if (native_type_size(t, dst.type) == 8u)
+ if (dst_sz == 8u)
rv64_emit32(mc, il && src_sz == 8u ? rv_fcvt_d_lu(rd, rs)
: rv_fcvt_d_wu(rd, rs));
else
@@ -2213,7 +2231,7 @@ static void rv_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
return;
}
if (src.kind == NATIVE_LOC_IMM) {
- rv_emit_load_imm(a->variant, t->mc, rv_is_64(t, dst.type) ? 1u : 0u,
+ rv_emit_load_imm(a->variant, t->mc, loc_is_64(t, dst) ? 1u : 0u,
loc_reg(dst), src.v.imm);
return;
}
@@ -2789,8 +2807,7 @@ static void rv_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
MemAccess mem, KitCgMemOrder mo) {
RvNativeTarget* a = rv_of(t);
MCEmitter* mc = t->mc;
- u32 sf =
- (mem.size ? mem.size : native_type_size(t, dst.type)) == 8u ? 1u : 0u;
+ u32 sf = (mem.size ? mem.size : loc_size32(t, dst)) == 8u ? 1u : 0u;
u32 base = rv_atomic_addr_reg(a, addr);
if (mo == KIT_CG_MO_SEQ_CST) rv64_emit32(mc, rv_fence_rw_rw());
if (rv_order_acquire(mo)) {
@@ -2798,10 +2815,9 @@ static void rv_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
rv64_emit32(mc, sf ? rv_lr_d(loc_reg(dst), base, 1, 0)
: rv_lr_w(loc_reg(dst), base, 1, 0));
} else {
- rv64_emit32(
- mc, enc_int_load(a->variant,
- mem.size ? mem.size : native_type_size(t, dst.type), 0,
- loc_reg(dst), base, 0));
+ rv64_emit32(mc, enc_int_load(a->variant,
+ mem.size ? mem.size : loc_size32(t, dst), 0,
+ loc_reg(dst), base, 0));
}
}
@@ -2809,7 +2825,7 @@ static void rv_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
MemAccess mem, KitCgMemOrder mo) {
RvNativeTarget* a = rv_of(t);
MCEmitter* mc = t->mc;
- u32 sz = mem.size ? mem.size : native_type_size(t, src.type);
+ u32 sz = mem.size ? mem.size : loc_size32(t, src);
/* RV_TMP0 holds the address; never collides with src (an allocable reg). */
u32 base = rv_atomic_addr_reg(a, addr);
if (rv_order_release(mo)) rv64_emit32(mc, rv_fence_rw_rw());
@@ -2823,8 +2839,7 @@ static void rv_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
RvNativeTarget* a = rv_of(t);
const RiscvVariant* v = a->variant;
MCEmitter* mc = t->mc;
- u32 sf =
- (mem.size ? mem.size : native_type_size(t, dst.type)) == 8u ? 1u : 0u;
+ u32 sf = (mem.size ? mem.size : loc_size32(t, dst)) == 8u ? 1u : 0u;
/* W-form add/sub apply only to a 32-bit value on rv64; on rv32 the base ops
* are the 32-bit ops. */
int w = !sf && v->has_w_forms;
@@ -2878,8 +2893,7 @@ static void rv_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
KitCgMemOrder success, KitCgMemOrder failure) {
RvNativeTarget* a = rv_of(t);
MCEmitter* mc = t->mc;
- u32 sf =
- (mem.size ? mem.size : native_type_size(t, prior.type)) == 8u ? 1u : 0u;
+ u32 sf = (mem.size ? mem.size : loc_size32(t, prior)) == 8u ? 1u : 0u;
u32 base = rv_atomic_addr_reg(a, addr); /* RV_TMP0 */
u32 rprior = loc_reg(prior);
u32 rexp = loc_reg(expected);
diff --git a/src/arch/x64/native.c b/src/arch/x64/native.c
@@ -155,6 +155,20 @@ static int x64_is_64(NativeTarget* t, KitCgTypeId type) {
return native_type_size(t, type) >= 8u || cg_type_is_ptr(t->c, type);
}
+/* Scalar byte width of a register operand. Reads the NDT-stamped szinfo
+ * descriptor (one byte) when present, else falls back to the live type query —
+ * byte-identical to native_type_size for in-range scalars. */
+static u32 loc_size32(NativeTarget* t, NativeLoc loc) {
+ if (loc.szinfo & NATIVE_SZINFO_VALID) return native_szinfo_size(loc.szinfo);
+ return native_type_size(t, loc.type);
+}
+
+/* The original x64_is_64 predicate exactly, but sourcing the width from the
+ * cached descriptor: `size >= 8 || is_ptr`. Byte-identical. */
+static int loc_is_64(NativeTarget* t, NativeLoc loc) {
+ return loc_size32(t, loc) >= 8u || cg_type_is_ptr(t->c, loc.type);
+}
+
static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0xfu; }
/* SSE scalar prefix: F2 (double / 8-byte) vs F3 (single / 4-byte). */
@@ -622,7 +636,7 @@ static void x64_emit_mem(X64NativeTarget* a, int is_load, NativeLoc reg,
MCEmitter* mc = t->mc;
u32 r = loc_reg(reg);
int fp = native_loc_is_fp(reg);
- u32 sz = mem.size ? mem.size : native_type_size(t, reg.type);
+ u32 sz = mem.size ? mem.size : loc_size32(t, reg);
u32 base, idx, scale;
i32 off;
@@ -714,26 +728,26 @@ static void x64_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
u32 rd = loc_reg(dst), rs = loc_reg(src);
if (dfp && sfp) {
if (rd == rs) return;
- emit_sse_rr(mc, sse_scalar_prefix(native_type_size(t, dst.type)), 0x10, rd,
+ emit_sse_rr(mc, sse_scalar_prefix(loc_size32(t, dst)), 0x10, rd,
rs);
return;
}
if (dfp && !sfp) { /* movd/movq gpr -> xmm: 66 0F 6E /r */
- int w = native_type_size(t, dst.type) == 8u;
+ int w = loc_size32(t, dst) == 8u;
emit_sse_rr_w(mc, 0x66, 0x6E, w, rd, rs);
return;
}
if (!dfp && sfp) { /* movd/movq xmm -> gpr: 66 0F 7E /r (xmm is reg field) */
- int w = native_type_size(t, src.type) == 8u;
+ int w = loc_size32(t, src) == 8u;
emit_sse_rr_w(mc, 0x66, 0x7E, w, rs, rd);
return;
}
if (rd == rs) return;
- emit_mov_rr(mc, x64_is_64(t, dst.type) ? 1 : 0, rd, rs);
+ emit_mov_rr(mc, loc_is_64(t, dst) ? 1 : 0, rd, rs);
}
static void x64_load_imm(NativeTarget* t, NativeLoc dst, i64 imm) {
- x64_emit_load_imm(t->mc, x64_is_64(t, dst.type) ? 1 : 0, loc_reg(dst), imm);
+ x64_emit_load_imm(t->mc, loc_is_64(t, dst) ? 1 : 0, loc_reg(dst), imm);
}
/* FP constant: materialize the bit pattern in a GPR scratch, then movd/movq
@@ -971,7 +985,7 @@ static void x64_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc aop,
/* FP binops: two-address. dst = aop op bop. */
if (op == BO_FADD || op == BO_FSUB || op == BO_FMUL || op == BO_FDIV) {
u32 ra = loc_reg(aop), rb = loc_reg(bop);
- u8 prefix = sse_scalar_prefix(native_type_size(t, dst.type));
+ u8 prefix = sse_scalar_prefix(loc_size32(t, dst));
u8 opcode;
switch (op) {
case BO_FADD:
@@ -1004,7 +1018,7 @@ static void x64_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc aop,
}
{
- int w = x64_is_64(t, dst.type) ? 1 : 0;
+ int w = loc_is_64(t, dst) ? 1 : 0;
int b_imm = bop.kind == NATIVE_LOC_IMM;
i64 imm = b_imm ? bop.v.imm : 0;
u32 ra = loc_reg(aop);
@@ -1165,7 +1179,7 @@ static void x64_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
MCEmitter* mc = t->mc;
u32 rd = loc_reg(dst), rs = loc_reg(src);
if (op == UO_FNEG) {
- int dbl = native_type_size(t, dst.type) == 8u;
+ int dbl = loc_size32(t, dst) == 8u;
if (rd != rs)
emit_sse_rr(mc, sse_scalar_prefix(dbl ? 8u : 4u), 0x10, rd, rs);
/* sign mask into fp scratch via gpr, then XORPS/XORPD. */
@@ -1176,7 +1190,7 @@ static void x64_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
return;
}
{
- int w = x64_is_64(t, dst.type) ? 1 : 0;
+ int w = loc_is_64(t, dst) ? 1 : 0;
switch (op) {
case UO_NEG:
if (rd != rs) emit_mov_rr(mc, w, rd, rs);
@@ -1240,12 +1254,12 @@ static void x64_emit_cmp_flags(NativeTarget* t, NativeLoc aop, NativeLoc bop,
X64NativeTarget* a = x64_of(t);
MCEmitter* mc = t->mc;
if (fp) {
- u8 prefix = native_type_size(t, aop.type) == 8u ? 0x66u : 0u;
+ u8 prefix = loc_size32(t, aop) == 8u ? 0x66u : 0u;
emit_sse_rr(mc, prefix, 0x2E, loc_reg(aop), loc_reg(bop)); /* ucomis */
return;
}
{
- int w = x64_is_64(t, aop.type) ? 1 : 0;
+ int w = loc_is_64(t, aop) ? 1 : 0;
u32 ra = loc_reg(aop);
if (bop.kind == NATIVE_LOC_IMM) {
i64 imm = bop.v.imm;
@@ -1352,14 +1366,14 @@ static void x64_convert(NativeTarget* t, ConvKind k, NativeLoc dst,
u32 rd = loc_reg(dst), rs = loc_reg(src);
switch (k) {
case CV_SEXT: {
- u32 src_sz = native_type_size(t, src.type);
- int w = x64_is_64(t, dst.type) ? 1 : 0;
+ u32 src_sz = loc_size32(t, src);
+ int w = loc_is_64(t, dst) ? 1 : 0;
emit_extend_rr(mc, w, 1, src_sz, rd, rs);
return;
}
case CV_ZEXT: {
- u32 src_sz = native_type_size(t, src.type);
- int w = x64_is_64(t, dst.type) ? 1 : 0;
+ u32 src_sz = loc_size32(t, src);
+ int w = loc_is_64(t, dst) ? 1 : 0;
emit_extend_rr(mc, w, 0, src_sz, rd, rs);
return;
}
@@ -1368,8 +1382,8 @@ static void x64_convert(NativeTarget* t, ConvKind k, NativeLoc dst,
return;
case CV_ITOF_S:
case CV_ITOF_U: {
- int w_src = x64_is_64(t, src.type) ? 1 : 0;
- u8 prefix = sse_scalar_prefix(native_type_size(t, dst.type));
+ int w_src = loc_is_64(t, src) ? 1 : 0;
+ u8 prefix = sse_scalar_prefix(loc_size32(t, dst));
if (k == CV_ITOF_U && w_src == 1) {
MCLabel L_high = mc_label_new(mc);
MCLabel L_done = mc_label_new(mc);
@@ -1398,12 +1412,12 @@ static void x64_convert(NativeTarget* t, ConvKind k, NativeLoc dst,
}
case CV_FTOI_S:
case CV_FTOI_U: {
- int w_dst = x64_is_64(t, dst.type) ? 1 : 0;
- u8 prefix = sse_scalar_prefix(native_type_size(t, src.type));
+ int w_dst = loc_is_64(t, dst) ? 1 : 0;
+ u8 prefix = sse_scalar_prefix(loc_size32(t, src));
/* Unsigned 64-bit FTOI needs the 2^63 bias dance; otherwise cvtt
* (with the destination widened to 64 for u32) is exact. */
if (k == CV_FTOI_U && w_dst == 1) {
- int dbl = native_type_size(t, src.type) == 8u;
+ int dbl = loc_size32(t, src) == 8u;
MCLabel L_small = mc_label_new(mc);
MCLabel L_done = mc_label_new(mc);
/* limit = 2^63 in fp scratch. */
@@ -1436,9 +1450,9 @@ static void x64_convert(NativeTarget* t, ConvKind k, NativeLoc dst,
return;
case CV_BITCAST:
if (!native_loc_is_fp(src) && native_loc_is_fp(dst)) {
- emit_sse_rr_w(mc, 0x66, 0x6E, x64_is_64(t, dst.type), rd, rs);
+ emit_sse_rr_w(mc, 0x66, 0x6E, loc_is_64(t, dst), rd, rs);
} else if (native_loc_is_fp(src) && !native_loc_is_fp(dst)) {
- emit_sse_rr_w(mc, 0x66, 0x7E, x64_is_64(t, src.type), rs, rd);
+ emit_sse_rr_w(mc, 0x66, 0x7E, loc_is_64(t, src), rs, rd);
} else {
x64_move(t, dst, src);
}
@@ -2224,7 +2238,7 @@ static void x64_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
return;
}
if (src.kind == NATIVE_LOC_IMM) {
- x64_emit_load_imm(t->mc, x64_is_64(t, dst.type) ? 1 : 0, loc_reg(dst),
+ x64_emit_load_imm(t->mc, loc_is_64(t, dst) ? 1 : 0, loc_reg(dst),
src.v.imm);
return;
}
@@ -3100,7 +3114,7 @@ static u32 x64_atomic_base(X64NativeTarget* a, NativeAddr addr) {
static void x64_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
MemAccess mem, KitCgMemOrder mo) {
X64NativeTarget* a = x64_of(t);
- u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
+ u32 sz = mem.size ? mem.size : loc_size32(t, dst);
u32 base;
(void)mo; /* x86 plain MOV is an acquire load. */
base = x64_atomic_base(a, addr);
@@ -3111,7 +3125,7 @@ static void x64_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
MemAccess mem, KitCgMemOrder mo) {
X64NativeTarget* a = x64_of(t);
MCEmitter* mc = t->mc;
- u32 sz = mem.size ? mem.size : native_type_size(t, src.type);
+ u32 sz = mem.size ? mem.size : loc_size32(t, src);
int w = sz == 8u ? 1 : 0;
u32 base = x64_atomic_base(a, addr);
u32 sr = loc_reg(src);
@@ -3135,7 +3149,7 @@ static void x64_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
KitCgMemOrder mo) {
X64NativeTarget* a = x64_of(t);
MCEmitter* mc = t->mc;
- u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
+ u32 sz = mem.size ? mem.size : loc_size32(t, dst);
int w = sz == 8u ? 1 : 0;
u32 base = x64_atomic_base(a, addr);
u32 dr = loc_reg(dst);
@@ -3215,7 +3229,7 @@ static void x64_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
KitCgMemOrder success, KitCgMemOrder failure) {
X64NativeTarget* a = x64_of(t);
MCEmitter* mc = t->mc;
- u32 sz = mem.size ? mem.size : native_type_size(t, prior.type);
+ u32 sz = mem.size ? mem.size : loc_size32(t, prior);
int w = sz == 8u ? 1 : 0;
u32 base = x64_atomic_base(a, addr);
u32 rprior = loc_reg(prior);
@@ -4035,7 +4049,7 @@ static void x64_asm_load_loc_to_reg(X64NativeTarget* a, SrcLoc loc,
return;
}
x64_emit_mem(a, 1, dst, x64_asm_loc_to_addr(a, loc, src),
- native_mem_for_type(t, dst.type, native_type_size(t, dst.type)));
+ native_mem_for_type(t, dst.type, loc_size32(t, dst)));
}
static void x64_asm_store_reg_to_loc(X64NativeTarget* a, SrcLoc loc,
@@ -4046,7 +4060,7 @@ static void x64_asm_store_reg_to_loc(X64NativeTarget* a, SrcLoc loc,
return;
}
x64_emit_mem(a, 0, src, x64_asm_loc_to_addr(a, loc, dst),
- native_mem_for_type(t, src.type, native_type_size(t, src.type)));
+ native_mem_for_type(t, src.type, loc_size32(t, src)));
}
static void x64_asm_bind_native(X64NativeTarget* a, SrcLoc loc, Operand* out,
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -144,7 +144,16 @@ static const NativeAllocClassInfo* nd_class_info(NativeDirectTarget* d,
}
/* Register-location constructor is shared as native_loc_reg in
- * native_target.h (arg order: type, cls, reg). */
+ * native_target.h (arg order: type, cls, reg). nd_loc_reg wraps it to stamp
+ * the cached scalar-width descriptor (szinfo) once, here at the single
+ * register-loc choke point, so the per-arch binop/move/convert/cmp hot path
+ * reads loc.szinfo instead of re-crossing the type bridge per operand. */
+static inline NativeLoc nd_loc_reg(NativeDirectTarget* d, KitCgTypeId type,
+ NativeAllocClass cls, Reg reg) {
+ NativeLoc loc = native_loc_reg(type, cls, reg);
+ if (d->native) native_loc_stamp_size(d->native, &loc);
+ return loc;
+}
static void nd_flush_local(NativeDirectTarget* d, CGLocal local);
static Reg nd_cache_reg_for(NativeDirectTarget* d, CGLocal local,
@@ -689,7 +698,7 @@ static void nd_flush_local(NativeDirectTarget* d, CGLocal local) {
if (l->dirty)
nd_store_reg_to_frame(
d, nd_home(d, local), l->type,
- native_loc_reg(l->type, (NativeAllocClass)l->cls, l->reg));
+ nd_loc_reg(d, l->type, (NativeAllocClass)l->cls, l->reg));
nd_cache_unlink(d, local);
d->reg_owner[l->cls][l->reg] = CG_LOCAL_NONE;
l->reg = REG_NONE;
@@ -770,7 +779,7 @@ static NativeAddr nd_addr_materialize(NativeDirectTarget* d, NativeAddr in,
if (out.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
NativeAllocClass cls = (NativeAllocClass)out.cls;
Reg r = nd_scratch_acquire(d, cls);
- NativeLoc dst = native_loc_reg(out.base_type, cls, r);
+ NativeLoc dst = nd_loc_reg(d, out.base_type, cls, r);
nd_load_frame_to_reg(d, dst, out.base.frame, out.base_type);
out.base_kind = NATIVE_ADDR_BASE_REG;
out.base.reg = r;
@@ -780,7 +789,7 @@ static NativeAddr nd_addr_materialize(NativeDirectTarget* d, NativeAddr in,
if (out.index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE) {
NativeAllocClass cls = (NativeAllocClass)out.index_cls;
Reg r = nd_scratch_acquire(d, cls);
- NativeLoc dst = native_loc_reg(out.index_type, cls, r);
+ NativeLoc dst = nd_loc_reg(d, out.index_type, cls, r);
nd_load_frame_to_reg(d, dst, out.index.frame, out.index_type);
out.index_kind = NATIVE_ADDR_INDEX_REG;
out.index.reg = r;
@@ -791,8 +800,9 @@ static NativeAddr nd_addr_materialize(NativeDirectTarget* d, NativeAddr in,
!d->native->addr_legal(d->native, &out, mem)) {
NativeAllocClass cls = NATIVE_REG_INT;
Reg r = nd_scratch_acquire(d, cls);
- NativeLoc dst = native_loc_reg(
- out.base_type ? out.base_type : builtin_id(KIT_CG_BUILTIN_I64), cls, r);
+ NativeLoc dst = nd_loc_reg(
+ d, out.base_type ? out.base_type : builtin_id(KIT_CG_BUILTIN_I64), cls,
+ r);
ND_REQUIRE_NATIVE(d, load_addr, "target does not materialize addresses");
d->native->load_addr(d->native, dst, out);
nd_addr_temps_release(d, temps);
@@ -825,7 +835,7 @@ static NativeLoc nd_materialize_loc(NativeDirectTarget* d, NativeLoc src,
NativeLoc dst;
if (src.kind == NATIVE_LOC_REG) return src;
r = nd_scratch_acquire(d, cls);
- dst = native_loc_reg(type ? type : src.type, cls, r);
+ dst = nd_loc_reg(d, type ? type : src.type, cls, r);
nd_copy_to_reg(d, dst, src);
return dst;
}
@@ -944,7 +954,7 @@ static NativeLoc nd_materialize_operand(NativeDirectTarget* d, Operand op) {
/* Cache hit: pin and reuse the live register, no reload. */
d->scratch_used[l->cls] |= 1u << l->reg;
nd_touch_local(d, l);
- return native_loc_reg(op.type, (NativeAllocClass)l->cls, l->reg);
+ return nd_loc_reg(d, op.type, (NativeAllocClass)l->cls, l->reg);
}
/* A live entry under a different access width must reach memory before we
* bypass the cache for this access. */
@@ -957,7 +967,7 @@ static NativeLoc nd_materialize_operand(NativeDirectTarget* d, Operand op) {
static NativeLoc nd_dst_scratch(NativeDirectTarget* d, Operand dst) {
NativeAllocClass cls = nd_class_for_type(d, dst.type);
Reg r = nd_scratch_acquire(d, cls);
- return native_loc_reg(dst.type, cls, r);
+ return nd_loc_reg(d, dst.type, cls, r);
}
/* Arithmetic/compare RHS: keep a constant operand as an immediate when the
@@ -993,7 +1003,7 @@ static NativeLoc nd_dst_reg(NativeDirectTarget* d, Operand dst) {
if (r != REG_NONE) {
d->scratch_used[l->cls] |= 1u << r; /* pin for the instruction */
nd_touch_local(d, l);
- return native_loc_reg(dst.type, (NativeAllocClass)l->cls, r);
+ return nd_loc_reg(d, dst.type, (NativeAllocClass)l->cls, r);
}
}
}
@@ -1500,8 +1510,8 @@ static void nd_copy(CgTarget* t, Operand dst, Operand src) {
if (r != REG_NONE) {
/* Register already holds the value as x's; just unpin + mark dirty. */
nd_dst_writeback(d, dst,
- native_loc_reg(dst.type, nd_class_for_type(d, dst.type),
- r));
+ nd_loc_reg(d, dst.type, nd_class_for_type(d, dst.type),
+ r));
return;
}
}
@@ -1849,8 +1859,8 @@ static void nd_convert(CgTarget* t, ConvKind op, Operand dst, Operand src) {
Reg r = nd_rename_killed_to_dst(d, dst, src);
if (r != REG_NONE) {
NativeAllocClass cls = nd_class_for_type(d, dst.type);
- sr = native_loc_reg(src.type, cls, r);
- dr = native_loc_reg(dst.type, cls, r);
+ sr = nd_loc_reg(d, src.type, cls, r);
+ dr = nd_loc_reg(d, dst.type, cls, r);
ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
d->native->convert(d->native, op, dr, sr);
nd_dst_writeback(d, dst, dr);
@@ -1890,7 +1900,7 @@ static void nd_call(CgTarget* t, const CGCallDesc* desc) {
NativeDirectLocal* l = nd_local(d, desc->args[i]);
int dead = i < 64u && ((desc->arg_dead_mask >> i) & 1u);
if (dead && l->reg != REG_NONE) {
- args[i] = native_loc_reg(l->type, (NativeAllocClass)l->cls, l->reg);
+ args[i] = nd_loc_reg(d, l->type, (NativeAllocClass)l->cls, l->reg);
d->scratch_used[l->cls] |= 1u << l->reg;
} else {
args[i] = nd_loc_frame(d, desc->args[i], 0);
@@ -1921,7 +1931,7 @@ static void nd_call(CgTarget* t, const CGCallDesc* desc) {
/* Pin until the post-call ret move so the callee materialize cannot evict
* this not-yet-valid entry as a scratch victim. */
d->scratch_used[rl->cls] |= 1u << r;
- results[0] = native_loc_reg(rl->type, (NativeAllocClass)rl->cls, r);
+ results[0] = nd_loc_reg(d, rl->type, (NativeAllocClass)rl->cls, r);
} else {
results[0] = nd_loc_frame(d, desc->result, 0);
}
@@ -2009,7 +2019,7 @@ static void nd_ret(CgTarget* t, CGLocal value) {
if (value != CG_LOCAL_NONE) {
NativeDirectLocal* l = nd_local(d, value);
loc = l->reg != REG_NONE
- ? native_loc_reg(l->type, (NativeAllocClass)l->cls, l->reg)
+ ? nd_loc_reg(d, l->type, (NativeAllocClass)l->cls, l->reg)
: nd_loc_frame(d, value, 0);
locp = &loc;
}
