commit 3b27e21fb67bd8457611ef5fbefb3bc8e9d73ca0
parent 36feeb5b099d9e835e67636a5ec87911283aead4
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 9 Jun 2026 21:19:21 -0700
cleanup wave C-1: behavior-preserving backend extractions
Adversarial review backlog, Wave C-1 (the "use the shared seam, don't bypass
it" core). All behavior-preserving — verified by test-aa64-inline /
test-rv64-inline / test-x64-inline / test-abi-classify / test-cg-api / test-opt
/ test-isa / test-toy (all backends) / test-smoke-x64 / test-smoke-rv64 /
test-libc / test-asm. Net -134 LOC, but the win is de-triplication.
A.1 — inline-asm operand-binding orchestration: aa_/x64_/rv_direct_asm_block
were ~170-line near-identical copies over the already-shared cg/native_asm.*
primitives. Hoisted the arch-neutral out/in/save/load/run/store/restore skeleton
into native_asm_bind_direct_operands (cg/native_asm.c), parameterized via a new
NativeAsmDirectHooks struct (scratch masks, OPK_REG/FP-class constants, panic,
and a few per-arch adapters). Each backend's *_direct_asm_block is now a thin
wrapper (−222 LOC each across the 3 native.c). The direct copies were not
actually drifted (only the scratch-mask seed differed); the genuinely-drifted
optimizer-path *_asm_block_native family is a separate follow-up (A.2, tracked).
C.1 — per-ABI argument classification: hoisted abi_compute_func_info_generic
(taking a classify_one callback + sret_consumes_int_arg) and
abi_classify_scalar_reg_part into abi.c; routed sysv_x64 / win64_x64 / aapcs64 /
rv64 through them, each keeping only its aggregate/scalar-specific classify_one.
SysV's vararg gp/fp-offset pass preserved verbatim. apple_x64 / apple_arm64 /
aapcs64_windows already delegate to these bases. ABI classification is
byte-identical (calling-convention correctness).
C.2 — soft-float: kit_cg_fp_binop's three f128/double/single blocks and the 8
float<->int conversions now go through api_softfp_binop + api_int_builtin_for_size
+ an (op,width)->suffix table (mirroring the existing api_softfp_cmp). Emitted
compiler-rt libcall names and operand types are byte-identical.
Also: const-corrected the NativeAsmDirectHooks.run_template bound-operand params
to non-const Operand* (they feed the arch inline_bind, matching the original
per-arch convention) — fixes a -Werror discards-qualifiers break.
Diffstat:
13 files changed, 619 insertions(+), 747 deletions(-)
diff --git a/doc/plan/CLEANUP-2026-06-09.md b/doc/plan/CLEANUP-2026-06-09.md
@@ -100,13 +100,19 @@ The core of "use the abstraction, don't bypass it." Grouped by theme; each bulle
confirmed finding.
### A. Inline-asm orchestration triplicated (med, 4)
-- [ ] **A.1** `aa_/x64_/rv_direct_asm_block` (`aa64/native.c:4711`, `x64/native.c:4383`,
- `riscv/native.c:4083`) are ~170-line line-for-line copies on top of the *already-shared*
- `cg/native_asm.c` primitives, and already drifting (3 different frame-slot conditions;
- rv64 has an extra output-staging branch). *Fix:* hoist
- `native_asm_bind_direct_operands(d, dir, constraints, n, ops, masks, bound[])` + one
- shared driver into `cg/native_asm.c`; each backend keeps only its register table + a
- thin vtable. (Single agent owns all 4 files.)
+- [x] **A.1** `aa_/x64_/rv_direct_asm_block` (`aa64/native.c:4711`, `x64/native.c:4383`,
+ `riscv/native.c:4083`) were ~170-line copies on top of the *already-shared*
+ `cg/native_asm.c` primitives. DONE: hoisted `native_asm_bind_direct_operands` + the shared
+ driver + `NativeAsmDirectHooks` into `cg/native_asm.{c,h}`; each backend's `*_direct_asm_block`
+ is now a thin wrapper supplying scratch masks / OPK constants / a panic + a few adapters
+ (−222 LOC × 3 native.c). The direct copies turned out **not** drifted (only the scratch-mask
+ seed differed); behavior preserved, verified by inline-asm × 3 / toy / smoke.
+- [ ] **A.2 (surfaced by A.1)** the *optimizer-path* hooks `aa_/x64_/rv_asm_block_native`
+ (`aa64/native.c:~4869`, `x64/native.c:~4000`, `riscv/native.c:~3753`) are the *genuinely*
+ drifted triplication: rv64's `rv_asm_block_native` carries an extra `staged_outs[i]=1` branch
+ (`riscv/native.c:~3789`) and the input loops have differing frame-slot scratch-load
+ conditions. Left verbatim by A.1. *Fix:* a second shared driver (or extend the direct one)
+ for the optimizer path — and reconcile/flag the rv64 staging drift (possible latent bug).
### B. Per-arch codegen plumbing duplicated (med, 7)
- [ ] **B.1** x64 callee-save offset formula open-coded 5+ times; collect+reverse-restore
@@ -122,11 +128,11 @@ confirmed finding.
across arches. *Fix:* share via `src/arch/registry` / `cg/native_direct_target`.
### C. ABI / soft-float / reloc lowering hand-copied (med→low, 7)
-- [ ] **C.1** `compute_func_info` driver + `classify_one` dispatch + single-register scalar
+- [x] **C.1** `compute_func_info` driver + `classify_one` dispatch + single-register scalar
tail near-identical across all 4 ABI vtable TUs (`src/abi/abi_sysv_x64.c:184-202,218-279`
+ siblings). *Fix:* `abi_compute_func_info_generic(classify_one cb, sret_consumes_int)` +
`abi_classify_scalar_reg_part` in `abi.c`; each TU keeps only aggregate/scalar specifics.
-- [ ] **C.2** `kit_cg_fp_binop` + 8 float↔int conversions open-code 16 width→type ternaries
+- [x] **C.2** `kit_cg_fp_binop` + 8 float↔int conversions open-code 16 width→type ternaries
(`src/cg/arith.c:1253-1704`) while `api_softfp_cmp` proves the parameterized form. *Fix:*
`api_softfp_binop` mirroring `api_softfp_cmp` + an `(op,width)→suffix` table.
- [ ] **C.3** RV B/J/S immediate bit-scramble reimplemented 3×. *Fix:* one shared encoder.
diff --git a/src/abi/abi.c b/src/abi/abi.c
@@ -182,6 +182,59 @@ void abi_classify_int128_pair(TargetABI* a, ABIArgInfo* out) {
out->indirect_align = 0;
}
+/* The single-register scalar tail shared by every per-ABI classify_scalar: a
+ * DIRECT argument with exactly one register part carrying the whole scalar.
+ * `is_fp` selects the part class; the caller owns the FP-eligibility decision. */
+void abi_classify_scalar_reg_part(TargetABI* a, ABIArgInfo* out, ABITypeInfo ti,
+ int is_fp) {
+ out->kind = ABI_ARG_DIRECT;
+ out->flags = ABI_AF_NONE;
+ out->indirect_align = 0;
+
+ ABIArgPart* parts = arena_new(a->c->tu, ABIArgPart);
+ memset(parts, 0, sizeof *parts);
+ parts->cls = is_fp ? ABI_CLASS_FP : ABI_CLASS_INT;
+ parts->loc = ABI_LOC_REG;
+ parts->size = ti.size;
+ parts->align = ti.align;
+ parts->src_offset = 0;
+
+ out->parts = parts;
+ out->nparts = 1;
+}
+
+/* Generic compute_func_info scaffold shared by the per-ABI vtables. Classifies
+ * the result then every parameter through `classify_one`, filling the shared
+ * has_sret / sret_consumes_int_arg / variadic / nparams fields. The per-ABI
+ * vararg metadata pass (SysV-x64 gp/fp offsets) runs in the caller on the
+ * returned info; all vararg_* fields are left zero here. */
+ABIFuncInfo* abi_compute_func_info_generic(TargetABI* a, KitCgTypeId fn,
+ ABIClassifyOneFn classify_one,
+ int sret_consumes_int_arg) {
+ ABIFuncInfo* info = arena_new(a->c->tu, ABIFuncInfo);
+ const CgType* fnty = cg_type_get(a->c, fn);
+ memset(info, 0, sizeof *info);
+
+ classify_one(a, cg_func_ret_type(fnty), &info->ret, /*is_return=*/1);
+ info->has_sret = (info->ret.kind == ABI_ARG_INDIRECT) ? 1 : 0;
+ info->sret_consumes_int_arg =
+ (sret_consumes_int_arg && info->has_sret) ? 1 : 0;
+ info->variadic = fnty->func.abi_variadic;
+
+ info->nparams = (u16)fnty->func.nparams;
+ if (fnty->func.nparams) {
+ ABIArgInfo* arr = arena_array(a->c->tu, ABIArgInfo, fnty->func.nparams);
+ memset(arr, 0, sizeof(ABIArgInfo) * fnty->func.nparams);
+ for (u32 i = 0; i < fnty->func.nparams; ++i) {
+ classify_one(a, fnty->func.params[i].type, &arr[i], /*is_return=*/0);
+ }
+ info->params = arr;
+ } else {
+ info->params = NULL;
+ }
+ return info;
+}
+
/* ---- function classification (vtabled) ---- */
const ABIFuncInfo* abi_cg_func_info(TargetABI* a, KitCgTypeId fn_type) {
diff --git a/src/abi/abi_aapcs64.c b/src/abi/abi_aapcs64.c
@@ -24,20 +24,7 @@ static void classify_scalar(TargetABI* a, KitCgTypeId t, ABIArgInfo* out) {
abi_classify_int128_pair(a, out);
return;
}
- out->kind = ABI_ARG_DIRECT;
- out->flags = ABI_AF_NONE;
- out->indirect_align = 0;
-
- ABIArgPart* parts = arena_new(a->c->tu, ABIArgPart);
- memset(parts, 0, sizeof *parts);
- parts->cls = (ti.scalar_kind == ABI_SC_FLOAT) ? ABI_CLASS_FP : ABI_CLASS_INT;
- parts->loc = ABI_LOC_REG;
- parts->size = ti.size;
- parts->align = ti.align;
- parts->src_offset = 0;
-
- out->parts = parts;
- out->nparts = 1;
+ abi_classify_scalar_reg_part(a, out, ti, ti.scalar_kind == ABI_SC_FLOAT);
}
static void classify_aggregate(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
@@ -101,30 +88,10 @@ static void classify_one(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
/* Non-static so apple_arm64_compute_func_info can delegate to it during
* the Phase 1 alias period — see abi_apple_arm64.c. */
ABIFuncInfo* aapcs64_compute_func_info(TargetABI* a, KitCgTypeId fn) {
- ABIFuncInfo* info = arena_new(a->c->tu, ABIFuncInfo);
- const CgType* fnty = cg_type_get(a->c, fn);
- memset(info, 0, sizeof *info);
-
- classify_one(a, cg_func_ret_type(fnty), &info->ret, /*is_return=*/1);
- info->has_sret = (info->ret.kind == ABI_ARG_INDIRECT) ? 1 : 0;
/* AArch64 returns the sret pointer in the dedicated x8 register, so it never
- * consumes an x0..x7 argument slot. (memset above already cleared the field;
- * set explicitly for documentation.) */
- info->sret_consumes_int_arg = 0;
- info->variadic = fnty->func.abi_variadic;
-
- info->nparams = (u16)fnty->func.nparams;
- if (fnty->func.nparams) {
- ABIArgInfo* arr = arena_array(a->c->tu, ABIArgInfo, fnty->func.nparams);
- memset(arr, 0, sizeof(ABIArgInfo) * fnty->func.nparams);
- for (u32 i = 0; i < fnty->func.nparams; ++i) {
- classify_one(a, fnty->func.params[i].type, &arr[i], /*is_return=*/0);
- }
- info->params = arr;
- } else {
- info->params = NULL;
- }
- return info;
+ * consumes an x0..x7 argument slot: sret_consumes_int_arg stays 0. */
+ return abi_compute_func_info_generic(a, fn, classify_one,
+ /*sret_consumes_int_arg=*/0);
}
const ABIVtable aapcs64_vtable = {
diff --git a/src/abi/abi_internal.h b/src/abi/abi_internal.h
@@ -77,4 +77,29 @@ struct TargetABI {
void abi_classify_void(ABIArgInfo* out);
void abi_classify_int128_pair(TargetABI* a, ABIArgInfo* out);
+/* The single-register scalar tail shared by every per-ABI classify_scalar:
+ * a DIRECT argument with one register part carrying the whole scalar. `is_fp`
+ * selects ABI_CLASS_FP vs ABI_CLASS_INT for the part (the per-ABI caller has
+ * already applied its own FP-eligibility rule). */
+void abi_classify_scalar_reg_part(TargetABI* a, ABIArgInfo* out, ABITypeInfo ti,
+ int is_fp);
+
+/* The per-argument dispatch each ABI plugs into the generic driver: classify a
+ * single function argument/result `t` into `out` (`is_return` chooses
+ * sret vs byval for indirect aggregates). */
+typedef void (*ABIClassifyOneFn)(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
+ int is_return);
+
+/* Generic compute_func_info scaffold shared by the per-ABI vtables: allocate
+ * the ABIFuncInfo, classify the result and every parameter through
+ * `classify_one`, and fill the shared has_sret / variadic / nparams fields.
+ * `sret_consumes_int_arg` is stored verbatim into ABIFuncInfo.has_sret's
+ * companion flag (ABIs that return the sret pointer in a dedicated register
+ * pass 0; ABIs that consume the first integer arg slot pass has_sret-derived 1
+ * by setting this to nonzero). The ABI-specific vararg metadata pass (SysV-x64)
+ * runs in the caller on the returned info. */
+ABIFuncInfo* abi_compute_func_info_generic(TargetABI* a, KitCgTypeId fn,
+ ABIClassifyOneFn classify_one,
+ int sret_consumes_int_arg);
+
#endif
diff --git a/src/abi/abi_rv64.c b/src/abi/abi_rv64.c
@@ -162,20 +162,7 @@ static void classify_scalar(TargetABI* a, KitCgTypeId t, ABIArgInfo* out) {
out->indirect_align = 0;
return;
}
- out->kind = ABI_ARG_DIRECT;
- out->flags = ABI_AF_NONE;
- out->indirect_align = 0;
-
- ABIArgPart* parts = arena_new(a->c->tu, ABIArgPart);
- memset(parts, 0, sizeof *parts);
- parts->cls = fp_part ? ABI_CLASS_FP : ABI_CLASS_INT;
- parts->loc = ABI_LOC_REG;
- parts->size = ti.size;
- parts->align = ti.align;
- parts->src_offset = 0;
-
- out->parts = parts;
- out->nparts = 1;
+ abi_classify_scalar_reg_part(a, out, ti, fp_part);
}
static u32 riscv32_scalar_split_lane_size(TargetABI* a, KitCgTypeId t) {
@@ -294,29 +281,10 @@ static void classify_one(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
}
static ABIFuncInfo* riscv_compute_func_info(TargetABI* a, KitCgTypeId fn) {
- ABIFuncInfo* info = arena_new(a->c->tu, ABIFuncInfo);
- const CgType* fnty = cg_type_get(a->c, fn);
- memset(info, 0, sizeof *info);
-
- classify_one(a, cg_func_ret_type(fnty), &info->ret, /*is_return=*/1);
- info->has_sret = (info->ret.kind == ABI_ARG_INDIRECT) ? 1 : 0;
/* RISC-V passes the sret pointer in a0 (the first integer arg register),
- * consuming that slot. */
- info->sret_consumes_int_arg = info->has_sret;
- info->variadic = fnty->func.abi_variadic;
-
- info->nparams = (u16)fnty->func.nparams;
- if (fnty->func.nparams) {
- ABIArgInfo* arr = arena_array(a->c->tu, ABIArgInfo, fnty->func.nparams);
- memset(arr, 0, sizeof(ABIArgInfo) * fnty->func.nparams);
- for (u32 i = 0; i < fnty->func.nparams; ++i) {
- classify_one(a, fnty->func.params[i].type, &arr[i], /*is_return=*/0);
- }
- info->params = arr;
- } else {
- info->params = NULL;
- }
- return info;
+ * consuming that slot, so sret_consumes_int_arg follows has_sret. */
+ return abi_compute_func_info_generic(a, fn, classify_one,
+ /*sret_consumes_int_arg=*/1);
}
const ABIVtable rv64_vtable = {
diff --git a/src/abi/abi_sysv_x64.c b/src/abi/abi_sysv_x64.c
@@ -33,20 +33,7 @@ static void classify_scalar(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
return;
}
- out->kind = ABI_ARG_DIRECT;
- out->flags = ABI_AF_NONE;
- out->indirect_align = 0;
-
- ABIArgPart* parts = arena_new(a->c->tu, ABIArgPart);
- memset(parts, 0, sizeof *parts);
- parts->cls = (ti.scalar_kind == ABI_SC_FLOAT) ? ABI_CLASS_FP : ABI_CLASS_INT;
- parts->loc = ABI_LOC_REG;
- parts->size = ti.size;
- parts->align = ti.align;
- parts->src_offset = 0;
-
- out->parts = parts;
- out->nparts = 1;
+ abi_classify_scalar_reg_part(a, out, ti, ti.scalar_kind == ABI_SC_FLOAT);
}
typedef enum SysVClass {
@@ -216,28 +203,10 @@ static void classify_one(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
(SYSV_X64_FP_BASE_OFFSET + SYSV_X64_FP_REG_COUNT * SYSV_X64_FP_SLOT_SIZE)
static ABIFuncInfo* sysv_x64_compute_func_info(TargetABI* a, KitCgTypeId fn) {
- ABIFuncInfo* info = arena_new(a->c->tu, ABIFuncInfo);
- const CgType* fnty = cg_type_get(a->c, fn);
- memset(info, 0, sizeof *info);
-
- classify_one(a, cg_func_ret_type(fnty), &info->ret, /*is_return=*/1);
- info->has_sret = (info->ret.kind == ABI_ARG_INDIRECT) ? 1 : 0;
/* SysV-x64 passes the sret pointer in rdi (the first integer arg register),
- * consuming that slot. */
- info->sret_consumes_int_arg = info->has_sret;
- info->variadic = fnty->func.abi_variadic;
-
- info->nparams = (u16)fnty->func.nparams;
- if (fnty->func.nparams) {
- ABIArgInfo* arr = arena_array(a->c->tu, ABIArgInfo, fnty->func.nparams);
- memset(arr, 0, sizeof(ABIArgInfo) * fnty->func.nparams);
- for (u32 i = 0; i < fnty->func.nparams; ++i) {
- classify_one(a, fnty->func.params[i].type, &arr[i], /*is_return=*/0);
- }
- info->params = arr;
- } else {
- info->params = NULL;
- }
+ * consuming that slot, so sret_consumes_int_arg follows has_sret. */
+ ABIFuncInfo* info = abi_compute_func_info_generic(a, fn, classify_one,
+ /*sret_consumes_int_arg=*/1);
/* Variadic register-save-area offsets at function entry. Counts the
* GP/FP register slots consumed by the fixed (named) parameters; va_start
diff --git a/src/abi/abi_win64_x64.c b/src/abi/abi_win64_x64.c
@@ -61,20 +61,7 @@ static void classify_scalar(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
return;
}
- out->kind = ABI_ARG_DIRECT;
- out->flags = ABI_AF_NONE;
- out->indirect_align = 0;
-
- ABIArgPart* parts = arena_new(a->c->tu, ABIArgPart);
- memset(parts, 0, sizeof *parts);
- parts->cls = (ti.scalar_kind == ABI_SC_FLOAT) ? ABI_CLASS_FP : ABI_CLASS_INT;
- parts->loc = ABI_LOC_REG;
- parts->size = ti.size;
- parts->align = ti.align;
- parts->src_offset = 0;
-
- out->parts = parts;
- out->nparts = 1;
+ abi_classify_scalar_reg_part(a, out, ti, ti.scalar_kind == ABI_SC_FLOAT);
}
static void classify_aggregate(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
@@ -131,29 +118,10 @@ static void classify_one(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
}
static ABIFuncInfo* win64_x64_compute_func_info(TargetABI* a, KitCgTypeId fn) {
- ABIFuncInfo* info = arena_new(a->c->tu, ABIFuncInfo);
- const CgType* fnty = cg_type_get(a->c, fn);
- memset(info, 0, sizeof *info);
-
- classify_one(a, cg_func_ret_type(fnty), &info->ret, /*is_return=*/1);
- info->has_sret = (info->ret.kind == ABI_ARG_INDIRECT) ? 1 : 0;
/* Win64 passes the sret pointer in rcx (the first integer arg register),
- * consuming that slot. */
- info->sret_consumes_int_arg = info->has_sret;
- info->variadic = fnty->func.abi_variadic;
-
- info->nparams = (u16)fnty->func.nparams;
- if (fnty->func.nparams) {
- ABIArgInfo* arr = arena_array(a->c->tu, ABIArgInfo, fnty->func.nparams);
- memset(arr, 0, sizeof(ABIArgInfo) * fnty->func.nparams);
- for (u32 i = 0; i < fnty->func.nparams; ++i) {
- classify_one(a, fnty->func.params[i].type, &arr[i], /*is_return=*/0);
- }
- info->params = arr;
- } else {
- info->params = NULL;
- }
- return info;
+ * consuming that slot, so sret_consumes_int_arg follows has_sret. */
+ return abi_compute_func_info_generic(a, fn, classify_one,
+ /*sret_consumes_int_arg=*/1);
}
const ABIVtable win64_x64_vtable = {
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -4566,16 +4566,7 @@ AA_UNUSED_FN static Reg aa_asm_alloc_reg(NativeDirectTarget* d,
return REG_NONE;
}
-static int aa_asm_resolve_pin_or_panic(NativeDirectTarget* d, Sym reg,
- const char* constraint,
- NativeAsmRegPin* pin) {
- NativeAsmRegPinStatus st =
- native_asm_resolve_pin(d->native, reg, constraint, pin);
- if (st == NATIVE_ASM_REG_PIN_ABSENT) return 0;
- if (st != NATIVE_ASM_REG_PIN_OK)
- aa_asm_panic(d, native_asm_pin_status_message(st));
- return 1;
-}
+/* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
AA_UNUSED_FN static void aa_direct_load_operand_to_reg(NativeDirectTarget* d,
Operand op,
@@ -4695,175 +4686,60 @@ AA_UNUSED_FN static AAAsmSavedClobber* aa_asm_save_callee_clobbers(
return saved;
}
+/* Hook adapters that bridge the arch-typed save/restore + assembler entry to
+ * the shared NativeAsmDirectHooks signatures (which speak NativeDirectTarget +
+ * an opaque save record). */
+static void* aa_asm_hook_save_callee_clobbers(NativeDirectTarget* d,
+ u32 int_mask, u32 fp_mask,
+ u32* nsaved_out) {
+ return aa_asm_save_callee_clobbers(aa_of(d->native), int_mask, fp_mask,
+ nsaved_out);
+}
+static void aa_asm_hook_restore_one(NativeDirectTarget* d, void* saved,
+ u32 idx) {
+ aa_asm_restore_one(aa_of(d->native), &((AAAsmSavedClobber*)saved)[idx]);
+}
+static void aa_asm_hook_run_template(NativeDirectTarget* d, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ Operand* bound_outs,
+ const AsmConstraint* ins, u32 nin,
+ Operand* bound_ins,
+ const Sym* clobbers, u32 nclob) {
+ AA64Asm* a = aa64_asm_open(d->base.c);
+ aa64_inline_bind(a, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
+ nclob);
+ aa64_asm_run_template(a, d->native->mc, tmpl);
+ aa64_asm_close(a);
+}
+
static void aa_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
const AsmConstraint* outs, u32 nout,
Operand* out_ops, const AsmConstraint* ins,
u32 nin, const Operand* in_ops,
const Sym* clobbers, u32 nclob,
u32 clobber_abi_sets) {
- Operand* bound_outs =
- nout ? arena_zarray(d->base.c->tu, Operand, nout) : NULL;
- Operand* bound_ins = nin ? arena_zarray(d->base.c->tu, Operand, nin) : NULL;
- u32 clob_int, clob_fp, abi_int, abi_fp, used_int, used_fp;
- AAAsmSavedClobber* saved;
- u32 nsaved;
- AA64Asm* a;
-
- aa_asm_clobber_masks(d->base.c, d->loc, clobbers, nclob, &clob_int, &clob_fp);
- native_asm_abi_clobber_masks(d->native, clobber_abi_sets, &abi_int, &abi_fp);
- clob_int |= abi_int;
- clob_fp |= abi_fp;
- used_int = clob_int | (1u << AA_TMP0) | (1u << AA_TMP1) | (1u << 18u) |
- (1u << AA_FP) | (1u << AA_LR) | (1u << AA_SP);
- used_fp = clob_fp | (1u << 20u) | (1u << 21u);
-
- for (u32 i = 0; i < nout; ++i) {
- const char* body = native_asm_constraint_body(outs[i].str);
- NativeAsmRegPin pin;
- if (aa_asm_resolve_pin_or_panic(d, outs[i].reg, outs[i].str, &pin)) {
- /* GNU local register variable: pin to the named hard register. */
- KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
- if (pin.cls == NATIVE_REG_FP) {
- used_fp |= 1u << pin.reg;
- clob_fp |= 1u << pin.reg;
- } else {
- used_int |= 1u << pin.reg;
- clob_int |= 1u << pin.reg;
- }
- aa_asm_bound_reg(&bound_outs[i], type, pin.cls, pin.reg);
- } else {
- NativeAsmConstraintInfo info;
- if (native_asm_constraint_reg_info(d->native, outs[i].str, &info)) {
- Reg reg = info.fixed_reg != REG_NONE
- ? info.fixed_reg
- : aa_asm_alloc_reg(d, info.cls, info.allowed_mask,
- &used_int, &used_fp);
- KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
- if (info.cls == NATIVE_REG_FP) {
- used_fp |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
- } else {
- used_int |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
- }
- aa_asm_bound_reg(&bound_outs[i], type, info.cls, reg);
- } else if (body[0] == 'm') {
- Reg reg = aa_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
- KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
- aa_asm_bound_mem(&bound_outs[i], type, reg);
- } else {
- aa_asm_panic(d, "unsupported output constraint");
- }
- }
- }
-
- for (u32 i = 0; i < nin; ++i) {
- const char* body = native_asm_constraint_body(ins[i].str);
- int matched = native_asm_match_index(body);
- if (matched >= 0) {
- if ((u32)matched >= nout)
- aa_asm_panic(d, "matching constraint out of range");
- if (native_asm_constraint_early(outs[matched].str))
- aa_asm_panic(d, "matching input names early-clobber output");
- if (bound_outs[matched].kind != AA64_INLINE_OPK_REG)
- aa_asm_panic(d, "matching constraint requires register output");
- bound_ins[i] = bound_outs[matched];
- continue;
- }
- NativeAsmRegPin pin;
- if (aa_asm_resolve_pin_or_panic(d, ins[i].reg, ins[i].str, &pin)) {
- /* GNU local register variable: pin to the named hard register. */
- KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
- if (pin.cls == NATIVE_REG_FP) {
- used_fp |= 1u << pin.reg;
- clob_fp |= 1u << pin.reg;
- } else {
- used_int |= 1u << pin.reg;
- clob_int |= 1u << pin.reg;
- }
- aa_asm_bound_reg(&bound_ins[i], type, pin.cls, pin.reg);
- } else {
- NativeAsmConstraintInfo info;
- if (native_asm_constraint_reg_info(d->native, ins[i].str, &info)) {
- Reg reg = info.fixed_reg != REG_NONE
- ? info.fixed_reg
- : aa_asm_alloc_reg(d, info.cls, info.allowed_mask,
- &used_int, &used_fp);
- KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
- if (info.cls == NATIVE_REG_FP) {
- used_fp |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
- } else {
- used_int |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
- }
- aa_asm_bound_reg(&bound_ins[i], type, info.cls, reg);
- } else if (body[0] == 'i') {
- if (in_ops[i].kind != OPK_IMM)
- aa_asm_panic(d, "immediate constraint requires immediate operand");
- bound_ins[i] = in_ops[i];
- } else if (body[0] == 'm') {
- Reg reg = aa_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
- KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
- aa_asm_bound_mem(&bound_ins[i], type, reg);
- } else {
- aa_asm_panic(d, "unsupported input constraint");
- }
- }
- }
-
- saved =
- aa_asm_save_callee_clobbers(aa_of(d->native), clob_int, clob_fp, &nsaved);
- for (u32 i = 0; i < nout; ++i) {
- if (bound_outs[i].kind == AA64_INLINE_OPK_REG) {
- NativeAllocClass cls = bound_outs[i].pad[0] == AA64_INLINE_OPCLS_FP
- ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- if (outs[i].dir == KIT_CG_ASM_INOUT) {
- aa_direct_load_operand_to_reg(
- d, out_ops[i],
- native_loc_reg(bound_outs[i].type, cls,
- (Reg)bound_outs[i].v.local));
- }
- } else if (bound_outs[i].kind == OPK_INDIRECT) {
- NativeLoc loc =
- native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- (Reg)bound_outs[i].v.ind.base);
- aa_direct_load_address_to_reg(d, out_ops[i], loc);
- }
- }
- for (u32 i = 0; i < nin; ++i) {
- if (bound_ins[i].kind == AA64_INLINE_OPK_REG) {
- NativeAllocClass cls = bound_ins[i].pad[0] == AA64_INLINE_OPCLS_FP
- ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- aa_direct_load_operand_to_reg(
- d, in_ops[i],
- native_loc_reg(bound_ins[i].type, cls, (Reg)bound_ins[i].v.local));
- } else if (bound_ins[i].kind == OPK_INDIRECT) {
- NativeLoc loc =
- native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- (Reg)bound_ins[i].v.ind.base);
- aa_direct_load_address_to_reg(d, in_ops[i], loc);
- }
- }
- a = aa64_asm_open(d->base.c);
- aa64_inline_bind(a, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
- nclob);
- aa64_asm_run_template(a, d->native->mc, tmpl);
- aa64_asm_close(a);
-
- for (u32 i = 0; i < nout; ++i) {
- NativeAllocClass cls;
- NativeLoc src;
- if (bound_outs[i].kind != AA64_INLINE_OPK_REG) continue;
- cls = bound_outs[i].pad[0] == AA64_INLINE_OPCLS_FP ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
- aa_direct_store_reg_to_operand(d, out_ops[i], src);
- }
- for (u32 i = nsaved; i > 0; --i)
- aa_asm_restore_one(aa_of(d->native), &saved[i - 1u]);
+ static const NativeAsmDirectHooks hooks = {
+ /* Reserve the two emit scratch regs, x18 (platform reg), fp/lr/sp. */
+ .scratch_int = (1u << AA_TMP0) | (1u << AA_TMP1) | (1u << 18u) |
+ (1u << AA_FP) | (1u << AA_LR) | (1u << AA_SP),
+ .scratch_fp = (1u << 20u) | (1u << 21u),
+ .opk_reg = AA64_INLINE_OPK_REG,
+ .opcls_fp = AA64_INLINE_OPCLS_FP,
+ .panic = aa_asm_panic,
+ .bound_reg = aa_asm_bound_reg,
+ .bound_mem = aa_asm_bound_mem,
+ .alloc_reg = aa_asm_alloc_reg,
+ .clobber_masks = aa_asm_clobber_masks,
+ .save_callee_clobbers = aa_asm_hook_save_callee_clobbers,
+ .restore_one = aa_asm_hook_restore_one,
+ .load_operand_to_reg = aa_direct_load_operand_to_reg,
+ .load_address_to_reg = aa_direct_load_address_to_reg,
+ .store_reg_to_operand = aa_direct_store_reg_to_operand,
+ .run_template = aa_asm_hook_run_template,
+ };
+ native_asm_bind_direct_operands(d, tmpl, outs, nout, out_ops, ins, nin,
+ in_ops, clobbers, nclob, clobber_abi_sets,
+ &hooks);
}
/* ---- NativeTarget (optimizer) asm hook ----
diff --git a/src/arch/riscv/native.c b/src/arch/riscv/native.c
@@ -3409,16 +3409,7 @@ static void rv_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
}
}
-static int rv_asm_resolve_pin_or_panic(NativeDirectTarget* d, Sym reg,
- const char* constraint,
- NativeAsmRegPin* pin) {
- NativeAsmRegPinStatus st =
- native_asm_resolve_pin(d->native, reg, constraint, pin);
- if (st == NATIVE_ASM_REG_PIN_ABSENT) return 0;
- if (st != NATIVE_ASM_REG_PIN_OK)
- rv_asm_panic(d, native_asm_pin_status_message(st));
- return 1;
-}
+/* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
/* Pick a free register from the arch's caller-saved allocable pools for an
* asm operand the direct path must self-allocate. */
@@ -4091,174 +4082,63 @@ static void rv_va_copy_(NativeDirectTarget* d, Operand dst, Operand src) {
rv_va_copy_core(a, dst_ap, src_ap);
}
+/* Hook adapters bridging the arch-typed save/restore + assembler entry to the
+ * shared NativeAsmDirectHooks signatures. */
+static void* rv_asm_hook_save_callee_clobbers(NativeDirectTarget* d,
+ u32 int_mask, u32 fp_mask,
+ u32* nsaved_out) {
+ return rv_asm_save_callee_clobbers(rv_of(d->native), int_mask, fp_mask,
+ nsaved_out);
+}
+static void rv_asm_hook_restore_one(NativeDirectTarget* d, void* saved,
+ u32 idx) {
+ rv_asm_restore_one(rv_of(d->native), &((RvAsmSavedClobber*)saved)[idx]);
+}
+static void rv_asm_hook_run_template(NativeDirectTarget* d, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ Operand* bound_outs,
+ const AsmConstraint* ins, u32 nin,
+ Operand* bound_ins,
+ const Sym* clobbers, u32 nclob) {
+ Rv64Asm* asmh = rv64_asm_open(d->base.c);
+ rv64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
+ nclob);
+ rv64_asm_run_template(asmh, d->native->mc, tmpl);
+ rv64_asm_close(asmh);
+}
+
static void rv_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
const AsmConstraint* outs, u32 nout,
Operand* out_ops, const AsmConstraint* ins,
u32 nin, const Operand* in_ops,
const Sym* clobbers, u32 nclob,
u32 clobber_abi_sets) {
- RvNativeTarget* a = rv_of(d->native);
- Compiler* c = d->base.c;
- Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
- Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
- u32 clob_int, clob_fp, abi_int, abi_fp, used_int, used_fp;
- RvAsmSavedClobber* saved;
- u32 nsaved, i;
- Rv64Asm* asmh;
-
- rv_asm_clobber_masks(c, d->loc, clobbers, nclob, &clob_int, &clob_fp);
- native_asm_abi_clobber_masks(d->native, clobber_abi_sets, &abi_int, &abi_fp);
- clob_int |= abi_int;
- clob_fp |= abi_fp;
- /* Reserve emit scratch (t0/t1/t2/t3), sp/gp/tp/zero/ra and the frame pointer
- * so the operand allocator never hands them out. */
- used_int = clob_int | (1u << RV_ZERO) | (1u << RV_RA) | (1u << RV_SP) |
- (1u << RV_GP) | (1u << RV_TP) | (1u << RV_TMP0) | (1u << RV_TMP1) |
- (1u << RV_TMP2) | (1u << RV_TMP3) | (1u << RV_S0);
- used_fp =
- clob_fp | (1u << RV_FTMP0) | (1u << RV_FTMP1) | (1u << 2u) | (1u << 3u);
-
- for (i = 0; i < nout; ++i) {
- const char* body = native_asm_constraint_body(outs[i].str);
- KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
- NativeAsmRegPin pin;
- if (rv_asm_resolve_pin_or_panic(d, outs[i].reg, outs[i].str, &pin)) {
- /* GNU local register variable: pin to the named hard register. */
- if (pin.cls == NATIVE_REG_FP) {
- used_fp |= 1u << pin.reg;
- clob_fp |= 1u << pin.reg;
- } else {
- used_int |= 1u << pin.reg;
- clob_int |= 1u << pin.reg;
- }
- rv_asm_bound_reg(&bound_outs[i], type, pin.cls, pin.reg);
- } else {
- NativeAsmConstraintInfo info;
- if (native_asm_constraint_reg_info(d->native, outs[i].str, &info)) {
- Reg reg = info.fixed_reg != REG_NONE
- ? info.fixed_reg
- : rv_asm_alloc_reg(d, info.cls, info.allowed_mask,
- &used_int, &used_fp);
- if (info.cls == NATIVE_REG_FP) {
- used_fp |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
- } else {
- used_int |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
- }
- rv_asm_bound_reg(&bound_outs[i], type, info.cls, reg);
- } else if (body[0] == 'm') {
- Reg reg = rv_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
- rv_asm_bound_mem(&bound_outs[i], type, reg);
- } else {
- rv_asm_panic(d, "unsupported output constraint");
- }
- }
- }
-
- for (i = 0; i < nin; ++i) {
- const char* body = native_asm_constraint_body(ins[i].str);
- int matched = native_asm_match_index(body);
- KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
- if (matched >= 0) {
- if ((u32)matched >= nout)
- rv_asm_panic(d, "matching constraint out of range");
- if (native_asm_constraint_early(outs[matched].str))
- rv_asm_panic(d, "matching input names early-clobber output");
- if (bound_outs[matched].kind != RV64_INLINE_OPK_REG)
- rv_asm_panic(d, "matching constraint requires register output");
- bound_ins[i] = bound_outs[matched];
- continue;
- }
- NativeAsmRegPin pin;
- if (rv_asm_resolve_pin_or_panic(d, ins[i].reg, ins[i].str, &pin)) {
- /* GNU local register variable: pin to the named hard register. */
- if (pin.cls == NATIVE_REG_FP) {
- used_fp |= 1u << pin.reg;
- clob_fp |= 1u << pin.reg;
- } else {
- used_int |= 1u << pin.reg;
- clob_int |= 1u << pin.reg;
- }
- rv_asm_bound_reg(&bound_ins[i], type, pin.cls, pin.reg);
- } else {
- NativeAsmConstraintInfo info;
- if (native_asm_constraint_reg_info(d->native, ins[i].str, &info)) {
- Reg reg = info.fixed_reg != REG_NONE
- ? info.fixed_reg
- : rv_asm_alloc_reg(d, info.cls, info.allowed_mask,
- &used_int, &used_fp);
- if (info.cls == NATIVE_REG_FP) {
- used_fp |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
- } else {
- used_int |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
- }
- rv_asm_bound_reg(&bound_ins[i], type, info.cls, reg);
- } else if (body[0] == 'i') {
- if (in_ops[i].kind != OPK_IMM)
- rv_asm_panic(d, "immediate constraint requires immediate operand");
- bound_ins[i] = in_ops[i];
- } else if (body[0] == 'm') {
- Reg reg = rv_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
- rv_asm_bound_mem(&bound_ins[i], type, reg);
- } else {
- rv_asm_panic(d, "unsupported input constraint");
- }
- }
- }
-
- saved = rv_asm_save_callee_clobbers(a, clob_int, clob_fp, &nsaved);
- for (i = 0; i < nout; ++i) {
- if (bound_outs[i].kind == RV64_INLINE_OPK_REG) {
- NativeAllocClass cls = bound_outs[i].pad[0] == RV64_INLINE_OPCLS_FP
- ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- if (outs[i].dir == KIT_CG_ASM_INOUT) {
- rv_direct_load_operand_to_reg(
- d, out_ops[i],
- native_loc_reg(bound_outs[i].type, cls,
- (Reg)bound_outs[i].v.local));
- }
- } else if (bound_outs[i].kind == OPK_INDIRECT) {
- NativeLoc loc =
- native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- (Reg)bound_outs[i].v.ind.base);
- rv_direct_load_address_to_reg(d, out_ops[i], loc);
- }
- }
- for (i = 0; i < nin; ++i) {
- if (bound_ins[i].kind == RV64_INLINE_OPK_REG) {
- NativeAllocClass cls = bound_ins[i].pad[0] == RV64_INLINE_OPCLS_FP
- ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- rv_direct_load_operand_to_reg(
- d, in_ops[i],
- native_loc_reg(bound_ins[i].type, cls, (Reg)bound_ins[i].v.local));
- } else if (bound_ins[i].kind == OPK_INDIRECT) {
- NativeLoc loc =
- native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- (Reg)bound_ins[i].v.ind.base);
- rv_direct_load_address_to_reg(d, in_ops[i], loc);
- }
- }
- asmh = rv64_asm_open(c);
- rv64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
- nclob);
- rv64_asm_run_template(asmh, d->native->mc, tmpl);
- rv64_asm_close(asmh);
-
- for (i = 0; i < nout; ++i) {
- NativeAllocClass cls;
- NativeLoc src;
- if (bound_outs[i].kind != RV64_INLINE_OPK_REG) continue;
- cls = bound_outs[i].pad[0] == RV64_INLINE_OPCLS_FP ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
- rv_direct_store_reg_to_operand(d, out_ops[i], src);
- }
- for (i = nsaved; i > 0; --i) rv_asm_restore_one(a, &saved[i - 1u]);
+ static const NativeAsmDirectHooks hooks = {
+ /* Reserve emit scratch (t0/t1/t2/t3), sp/gp/tp/zero/ra and the frame
+ * pointer so the operand allocator never hands them out. */
+ .scratch_int = (1u << RV_ZERO) | (1u << RV_RA) | (1u << RV_SP) |
+ (1u << RV_GP) | (1u << RV_TP) | (1u << RV_TMP0) |
+ (1u << RV_TMP1) | (1u << RV_TMP2) | (1u << RV_TMP3) |
+ (1u << RV_S0),
+ .scratch_fp =
+ (1u << RV_FTMP0) | (1u << RV_FTMP1) | (1u << 2u) | (1u << 3u),
+ .opk_reg = RV64_INLINE_OPK_REG,
+ .opcls_fp = RV64_INLINE_OPCLS_FP,
+ .panic = rv_asm_panic,
+ .bound_reg = rv_asm_bound_reg,
+ .bound_mem = rv_asm_bound_mem,
+ .alloc_reg = rv_asm_alloc_reg,
+ .clobber_masks = rv_asm_clobber_masks,
+ .save_callee_clobbers = rv_asm_hook_save_callee_clobbers,
+ .restore_one = rv_asm_hook_restore_one,
+ .load_operand_to_reg = rv_direct_load_operand_to_reg,
+ .load_address_to_reg = rv_direct_load_address_to_reg,
+ .store_reg_to_operand = rv_direct_store_reg_to_operand,
+ .run_template = rv_asm_hook_run_template,
+ };
+ native_asm_bind_direct_operands(d, tmpl, outs, nout, out_ops, ins, nin,
+ in_ops, clobbers, nclob, clobber_abi_sets,
+ &hooks);
}
static const NativeOps rv_direct_ops = {
diff --git a/src/arch/x64/native.c b/src/arch/x64/native.c
@@ -3692,16 +3692,7 @@ static void x64_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
}
}
-static int x64_asm_resolve_pin_or_panic(NativeDirectTarget* d, Sym reg,
- const char* constraint,
- NativeAsmRegPin* pin) {
- NativeAsmRegPinStatus st =
- native_asm_resolve_pin(d->native, reg, constraint, pin);
- if (st == NATIVE_ASM_REG_PIN_ABSENT) return 0;
- if (st != NATIVE_ASM_REG_PIN_OK)
- x64_asm_panic(d, native_asm_pin_status_message(st));
- return 1;
-}
+/* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
/* Pick a free register from caller-saved allocable pools for an asm operand the
* direct path self-allocates. */
@@ -4380,174 +4371,62 @@ static void x64_va_copy_(NativeDirectTarget* d, Operand dst, Operand src) {
x64_va_copy_core(a, dst_ap, src_ap);
}
+/* Hook adapters bridging the arch-typed save/restore + assembler entry to the
+ * shared NativeAsmDirectHooks signatures. */
+static void* x64_asm_hook_save_callee_clobbers(NativeDirectTarget* d,
+ u32 int_mask, u32 fp_mask,
+ u32* nsaved_out) {
+ return x64_asm_save_callee_clobbers(x64_of(d->native), int_mask, fp_mask,
+ nsaved_out);
+}
+static void x64_asm_hook_restore_one(NativeDirectTarget* d, void* saved,
+ u32 idx) {
+ x64_asm_restore_one(x64_of(d->native), &((X64AsmSavedClobber*)saved)[idx]);
+}
+static void x64_asm_hook_run_template(NativeDirectTarget* d, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ Operand* bound_outs,
+ const AsmConstraint* ins, u32 nin,
+ Operand* bound_ins,
+ const Sym* clobbers, u32 nclob) {
+ X64Asm* asmh = x64_asm_open(d->base.c);
+ x64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
+ nclob);
+ x64_asm_run_template(asmh, d->native->mc, tmpl);
+ x64_asm_close(asmh);
+}
+
static void x64_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
const AsmConstraint* outs, u32 nout,
Operand* out_ops, const AsmConstraint* ins,
u32 nin, const Operand* in_ops,
const Sym* clobbers, u32 nclob,
u32 clobber_abi_sets) {
- X64NativeTarget* a = x64_of(d->native);
- Compiler* c = d->base.c;
- Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
- Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
- u32 clob_int, clob_fp, abi_int, abi_fp, used_int, used_fp;
- X64AsmSavedClobber* saved;
- u32 nsaved, i;
- X64Asm* asmh;
-
- x64_asm_clobber_masks(c, d->loc, clobbers, nclob, &clob_int, &clob_fp);
- native_asm_abi_clobber_masks(d->native, clobber_abi_sets, &abi_int, &abi_fp);
- clob_int |= abi_int;
- clob_fp |= abi_fp;
- /* Reserve emit scratch (r10,r11), driver scratch (r8,r9), rax (reserved;
- * only self-allocated here when explicitly pinned), sp/bp, and clobbers. */
- used_int = clob_int | (1u << X64_RAX) | (1u << X64_R11) | (1u << X64_RSP) |
- (1u << X64_RBP) | (1u << X64_R8) | (1u << X64_R9) |
- (1u << X64_R10);
- used_fp = clob_fp | (1u << X64_XMM4) | (1u << X64_XMM5) |
- (1u << (X64_XMM0 + 14)) | (1u << X64_XMM15);
-
- for (i = 0; i < nout; ++i) {
- const char* body = native_asm_constraint_body(outs[i].str);
- KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
- NativeAsmRegPin pin;
- if (x64_asm_resolve_pin_or_panic(d, outs[i].reg, outs[i].str, &pin)) {
- /* GNU local register variable: pin to the named hard register. */
- if (pin.cls == NATIVE_REG_FP) {
- used_fp |= 1u << pin.reg;
- clob_fp |= 1u << pin.reg;
- } else {
- used_int |= 1u << pin.reg;
- clob_int |= 1u << pin.reg;
- }
- x64_asm_bound_reg(&bound_outs[i], type, pin.cls, pin.reg);
- } else {
- NativeAsmConstraintInfo info;
- if (native_asm_constraint_reg_info(d->native, outs[i].str, &info)) {
- Reg reg = info.fixed_reg != REG_NONE
- ? info.fixed_reg
- : x64_asm_alloc_reg(d, info.cls, info.allowed_mask,
- &used_int, &used_fp);
- if (info.cls == NATIVE_REG_FP) {
- used_fp |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
- } else {
- used_int |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
- }
- x64_asm_bound_reg(&bound_outs[i], type, info.cls, reg);
- } else if (body[0] == 'm') {
- Reg reg = x64_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
- x64_asm_bound_mem(&bound_outs[i], type, reg);
- } else {
- x64_asm_panic(d, "unsupported output constraint");
- }
- }
- }
-
- for (i = 0; i < nin; ++i) {
- const char* body = native_asm_constraint_body(ins[i].str);
- int matched = native_asm_match_index(body);
- KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
- if (matched >= 0) {
- if ((u32)matched >= nout)
- x64_asm_panic(d, "matching constraint out of range");
- if (native_asm_constraint_early(outs[matched].str))
- x64_asm_panic(d, "matching input names early-clobber output");
- if (bound_outs[matched].kind != X64_INLINE_OPK_REG)
- x64_asm_panic(d, "matching constraint requires register output");
- bound_ins[i] = bound_outs[matched];
- continue;
- }
- NativeAsmRegPin pin;
- if (x64_asm_resolve_pin_or_panic(d, ins[i].reg, ins[i].str, &pin)) {
- /* GNU local register variable: pin to the named hard register. */
- if (pin.cls == NATIVE_REG_FP) {
- used_fp |= 1u << pin.reg;
- clob_fp |= 1u << pin.reg;
- } else {
- used_int |= 1u << pin.reg;
- clob_int |= 1u << pin.reg;
- }
- x64_asm_bound_reg(&bound_ins[i], type, pin.cls, pin.reg);
- } else {
- NativeAsmConstraintInfo info;
- if (native_asm_constraint_reg_info(d->native, ins[i].str, &info)) {
- Reg reg = info.fixed_reg != REG_NONE
- ? info.fixed_reg
- : x64_asm_alloc_reg(d, info.cls, info.allowed_mask,
- &used_int, &used_fp);
- if (info.cls == NATIVE_REG_FP) {
- used_fp |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
- } else {
- used_int |= 1u << reg;
- if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
- }
- x64_asm_bound_reg(&bound_ins[i], type, info.cls, reg);
- } else if (body[0] == 'i') {
- if (in_ops[i].kind != OPK_IMM)
- x64_asm_panic(d, "immediate constraint requires immediate operand");
- bound_ins[i] = in_ops[i];
- } else if (body[0] == 'm') {
- Reg reg = x64_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
- x64_asm_bound_mem(&bound_ins[i], type, reg);
- } else {
- x64_asm_panic(d, "unsupported input constraint");
- }
- }
- }
-
- saved = x64_asm_save_callee_clobbers(a, clob_int, clob_fp, &nsaved);
- for (i = 0; i < nout; ++i) {
- if (bound_outs[i].kind == X64_INLINE_OPK_REG) {
- NativeAllocClass cls = bound_outs[i].pad[0] == X64_INLINE_OPCLS_FP
- ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- if (outs[i].dir == KIT_CG_ASM_INOUT) {
- x64_direct_load_operand_to_reg(
- d, out_ops[i],
- native_loc_reg(bound_outs[i].type, cls,
- (Reg)bound_outs[i].v.local));
- }
- } else if (bound_outs[i].kind == OPK_INDIRECT) {
- NativeLoc loc =
- native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- (Reg)bound_outs[i].v.ind.base);
- x64_direct_load_address_to_reg(d, out_ops[i], loc);
- }
- }
- for (i = 0; i < nin; ++i) {
- if (bound_ins[i].kind == X64_INLINE_OPK_REG) {
- NativeAllocClass cls = bound_ins[i].pad[0] == X64_INLINE_OPCLS_FP
- ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- x64_direct_load_operand_to_reg(
- d, in_ops[i],
- native_loc_reg(bound_ins[i].type, cls, (Reg)bound_ins[i].v.local));
- } else if (bound_ins[i].kind == OPK_INDIRECT) {
- NativeLoc loc =
- native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- (Reg)bound_ins[i].v.ind.base);
- x64_direct_load_address_to_reg(d, in_ops[i], loc);
- }
- }
- asmh = x64_asm_open(c);
- x64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
- nclob);
- x64_asm_run_template(asmh, d->native->mc, tmpl);
- x64_asm_close(asmh);
-
- for (i = 0; i < nout; ++i) {
- NativeAllocClass cls;
- NativeLoc src;
- if (bound_outs[i].kind != X64_INLINE_OPK_REG) continue;
- cls = bound_outs[i].pad[0] == X64_INLINE_OPCLS_FP ? NATIVE_REG_FP
- : NATIVE_REG_INT;
- src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
- x64_direct_store_reg_to_operand(d, out_ops[i], src);
- }
- for (i = nsaved; i > 0; --i) x64_asm_restore_one(a, &saved[i - 1u]);
+ static const NativeAsmDirectHooks hooks = {
+ /* Reserve emit scratch (r10,r11), driver scratch (r8,r9), rax (reserved;
+ * only self-allocated here when explicitly pinned), and sp/bp. */
+ .scratch_int = (1u << X64_RAX) | (1u << X64_R11) | (1u << X64_RSP) |
+ (1u << X64_RBP) | (1u << X64_R8) | (1u << X64_R9) |
+ (1u << X64_R10),
+ .scratch_fp = (1u << X64_XMM4) | (1u << X64_XMM5) |
+ (1u << (X64_XMM0 + 14)) | (1u << X64_XMM15),
+ .opk_reg = X64_INLINE_OPK_REG,
+ .opcls_fp = X64_INLINE_OPCLS_FP,
+ .panic = x64_asm_panic,
+ .bound_reg = x64_asm_bound_reg,
+ .bound_mem = x64_asm_bound_mem,
+ .alloc_reg = x64_asm_alloc_reg,
+ .clobber_masks = x64_asm_clobber_masks,
+ .save_callee_clobbers = x64_asm_hook_save_callee_clobbers,
+ .restore_one = x64_asm_hook_restore_one,
+ .load_operand_to_reg = x64_direct_load_operand_to_reg,
+ .load_address_to_reg = x64_direct_load_address_to_reg,
+ .store_reg_to_operand = x64_direct_store_reg_to_operand,
+ .run_template = x64_asm_hook_run_template,
+ };
+ native_asm_bind_direct_operands(d, tmpl, outs, nout, out_ops, ins, nin,
+ in_ops, clobbers, nclob, clobber_abi_sets,
+ &hooks);
}
static const NativeOps x64_direct_ops = {
diff --git a/src/cg/arith.c b/src/cg/arith.c
@@ -1241,6 +1241,25 @@ static const char* api_softsf_binop_helper(KitCgFpBinOp op) {
return NULL;
}
+/* Soft-float binary arithmetic via a single libcall `name(a,b)`, both operands
+ * (and the result) of type `opty`. Consumes the two operands on the stack and
+ * pushes the result. Shared by the f128 (tf), soft-double (df) and soft-single
+ * (sf) paths — only the helper name and operand type differ; the runtime ABI
+ * (two same-type args, same-type result) is width-neutral. `what` names the
+ * width in the panic raised when `op` has no helper. Mirrors api_softfp_cmp. */
+static void api_softfp_binop(KitCg* g, const char* name, KitCgTypeId opty,
+ const char* what) {
+ KitCgTypeId ps[2];
+ ApiSValue args[2];
+ if (!name)
+ compiler_panic(g->c, g->cur_loc, "KitCg: unsupported %s binop", what);
+ args[1] = api_pop(g);
+ args[0] = api_pop(g);
+ ps[0] = opty;
+ ps[1] = opty;
+ api_runtime_call_values(g, name, opty, ps, 2, args);
+}
+
void api_f128_call_unary(KitCg* g, const char* name, KitCgTypeId ret,
KitCgTypeId param) {
ApiSValue args[1];
@@ -1253,45 +1272,18 @@ void api_f128_call_unary(KitCg* g, const char* name, KitCgTypeId ret,
void kit_cg_fp_binop(KitCg* g, KitCgFpBinOp op, uint32_t flags) {
(void)flags;
if (api_f128_stack_top(g, 0) || api_f128_stack_top(g, 1)) {
- KitCgTypeId f128 = builtin_id(KIT_CG_BUILTIN_F128);
- KitCgTypeId ps[2];
- ApiSValue args[2];
- const char* name = api_f128_binop_helper(op);
- if (!name)
- compiler_panic(g->c, g->cur_loc, "KitCg: unsupported f128 binop");
- args[1] = api_pop(g);
- args[0] = api_pop(g);
- ps[0] = f128;
- ps[1] = f128;
- api_runtime_call_values(g, name, f128, ps, 2, args);
+ api_softfp_binop(g, api_f128_binop_helper(op),
+ builtin_id(KIT_CG_BUILTIN_F128), "f128");
return;
}
if (api_soft_double_stack_top(g, 0) || api_soft_double_stack_top(g, 1)) {
- KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
- KitCgTypeId ps[2];
- ApiSValue args[2];
- const char* name = api_softdf_binop_helper(op);
- if (!name)
- compiler_panic(g->c, g->cur_loc, "KitCg: unsupported soft double binop");
- args[1] = api_pop(g);
- args[0] = api_pop(g);
- ps[0] = f64;
- ps[1] = f64;
- api_runtime_call_values(g, name, f64, ps, 2, args);
+ api_softfp_binop(g, api_softdf_binop_helper(op),
+ builtin_id(KIT_CG_BUILTIN_F64), "soft double");
return;
}
if (api_soft_single_stack_top(g, 0) || api_soft_single_stack_top(g, 1)) {
- KitCgTypeId f32 = builtin_id(KIT_CG_BUILTIN_F32);
- KitCgTypeId ps[2];
- ApiSValue args[2];
- const char* name = api_softsf_binop_helper(op);
- if (!name)
- compiler_panic(g->c, g->cur_loc, "KitCg: unsupported soft single binop");
- args[1] = api_pop(g);
- args[0] = api_pop(g);
- ps[0] = f32;
- ps[1] = f32;
- api_runtime_call_values(g, name, f32, ps, 2, args);
+ api_softfp_binop(g, api_softsf_binop_helper(op),
+ builtin_id(KIT_CG_BUILTIN_F32), "soft single");
return;
}
api_cg_binop(g, api_map_fp_binop(op), 0);
@@ -1525,19 +1517,69 @@ void kit_cg_fptrunc(KitCg* g, KitCgTypeId dst) {
api_cg_convert_kind(g, dst, CV_FTRUNC);
}
+/* The integer builtin a soft-float int<->float conversion uses for an operand
+ * whose ABI size is `sz` bytes: i128 (>8), i64 (>4), else i32. Shared by all
+ * eight conversion blocks (the same width ladder appeared in each). */
+static KitCgTypeId api_int_builtin_for_size(u32 sz) {
+ return sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
+ : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
+ : builtin_id(KIT_CG_BUILTIN_I32));
+}
+
+/* The compiler-rt integer-width suffix matching api_int_builtin_for_size:
+ * "ti" (i128), "di" (i64), "si" (i32). */
+static const char* api_int_suffix_for_size(u32 sz) {
+ return sz > 8 ? "ti" : (sz > 4 ? "di" : "si");
+}
+
+/* The four width-laddered soft-float int<->float conversion families. The
+ * compiler-rt name is a fixed prefix, the width-derived integer suffix
+ * (api_int_suffix_for_size) and a fixed float suffix ("tf" for f128, "df" for
+ * soft double); int->float places the integer suffix before the float suffix,
+ * float->int after it. */
+typedef enum ApiFpConvOp {
+ API_FPCONV_SINT_TO_FLOAT, /* __float<int><flt> */
+ API_FPCONV_UINT_TO_FLOAT, /* __floatun<int><flt> */
+ API_FPCONV_FLOAT_TO_SINT, /* __fix<flt><int> */
+ API_FPCONV_FLOAT_TO_UINT, /* __fixuns<flt><int> */
+} ApiFpConvOp;
+
+typedef struct FpConvDesc {
+ const char* prefix;
+ int int_first; /* 1: prefix+int+flt (int->float); 0: prefix+flt+int */
+} FpConvDesc;
+
+static const FpConvDesc kFpConvTable[] = {
+ [API_FPCONV_SINT_TO_FLOAT] = {"__float", 1},
+ [API_FPCONV_UINT_TO_FLOAT] = {"__floatun", 1},
+ [API_FPCONV_FLOAT_TO_SINT] = {"__fix", 0},
+ [API_FPCONV_FLOAT_TO_UINT] = {"__fixuns", 0},
+};
+
+/* Build the compiler-rt conversion libcall name for `op` with the given integer
+ * width (`sz` bytes) and float suffix `flt` ("tf"/"df") into `buf`. The result
+ * is byte-identical to the names the eight blocks previously open-coded. */
+static void api_fp_conv_name(char* buf, size_t cap, ApiFpConvOp op,
+ const char* flt, u32 sz) {
+ const FpConvDesc* d = &kFpConvTable[op];
+ const char* is = api_int_suffix_for_size(sz);
+ if (d->int_first)
+ snprintf(buf, cap, "%s%s%s", d->prefix, is, flt);
+ else
+ snprintf(buf, cap, "%s%s%s", d->prefix, flt, is);
+}
+
void kit_cg_sint_to_float(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
(void)rounding;
if (api_is_f128_type(g->c, resolve_type(g->c, dst))) {
ApiSValue v = api_pop(g);
KitCgTypeId sty = api_unalias_type(g->c, api_sv_type(&v));
u32 sz = (u32)abi_cg_sizeof(g->c->abi, sty);
- KitCgTypeId pty = sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
- : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
- : builtin_id(KIT_CG_BUILTIN_I32));
- const char* name =
- sz > 8 ? "__floattitf" : (sz > 4 ? "__floatditf" : "__floatsitf");
+ char name[16];
+ api_fp_conv_name(name, sizeof name, API_FPCONV_SINT_TO_FLOAT, "tf", sz);
api_push(g, v);
- api_f128_call_unary(g, name, resolve_type(g->c, dst), pty);
+ api_f128_call_unary(g, name, resolve_type(g->c, dst),
+ api_int_builtin_for_size(sz));
return;
}
/* signed int -> soft double: __floatsidf (i32) / __floatdidf (i64). */
@@ -1545,13 +1587,11 @@ void kit_cg_sint_to_float(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
ApiSValue v = api_pop(g);
KitCgTypeId sty = api_unalias_type(g->c, api_sv_type(&v));
u32 sz = (u32)abi_cg_sizeof(g->c->abi, sty);
- KitCgTypeId pty = sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
- : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
- : builtin_id(KIT_CG_BUILTIN_I32));
- const char* name =
- sz > 8 ? "__floattidf" : (sz > 4 ? "__floatdidf" : "__floatsidf");
+ char name[16];
+ api_fp_conv_name(name, sizeof name, API_FPCONV_SINT_TO_FLOAT, "df", sz);
api_push(g, v);
- api_f128_call_unary(g, name, resolve_type(g->c, dst), pty);
+ api_f128_call_unary(g, name, resolve_type(g->c, dst),
+ api_int_builtin_for_size(sz));
return;
}
/* signed split-i64 -> hardware single float: use __floatdisf. */
@@ -1575,13 +1615,11 @@ void kit_cg_uint_to_float(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
ApiSValue v = api_pop(g);
KitCgTypeId sty = api_unalias_type(g->c, api_sv_type(&v));
u32 sz = (u32)abi_cg_sizeof(g->c->abi, sty);
- KitCgTypeId pty = sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
- : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
- : builtin_id(KIT_CG_BUILTIN_I32));
- const char* name =
- sz > 8 ? "__floatuntitf" : (sz > 4 ? "__floatunditf" : "__floatunsitf");
+ char name[16];
+ api_fp_conv_name(name, sizeof name, API_FPCONV_UINT_TO_FLOAT, "tf", sz);
api_push(g, v);
- api_f128_call_unary(g, name, resolve_type(g->c, dst), pty);
+ api_f128_call_unary(g, name, resolve_type(g->c, dst),
+ api_int_builtin_for_size(sz));
return;
}
/* unsigned int -> soft double: __floatunsidf (i32) / __floatundidf (i64). */
@@ -1589,13 +1627,11 @@ void kit_cg_uint_to_float(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
ApiSValue v = api_pop(g);
KitCgTypeId sty = api_unalias_type(g->c, api_sv_type(&v));
u32 sz = (u32)abi_cg_sizeof(g->c->abi, sty);
- KitCgTypeId pty = sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
- : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
- : builtin_id(KIT_CG_BUILTIN_I32));
- const char* name =
- sz > 8 ? "__floatuntidf" : (sz > 4 ? "__floatundidf" : "__floatunsidf");
+ char name[16];
+ api_fp_conv_name(name, sizeof name, API_FPCONV_UINT_TO_FLOAT, "df", sz);
api_push(g, v);
- api_f128_call_unary(g, name, resolve_type(g->c, dst), pty);
+ api_f128_call_unary(g, name, resolve_type(g->c, dst),
+ api_int_builtin_for_size(sz));
return;
}
/* unsigned i64 -> hardware single float: __floatundisf. */
@@ -1618,11 +1654,9 @@ void kit_cg_float_to_sint(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
if (api_f128_stack_top(g, 0)) {
KitCgTypeId dty = resolve_type(g->c, dst);
u32 sz = (u32)abi_cg_sizeof(g->c->abi, dty);
- KitCgTypeId rty = sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
- : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
- : builtin_id(KIT_CG_BUILTIN_I32));
- const char* name =
- sz > 8 ? "__fixtfti" : (sz > 4 ? "__fixtfdi" : "__fixtfsi");
+ KitCgTypeId rty = api_int_builtin_for_size(sz);
+ char name[16];
+ api_fp_conv_name(name, sizeof name, API_FPCONV_FLOAT_TO_SINT, "tf", sz);
api_f128_call_unary(g, name, rty, builtin_id(KIT_CG_BUILTIN_F128));
if (rty != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
return;
@@ -1632,11 +1666,9 @@ void kit_cg_float_to_sint(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
KitCgTypeId dty = resolve_type(g->c, dst);
KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
u32 sz = (u32)abi_cg_sizeof(g->c->abi, dty);
- KitCgTypeId rty = sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
- : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
- : builtin_id(KIT_CG_BUILTIN_I32));
- const char* name =
- sz > 8 ? "__fixdfti" : (sz > 4 ? "__fixdfdi" : "__fixdfsi");
+ KitCgTypeId rty = api_int_builtin_for_size(sz);
+ char name[16];
+ api_fp_conv_name(name, sizeof name, API_FPCONV_FLOAT_TO_SINT, "df", sz);
api_f128_call_unary(g, name, rty, f64);
if (rty != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
return;
@@ -1663,11 +1695,9 @@ void kit_cg_float_to_uint(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
if (api_f128_stack_top(g, 0)) {
KitCgTypeId dty = resolve_type(g->c, dst);
u32 sz = (u32)abi_cg_sizeof(g->c->abi, dty);
- KitCgTypeId rty = sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
- : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
- : builtin_id(KIT_CG_BUILTIN_I32));
- const char* name =
- sz > 8 ? "__fixunstfti" : (sz > 4 ? "__fixunstfdi" : "__fixunstfsi");
+ KitCgTypeId rty = api_int_builtin_for_size(sz);
+ char name[16];
+ api_fp_conv_name(name, sizeof name, API_FPCONV_FLOAT_TO_UINT, "tf", sz);
api_f128_call_unary(g, name, rty, builtin_id(KIT_CG_BUILTIN_F128));
if (rty != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
return;
@@ -1677,11 +1707,9 @@ void kit_cg_float_to_uint(KitCg* g, KitCgTypeId dst, KitCgRounding rounding) {
KitCgTypeId dty = resolve_type(g->c, dst);
KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
u32 sz = (u32)abi_cg_sizeof(g->c->abi, dty);
- KitCgTypeId rty = sz > 8 ? builtin_id(KIT_CG_BUILTIN_I128)
- : (sz > 4 ? builtin_id(KIT_CG_BUILTIN_I64)
- : builtin_id(KIT_CG_BUILTIN_I32));
- const char* name =
- sz > 8 ? "__fixunsdfti" : (sz > 4 ? "__fixunsdfdi" : "__fixunsdfsi");
+ KitCgTypeId rty = api_int_builtin_for_size(sz);
+ char name[16];
+ api_fp_conv_name(name, sizeof name, API_FPCONV_FLOAT_TO_UINT, "df", sz);
api_f128_call_unary(g, name, rty, f64);
if (rty != dty) api_cg_convert_kind(g, dty, CV_TRUNC);
return;
diff --git a/src/cg/native_asm.c b/src/cg/native_asm.c
@@ -3,6 +3,9 @@
#include "arch/mc.h"
#include "asm/asm.h"
#include "asm/asm_lex.h"
+#include "cg/native_direct_target.h" /* NativeDirectTarget, for the direct binder */
+#include "cg/type.h" /* builtin_id */
+#include "core/arena.h" /* arena_zarray */
#include "core/pool.h" /* pool_slice for native_asm_resolve_pin */
void native_file_scope_asm(NativeTarget* t, const char* src, size_t len) {
@@ -202,3 +205,178 @@ const char* native_asm_pin_status_message(NativeAsmRegPinStatus st) {
}
return "invalid asm register variable";
}
+
+/* Resolve an operand's explicit hard-register pin, panicking (through the
+ * arch prefix) on any invalid pin. Returns 1 when a valid pin exists, 0 when
+ * the operand has no pin -- the same contract the former per-arch
+ * *_asm_resolve_pin_or_panic helpers had. */
+static int native_asm_direct_resolve_pin(NativeDirectTarget* d,
+ const NativeAsmDirectHooks* h, Sym reg,
+ const char* constraint,
+ NativeAsmRegPin* pin) {
+ NativeAsmRegPinStatus st =
+ native_asm_resolve_pin(d->native, reg, constraint, pin);
+ if (st == NATIVE_ASM_REG_PIN_ABSENT) return 0;
+ if (st != NATIVE_ASM_REG_PIN_OK)
+ h->panic(d, native_asm_pin_status_message(st));
+ return 1;
+}
+
+void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ Operand* out_ops, const AsmConstraint* ins,
+ u32 nin, const Operand* in_ops,
+ const Sym* clobbers, u32 nclob,
+ u32 clobber_abi_sets,
+ const NativeAsmDirectHooks* h) {
+ Compiler* c = d->base.c;
+ Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
+ Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
+ u32 clob_int, clob_fp, abi_int, abi_fp, used_int, used_fp;
+ void* saved;
+ u32 nsaved, i;
+
+ h->clobber_masks(c, d->loc, clobbers, nclob, &clob_int, &clob_fp);
+ native_asm_abi_clobber_masks(d->native, clobber_abi_sets, &abi_int, &abi_fp);
+ clob_int |= abi_int;
+ clob_fp |= abi_fp;
+ used_int = clob_int | h->scratch_int;
+ used_fp = clob_fp | h->scratch_fp;
+
+ for (i = 0; i < nout; ++i) {
+ const char* body = native_asm_constraint_body(outs[i].str);
+ KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
+ NativeAsmRegPin pin;
+ if (native_asm_direct_resolve_pin(d, h, outs[i].reg, outs[i].str, &pin)) {
+ /* GNU local register variable: pin to the named hard register. */
+ if (pin.cls == NATIVE_REG_FP) {
+ used_fp |= 1u << pin.reg;
+ clob_fp |= 1u << pin.reg;
+ } else {
+ used_int |= 1u << pin.reg;
+ clob_int |= 1u << pin.reg;
+ }
+ h->bound_reg(&bound_outs[i], type, pin.cls, pin.reg);
+ } else {
+ NativeAsmConstraintInfo info;
+ if (native_asm_constraint_reg_info(d->native, outs[i].str, &info)) {
+ Reg reg = info.fixed_reg != REG_NONE
+ ? info.fixed_reg
+ : h->alloc_reg(d, info.cls, info.allowed_mask, &used_int,
+ &used_fp);
+ if (info.cls == NATIVE_REG_FP) {
+ used_fp |= 1u << reg;
+ if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
+ } else {
+ used_int |= 1u << reg;
+ if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
+ }
+ h->bound_reg(&bound_outs[i], type, info.cls, reg);
+ } else if (body[0] == 'm') {
+ Reg reg = h->alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
+ h->bound_mem(&bound_outs[i], type, reg);
+ } else {
+ h->panic(d, "unsupported output constraint");
+ }
+ }
+ }
+
+ for (i = 0; i < nin; ++i) {
+ const char* body = native_asm_constraint_body(ins[i].str);
+ int matched = native_asm_match_index(body);
+ KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
+ if (matched >= 0) {
+ if ((u32)matched >= nout)
+ h->panic(d, "matching constraint out of range");
+ if (native_asm_constraint_early(outs[matched].str))
+ h->panic(d, "matching input names early-clobber output");
+ if (bound_outs[matched].kind != h->opk_reg)
+ h->panic(d, "matching constraint requires register output");
+ bound_ins[i] = bound_outs[matched];
+ continue;
+ }
+ NativeAsmRegPin pin;
+ if (native_asm_direct_resolve_pin(d, h, ins[i].reg, ins[i].str, &pin)) {
+ /* GNU local register variable: pin to the named hard register. */
+ if (pin.cls == NATIVE_REG_FP) {
+ used_fp |= 1u << pin.reg;
+ clob_fp |= 1u << pin.reg;
+ } else {
+ used_int |= 1u << pin.reg;
+ clob_int |= 1u << pin.reg;
+ }
+ h->bound_reg(&bound_ins[i], type, pin.cls, pin.reg);
+ } else {
+ NativeAsmConstraintInfo info;
+ if (native_asm_constraint_reg_info(d->native, ins[i].str, &info)) {
+ Reg reg = info.fixed_reg != REG_NONE
+ ? info.fixed_reg
+ : h->alloc_reg(d, info.cls, info.allowed_mask, &used_int,
+ &used_fp);
+ if (info.cls == NATIVE_REG_FP) {
+ used_fp |= 1u << reg;
+ if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
+ } else {
+ used_int |= 1u << reg;
+ if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
+ }
+ h->bound_reg(&bound_ins[i], type, info.cls, reg);
+ } else if (body[0] == 'i') {
+ if (in_ops[i].kind != OPK_IMM)
+ h->panic(d, "immediate constraint requires immediate operand");
+ bound_ins[i] = in_ops[i];
+ } else if (body[0] == 'm') {
+ Reg reg = h->alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
+ h->bound_mem(&bound_ins[i], type, reg);
+ } else {
+ h->panic(d, "unsupported input constraint");
+ }
+ }
+ }
+
+ saved = h->save_callee_clobbers(d, clob_int, clob_fp, &nsaved);
+ for (i = 0; i < nout; ++i) {
+ if (bound_outs[i].kind == h->opk_reg) {
+ NativeAllocClass cls =
+ bound_outs[i].pad[0] == h->opcls_fp ? NATIVE_REG_FP : NATIVE_REG_INT;
+ if (outs[i].dir == KIT_CG_ASM_INOUT) {
+ h->load_operand_to_reg(
+ d, out_ops[i],
+ native_loc_reg(bound_outs[i].type, cls,
+ (Reg)bound_outs[i].v.local));
+ }
+ } else if (bound_outs[i].kind == OPK_INDIRECT) {
+ NativeLoc loc =
+ native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
+ (Reg)bound_outs[i].v.ind.base);
+ h->load_address_to_reg(d, out_ops[i], loc);
+ }
+ }
+ for (i = 0; i < nin; ++i) {
+ if (bound_ins[i].kind == h->opk_reg) {
+ NativeAllocClass cls =
+ bound_ins[i].pad[0] == h->opcls_fp ? NATIVE_REG_FP : NATIVE_REG_INT;
+ h->load_operand_to_reg(
+ d, in_ops[i],
+ native_loc_reg(bound_ins[i].type, cls, (Reg)bound_ins[i].v.local));
+ } else if (bound_ins[i].kind == OPK_INDIRECT) {
+ NativeLoc loc =
+ native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
+ (Reg)bound_ins[i].v.ind.base);
+ h->load_address_to_reg(d, in_ops[i], loc);
+ }
+ }
+
+ h->run_template(d, tmpl, outs, nout, bound_outs, ins, nin, bound_ins,
+ clobbers, nclob);
+
+ for (i = 0; i < nout; ++i) {
+ NativeAllocClass cls;
+ NativeLoc src;
+ if (bound_outs[i].kind != h->opk_reg) continue;
+ cls = bound_outs[i].pad[0] == h->opcls_fp ? NATIVE_REG_FP : NATIVE_REG_INT;
+ src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
+ h->store_reg_to_operand(d, out_ops[i], src);
+ }
+ for (i = nsaved; i > 0; --i) h->restore_one(d, saved, i - 1u);
+}
diff --git a/src/cg/native_asm.h b/src/cg/native_asm.h
@@ -92,4 +92,79 @@ const char* native_asm_pin_status_message(NativeAsmRegPinStatus st);
int native_asm_constraint_reg_class(const char* constraint,
NativeAllocClass* cls_out);
+/* ---- Direct (-O0) inline-asm operand binding ----
+ *
+ * The direct CgTarget path lowers an inline-asm block by self-allocating a
+ * register/memory location for every operand, saving/restoring the
+ * callee-saved registers the asm clobbers, pre-loading INOUT outputs and all
+ * inputs into their bound registers, running the template, then storing the
+ * register outputs back. That orchestration is arch-neutral: it was a
+ * ~170-line near-identical copy in each of the aa64/x64/rv64 backends. It lives
+ * here as the single source of truth, parameterized over the genuinely
+ * arch-specific bits via NativeAsmDirectHooks; each backend's *_direct_asm_block
+ * is a thin wrapper that supplies the hooks.
+ *
+ * The shared driver depends on NativeDirectTarget only through pointers, so the
+ * full struct is needed at the call site (the .c includes it), not here. */
+typedef struct NativeDirectTarget NativeDirectTarget;
+
+typedef struct NativeAsmDirectHooks {
+ /* Registers the operand allocator must never hand out (emit scratch, ABI
+ * fixed regs, frame pointer, ...), seeded into the used masks alongside the
+ * clobber masks. Per-arch. */
+ u32 scratch_int;
+ u32 scratch_fp;
+ /* Arch inline-operand discriminators: Operand.kind for a bound register
+ * pseudo-operand, and the Operand.pad[0] value marking the FP class. */
+ u8 opk_reg;
+ u8 opcls_fp;
+ u8 pad[2];
+
+ /* Panic with the arch's inline-asm message prefix. Does not return. */
+ void (*panic)(NativeDirectTarget* d, const char* msg);
+ /* Build the arch's bound register / memory pseudo-operand. */
+ void (*bound_reg)(Operand* out, KitCgTypeId type, NativeAllocClass cls,
+ Reg reg);
+ void (*bound_mem)(Operand* out, KitCgTypeId type, Reg base);
+ /* Pick a free register from the arch's allocable pool for cls, honoring
+ * allowed_mask and the running used masks (which it updates). Panics on
+ * exhaustion. */
+ Reg (*alloc_reg)(NativeDirectTarget* d, NativeAllocClass cls, u32 allowed_mask,
+ u32* used_int, u32* used_fp);
+ /* Parse the asm clobber list into per-class register masks (arch register
+ * naming). */
+ void (*clobber_masks)(Compiler* c, SrcLoc loc, const Sym* clobbers, u32 nclob,
+ u32* int_mask, u32* fp_mask);
+ /* Spill the callee-saved registers the asm clobbers, returning an opaque,
+ * block-lived save record plus its element count. restore_one undoes element
+ * idx; the driver replays them in reverse. */
+ void* (*save_callee_clobbers)(NativeDirectTarget* d, u32 int_mask, u32 fp_mask,
+ u32* nsaved_out);
+ void (*restore_one)(NativeDirectTarget* d, void* saved, u32 idx);
+ /* Load a semantic operand's value / address into a register, or store a
+ * register back to a semantic operand. */
+ void (*load_operand_to_reg)(NativeDirectTarget* d, Operand op, NativeLoc dst);
+ void (*load_address_to_reg)(NativeDirectTarget* d, Operand op, NativeLoc dst);
+ void (*store_reg_to_operand)(NativeDirectTarget* d, Operand op, NativeLoc src);
+ /* Open the arch asm assembler, bind the resolved operands, run the template,
+ * and close — emitting the asm body into the target's MCEmitter. */
+ void (*run_template)(NativeDirectTarget* d, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ Operand* bound_outs, const AsmConstraint* ins,
+ u32 nin, Operand* bound_ins, const Sym* clobbers,
+ u32 nclob);
+} NativeAsmDirectHooks;
+
+/* Drive the arch-neutral direct inline-asm orchestration: out-loop / in-loop /
+ * save / load / run / store / restore. Behavior is identical to the former
+ * per-arch *_direct_asm_block bodies; the hooks supply every arch-specific
+ * step. */
+void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
+ const AsmConstraint* outs, u32 nout,
+ Operand* out_ops, const AsmConstraint* ins,
+ u32 nin, const Operand* in_ops,
+ const Sym* clobbers, u32 nclob,
+ u32 clobber_abi_sets,
+ const NativeAsmDirectHooks* h);
+
#endif
