commit 618d04fce07df20407a1a5f54c3cd41415e8ee37
parent 148b695cab846884c7340c153f645c3a6291866a
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 18 Jul 2026 08:23:33 -0700
Add portable ISA builtins
Diffstat:
27 files changed, 931 insertions(+), 7 deletions(-)
diff --git a/doc/plan/TODO.md b/doc/plan/TODO.md
@@ -134,6 +134,12 @@ doc at `HEAD~`).
always builds a native `Linker`. Same-invocation source batches are merged
before final module emission; independently emitted Wasm objects still need
linker metadata + relocation apply.
+- **Runtime helpers in final modules**: an undefined helper call currently
+ becomes a host import, so non-trivial fallbacks such as 64-bit multiply-high
+ are expanded at every use site. Short term, intern private module-local
+ helpers and emit each body once (keeping cheap operations inline); then teach
+ the source-batch path to include the `wasm32` runtime sources. Once relocatable
+ Wasm linking lands, resolve these calls normally from `libkit_rt.a`.
- **Feature gaps**: atomics, wrapper ABI; frontend lowering beyond the staged
MVP; validator diagnostics for unsupported proposals.
- **wasm64 + WASI**: wasm64 is a reserved spelling rejected by target
diff --git a/include/kit/cg.h b/include/kit/cg.h
@@ -1329,6 +1329,11 @@ typedef enum KitCgIntrinsic {
* riscv64 RDCYCLE). Targets without a single-register 64-bit counter read
* report unsupported. */
KIT_CG_INTRIN_READCYCLECOUNTER, /* push u64 */
+ /* High half of a full-width integer product. Both pop two equal-width
+ * integer operands and push a result of that width. SMUL_HIGH interprets
+ * both operands as two's-complement signed; UMUL_HIGH is unsigned. */
+ KIT_CG_INTRIN_SMUL_HIGH,
+ KIT_CG_INTRIN_UMUL_HIGH,
} KitCgIntrinsic;
typedef enum KitCgBarrierScope {
diff --git a/lang/c/parse/parse_expr.c b/lang/c/parse/parse_expr.c
@@ -1090,6 +1090,350 @@ static FrameSlot builtin_tmp_slot(Parser* p, const Type* ty) {
return c_cg_local(p, &fsd);
}
+static void builtin_store_top(Parser* p, FrameSlot slot, const Type* ty) {
+ c_cg_push_local_typed(p, slot, ty);
+ c_cg_swap(p);
+ c_cg_store_void(p);
+}
+
+static void builtin_load_slot(Parser* p, FrameSlot slot, const Type* ty) {
+ c_cg_push_local_typed(p, slot, ty);
+ c_cg_load(p);
+}
+
+typedef struct BuiltinBitInfo {
+ const char* name;
+ KitCgIntrinsic intrin;
+ TypeKind arg_kind;
+ TypeKind result_kind;
+} BuiltinBitInfo;
+
+static int parse_builtin_bit_call(Parser* p, Sym name, SrcLoc loc) {
+ static const BuiltinBitInfo infos[] = {
+ {"__builtin_popcount", KIT_CG_INTRIN_POPCOUNT, TY_UINT, TY_INT},
+ {"__builtin_popcountl", KIT_CG_INTRIN_POPCOUNT, TY_ULONG, TY_INT},
+ {"__builtin_popcountll", KIT_CG_INTRIN_POPCOUNT, TY_ULLONG, TY_INT},
+ {"__builtin_bswap16", KIT_CG_INTRIN_BSWAP, TY_USHORT, TY_USHORT},
+ {"__builtin_bswap32", KIT_CG_INTRIN_BSWAP, TY_UINT, TY_UINT},
+ {"__builtin_bswap64", KIT_CG_INTRIN_BSWAP, TY_ULLONG, TY_ULLONG},
+ };
+ const BuiltinBitInfo* info = NULL;
+ const Type* arg_ty;
+ const Type* result_ty;
+ size_t i;
+ for (i = 0; i < sizeof infos / sizeof infos[0]; ++i) {
+ if (sym_eq_cstr(p, name, infos[i].name)) {
+ info = &infos[i];
+ break;
+ }
+ }
+ if (!info) return 0;
+ arg_ty = type_prim(p->pool, info->arg_kind);
+ result_ty = type_prim(p->pool, info->result_kind);
+ advance(p);
+ expect_punct(p, '(', "'(' after bit builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ coerce_top_to_type(p, arg_ty);
+ expect_punct(p, ')', "')' after bit builtin");
+ c_cg_set_loc(p, loc);
+ kit_cg_intrinsic(p->cg, info->intrin, 1, c_cg_tid(p, result_ty));
+ c_cg_retag_top(p, result_ty);
+ return 1;
+}
+
+typedef struct BuiltinRotateInfo {
+ const char* name;
+ TypeKind type_kind;
+ u8 left;
+} BuiltinRotateInfo;
+
+static int parse_builtin_rotate_call(Parser* p, Sym name) {
+ static const BuiltinRotateInfo infos[] = {
+ {"__builtin_rotateleft8", TY_UCHAR, 1},
+ {"__builtin_rotateleft16", TY_USHORT, 1},
+ {"__builtin_rotateleft32", TY_UINT, 1},
+ {"__builtin_rotateleft64", TY_ULLONG, 1},
+ {"__builtin_rotateright8", TY_UCHAR, 0},
+ {"__builtin_rotateright16", TY_USHORT, 0},
+ {"__builtin_rotateright32", TY_UINT, 0},
+ {"__builtin_rotateright64", TY_ULLONG, 0},
+ };
+ const BuiltinRotateInfo* info = NULL;
+ const Type* ty;
+ FrameSlot value_slot;
+ FrameSlot count_slot;
+ u32 bits;
+ size_t i;
+ for (i = 0; i < sizeof infos / sizeof infos[0]; ++i) {
+ if (sym_eq_cstr(p, name, infos[i].name)) {
+ info = &infos[i];
+ break;
+ }
+ }
+ if (!info) return 0;
+ ty = type_prim(p->pool, info->type_kind);
+ bits = integer_type_bits(p, ty);
+ value_slot = builtin_tmp_slot(p, ty);
+ count_slot = builtin_tmp_slot(p, ty);
+
+ advance(p);
+ expect_punct(p, '(', "'(' after rotate builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ coerce_top_to_type(p, ty);
+ builtin_store_top(p, value_slot, ty);
+ expect_punct(p, ',', "',' in rotate builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ coerce_top_to_type(p, ty);
+ builtin_store_top(p, count_slot, ty);
+ expect_punct(p, ')', "')' after rotate builtin");
+
+ /* CG shifts with KIT_CG_INTOP_NONE reduce the count modulo the operand
+ * width. Thus width-count deliberately becomes a zero shift when count is
+ * zero, avoiding the undefined full-width shift in the equivalent C idiom. */
+ builtin_load_slot(p, value_slot, ty);
+ builtin_load_slot(p, count_slot, ty);
+ c_cg_binop(p, info->left ? BO_SHL : BO_SHR_U);
+ builtin_load_slot(p, value_slot, ty);
+ c_cg_push_int(p, (i64)bits, ty);
+ builtin_load_slot(p, count_slot, ty);
+ c_cg_binop(p, BO_ISUB);
+ c_cg_binop(p, info->left ? BO_SHR_U : BO_SHL);
+ c_cg_binop(p, BO_OR);
+ return 1;
+}
+
+static int parse_builtin_prefetch_call(Parser* p, Sym name, SrcLoc loc) {
+ const Type* ptr_ty;
+ u32 nargs = 1;
+ i64 rw = 0;
+ i64 locality = 3;
+ if (!sym_eq_cstr(p, name, "__builtin_prefetch")) return 0;
+ advance(p);
+ expect_punct(p, '(', "'(' after __builtin_prefetch");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ ptr_ty = c_cg_top_type(p);
+ if (!ptr_ty || ptr_ty->kind != TY_PTR)
+ perr(p, "__builtin_prefetch address must be a pointer");
+ if (accept_punct(p, ',')) {
+ rw = eval_const_int(p, pp_materialize_loc(p->pp, p->cur.loc));
+ if (rw < 0 || rw > 1) perr(p, "__builtin_prefetch rw must be 0 or 1");
+ c_cg_push_int(p, rw, ty_int(p));
+ nargs = 2;
+ if (accept_punct(p, ',')) {
+ locality = eval_const_int(p, pp_materialize_loc(p->pp, p->cur.loc));
+ if (locality < 0 || locality > 3)
+ perr(p, "__builtin_prefetch locality must be in [0, 3]");
+ c_cg_push_int(p, locality, ty_int(p));
+ nargs = 3;
+ }
+ }
+ expect_punct(p, ')', "')' after __builtin_prefetch");
+ c_cg_set_loc(p, loc);
+ kit_cg_intrinsic(p->cg, KIT_CG_INTRIN_PREFETCH, nargs,
+ c_cg_tid(p, type_void(p->pool)));
+ c_cg_push_int(p, 0, ty_int(p));
+ return 1;
+}
+
+static int parse_builtin_assume_aligned_call(Parser* p, Sym name, SrcLoc loc) {
+ const Type* ptr_ty;
+ i64 align;
+ i64 offset = 0;
+ u32 nargs = 2;
+ if (!sym_eq_cstr(p, name, "__builtin_assume_aligned")) return 0;
+ advance(p);
+ expect_punct(p, '(', "'(' after __builtin_assume_aligned");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ ptr_ty = c_cg_top_type(p);
+ if (!ptr_ty || ptr_ty->kind != TY_PTR)
+ perr(p, "__builtin_assume_aligned argument must be a pointer");
+ expect_punct(p, ',', "',' in __builtin_assume_aligned");
+ align = eval_const_int(p, pp_materialize_loc(p->pp, p->cur.loc));
+ if (align <= 0 || ((u64)align & ((u64)align - 1u)) != 0)
+ perr(p, "__builtin_assume_aligned alignment must be a power of two");
+ c_cg_push_int(p, align, ty_size_t(p));
+ if (accept_punct(p, ',')) {
+ offset = eval_const_int(p, pp_materialize_loc(p->pp, p->cur.loc));
+ c_cg_push_int(p, offset, ty_size_t(p));
+ nargs = 3;
+ }
+ expect_punct(p, ')', "')' after __builtin_assume_aligned");
+ c_cg_set_loc(p, loc);
+ kit_cg_intrinsic(p->cg, KIT_CG_INTRIN_ASSUME_ALIGNED, nargs,
+ c_cg_tid(p, ptr_ty));
+ c_cg_retag_top(p, ptr_ty);
+ return 1;
+}
+
+static int parse_builtin_cpu_relax_call(Parser* p, Sym name, SrcLoc loc) {
+ if (!sym_eq_cstr(p, name, "__builtin_kit_cpu_relax")) return 0;
+ advance(p);
+ expect_punct(p, '(', "'(' after __builtin_kit_cpu_relax");
+ expect_punct(p, ')', "')' after __builtin_kit_cpu_relax");
+ c_cg_set_loc(p, loc);
+ kit_cg_intrinsic(p->cg, KIT_CG_INTRIN_CPU_YIELD, 0,
+ c_cg_tid(p, type_void(p->pool)));
+ c_cg_push_int(p, 0, ty_int(p));
+ return 1;
+}
+
+static int parse_builtin_target_has_call(Parser* p, Sym name) {
+ Tok tok;
+ u8* bytes;
+ size_t nbytes = 0;
+ int enabled;
+ Heap* h;
+ if (!sym_eq_cstr(p, name, "__builtin_kit_target_has")) return 0;
+ advance(p);
+ expect_punct(p, '(', "'(' after __builtin_kit_target_has");
+ if (p->cur.kind != TOK_STR ||
+ (p->cur.flags & (TF_STR_WIDE | TF_STR_U16 | TF_STR_U32)))
+ perr(p, "__builtin_kit_target_has expects an ordinary string literal");
+ tok = p->cur;
+ bytes = decode_string_literal(p, &tok, &nbytes);
+ advance(p);
+ expect_punct(p, ')', "')' after __builtin_kit_target_has");
+ enabled = kit_target_has_feature(
+ kit_compiler_target(p->c),
+ (KitSlice){.s = (const char*)bytes, .len = nbytes ? nbytes - 1u : 0u});
+ h = kit_compiler_context(p->c)->heap;
+ h->free(h, bytes, 0);
+ c_cg_push_int(p, enabled, ty_int(p));
+ return 1;
+}
+
+typedef struct BuiltinCarryInfo {
+ const char* name;
+ TypeKind type_kind;
+ u8 subtract;
+} BuiltinCarryInfo;
+
+static int parse_builtin_carry_call(Parser* p, Sym name, SrcLoc loc) {
+ static const BuiltinCarryInfo infos[] = {
+ {"__builtin_addc", TY_UINT, 0}, {"__builtin_addcl", TY_ULONG, 0},
+ {"__builtin_addcll", TY_ULLONG, 0}, {"__builtin_subc", TY_UINT, 1},
+ {"__builtin_subcl", TY_ULONG, 1}, {"__builtin_subcll", TY_ULLONG, 1},
+ };
+ const BuiltinCarryInfo* info = NULL;
+ const Type* ty;
+ const Type* bool_ty = type_prim(p->pool, TY_BOOL);
+ const Type* ptr_ty;
+ FrameSlot a_slot, b_slot, in_slot, ptr_slot, value_slot, flag1_slot;
+ KitCgIntrinsic intrin;
+ size_t i;
+ for (i = 0; i < sizeof infos / sizeof infos[0]; ++i) {
+ if (sym_eq_cstr(p, name, infos[i].name)) {
+ info = &infos[i];
+ break;
+ }
+ }
+ if (!info) return 0;
+ ty = type_prim(p->pool, info->type_kind);
+ a_slot = builtin_tmp_slot(p, ty);
+ b_slot = builtin_tmp_slot(p, ty);
+ in_slot = builtin_tmp_slot(p, ty);
+ value_slot = builtin_tmp_slot(p, ty);
+ flag1_slot = builtin_tmp_slot(p, bool_ty);
+ intrin = info->subtract ? KIT_CG_INTRIN_USUB_OVERFLOW
+ : KIT_CG_INTRIN_UADD_OVERFLOW;
+
+ advance(p);
+ expect_punct(p, '(', "'(' after carry builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ coerce_top_to_type(p, ty);
+ builtin_store_top(p, a_slot, ty);
+ expect_punct(p, ',', "',' in carry builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ coerce_top_to_type(p, ty);
+ builtin_store_top(p, b_slot, ty);
+ expect_punct(p, ',', "',' in carry builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ coerce_top_to_type(p, ty);
+ builtin_store_top(p, in_slot, ty);
+ expect_punct(p, ',', "',' in carry builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ ptr_ty = c_cg_top_type(p);
+ if (!ptr_ty || ptr_ty->kind != TY_PTR ||
+ !type_compatible(type_unqual(p->pool, ptr_ty->ptr.pointee), ty))
+ perr(p, "carry builtin output must point to its unsigned operand type");
+ ptr_slot = builtin_tmp_slot(p, ptr_ty);
+ builtin_store_top(p, ptr_slot, ptr_ty);
+ expect_punct(p, ')', "')' after carry builtin");
+
+ c_cg_set_loc(p, loc);
+ builtin_load_slot(p, a_slot, ty);
+ builtin_load_slot(p, b_slot, ty);
+ kit_cg_intrinsic(p->cg, intrin, 2, c_cg_tid(p, ty));
+ c_cg_retag_at(p, 1, ty, 0);
+ c_cg_retag_top(p, bool_ty);
+ builtin_store_top(p, flag1_slot, bool_ty);
+ builtin_store_top(p, value_slot, ty);
+
+ builtin_load_slot(p, value_slot, ty);
+ builtin_load_slot(p, in_slot, ty);
+ kit_cg_intrinsic(p->cg, intrin, 2, c_cg_tid(p, ty));
+ c_cg_retag_at(p, 1, ty, 0);
+ c_cg_retag_top(p, bool_ty);
+ builtin_load_slot(p, flag1_slot, bool_ty);
+ c_cg_binop(p, BO_OR);
+ c_cg_convert(p, ty);
+
+ builtin_load_slot(p, ptr_slot, ptr_ty);
+ c_cg_deref(p, ty);
+ c_cg_swap(p);
+ c_cg_store_void(p);
+ return 1; /* the second arithmetic result remains below the stored flag */
+}
+
+typedef struct BuiltinMulHighInfo {
+ const char* name;
+ TypeKind type_kind;
+ KitCgIntrinsic intrin;
+} BuiltinMulHighInfo;
+
+static int parse_builtin_mul_high_call(Parser* p, Sym name, SrcLoc loc) {
+ static const BuiltinMulHighInfo infos[] = {
+ {"__builtin_kit_umul_high32", TY_UINT, KIT_CG_INTRIN_UMUL_HIGH},
+ {"__builtin_kit_umul_high64", TY_ULLONG, KIT_CG_INTRIN_UMUL_HIGH},
+ {"__builtin_kit_smul_high32", TY_INT, KIT_CG_INTRIN_SMUL_HIGH},
+ {"__builtin_kit_smul_high64", TY_LLONG, KIT_CG_INTRIN_SMUL_HIGH},
+ };
+ const BuiltinMulHighInfo* info = NULL;
+ const Type* ty;
+ size_t i;
+ for (i = 0; i < sizeof infos / sizeof infos[0]; ++i) {
+ if (sym_eq_cstr(p, name, infos[i].name)) {
+ info = &infos[i];
+ break;
+ }
+ }
+ if (!info) return 0;
+ ty = type_prim(p->pool, info->type_kind);
+ advance(p);
+ expect_punct(p, '(', "'(' after multiply-high builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ coerce_top_to_type(p, ty);
+ expect_punct(p, ',', "',' in multiply-high builtin");
+ parse_assign_expr(p);
+ to_rvalue(p);
+ coerce_top_to_type(p, ty);
+ expect_punct(p, ')', "')' after multiply-high builtin");
+ c_cg_set_loc(p, loc);
+ kit_cg_intrinsic(p->cg, info->intrin, 2, c_cg_tid(p, ty));
+ c_cg_retag_top(p, ty);
+ return 1;
+}
+
/* The type-generic overflow builtins (no s/u + width suffix). They infer the
* operation type from the result pointer's pointee and the signed/unsigned
* intrinsic from that type's signedness, then reuse the same per-type
@@ -1531,6 +1875,10 @@ static int try_parse_builtin_call(Parser* p) {
Sym name = tok_ident(&p->cur);
SrcLoc loc = pp_materialize_loc(p->pp, p->cur.loc);
+ /* A resolved-target feature query is a parser-folded integer constant, not a
+ * runtime call, so it remains valid inside integer constant expressions. */
+ if (parse_builtin_target_has_call(p, name)) return 1;
+
if (c_const_guard_active(p) && name != p->sym_b_offsetof &&
name != p->sym_b_constant_p) {
c_const_guard_note_at(p, loc,
@@ -1539,6 +1887,14 @@ static int try_parse_builtin_call(Parser* p) {
if (parse_kit_syscall_call(p, name, loc)) return 1;
+ if (parse_builtin_bit_call(p, name, loc)) return 1;
+ if (parse_builtin_rotate_call(p, name)) return 1;
+ if (parse_builtin_prefetch_call(p, name, loc)) return 1;
+ if (parse_builtin_assume_aligned_call(p, name, loc)) return 1;
+ if (parse_builtin_cpu_relax_call(p, name, loc)) return 1;
+ if (parse_builtin_carry_call(p, name, loc)) return 1;
+ if (parse_builtin_mul_high_call(p, name, loc)) return 1;
+
if (name == p->sym_b_memcpy || name == p->sym_b_memmove ||
name == p->sym_b_memcmp || name == p->sym_b_memset) {
return parse_builtin_mem_call(p, name, loc);
diff --git a/rt/lib/int32/int32.c b/rt/lib/int32/int32.c
@@ -113,3 +113,33 @@ COMPILER_RT_ABI di_int __muldi3(di_int a, di_int b) {
r.s.high += x.s.high * y.s.low + x.s.low * y.s.high;
return r.all;
}
+
+/* High halves of a full 64x64 product. These are Kit runtime entry points for
+ * 32-bit targets, where a CG intrinsic operand is a register pair rather than
+ * the single register used by the native intrinsic dispatch. */
+COMPILER_RT_ABI du_int __kit_umulhdi3(du_int a, du_int b) {
+ const int word_bits = (int)(sizeof(su_int) * CHAR_BIT);
+ udwords x;
+ udwords y;
+ du_int p00;
+ du_int p01;
+ du_int p10;
+ du_int p11;
+ du_int middle;
+ x.all = a;
+ y.all = b;
+ p00 = (du_int)x.s.low * (du_int)y.s.low;
+ p01 = (du_int)x.s.low * (du_int)y.s.high;
+ p10 = (du_int)x.s.high * (du_int)y.s.low;
+ p11 = (du_int)x.s.high * (du_int)y.s.high;
+ middle = (p00 >> word_bits) + (su_int)p01 + (su_int)p10;
+ return p11 + (p01 >> word_bits) + (p10 >> word_bits) +
+ (middle >> word_bits);
+}
+
+COMPILER_RT_ABI di_int __kit_mulhdi3(di_int a, di_int b) {
+ du_int hi = __kit_umulhdi3((du_int)a, (du_int)b);
+ if (a < 0) hi -= (du_int)b;
+ if (b < 0) hi -= (du_int)a;
+ return (di_int)hi;
+}
diff --git a/src/arch/aa64/arch.c b/src/arch/aa64/arch.c
@@ -188,6 +188,8 @@ static int aa64_supports_intrinsic(const Compiler* c, KitCgIntrinsic intrin) {
case KIT_CG_INTRIN_FRAME_ADDRESS:
case KIT_CG_INTRIN_RETURN_ADDRESS:
case KIT_CG_INTRIN_READCYCLECOUNTER: /* MRS CNTVCT_EL0 */
+ case KIT_CG_INTRIN_SMUL_HIGH:
+ case KIT_CG_INTRIN_UMUL_HIGH:
return 1;
case KIT_CG_INTRIN_SYSCALL:
return c->target.os == KIT_OS_LINUX ||
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -597,6 +597,12 @@ static u32 aa_str_uimm(u32 size, u32 rt, u32 rn, u32 byte_off) {
return aa_str_uimm_v(size, 0, rt, rn, byte_off);
}
+/* PRFM (immediate), zero byte offset. prfop is the architectural Rt field:
+ * type[4:3] | level[2:1] | policy[0]. */
+static u32 aa_prfm(u32 prfop, u32 rn) {
+ return 0xF9800000u | ((rn & 31u) << 5) | (prfop & 31u);
+}
+
/* Register-offset load/store with an explicit load opcode (ld_opc is consulted
* only when load != 0; AA64_LDST_OPC_LDRS_X gives a sign-extending ldrsb/ldrsh
* into the X register) and an explicit index-extend `option`
@@ -4288,6 +4294,29 @@ static void aa_intrinsic(NativeTarget* t, IntrinKind kind,
return;
}
break;
+ case INTRIN_SMUL_HIGH:
+ case INTRIN_UMUL_HIGH:
+ if (ndst == 1u && narg == 2u) {
+ u32 sf = loc_is_64(t, dsts[0]);
+ u32 rd = loc_reg(dsts[0]);
+ u32 rn = loc_reg(args[0]);
+ u32 rm = loc_reg(args[1]);
+ if (sf) {
+ aa_emit32(t->mc, kind == INTRIN_SMUL_HIGH
+ ? aa_smulh(rd, rn, rm)
+ : aa_umulh(rd, rn, rm));
+ } else {
+ aa_emit32(t->mc, kind == INTRIN_SMUL_HIGH
+ ? aa_smaddl(AA_TMP0, rn, rm, AA64_ZR)
+ : aa_umaddl(AA_TMP0, rn, rm, AA64_ZR));
+ if (kind == INTRIN_SMUL_HIGH)
+ aa_asr_imm(t, 1, rd, AA_TMP0, 32);
+ else
+ aa_lsr_imm(t, 1, rd, AA_TMP0, 32);
+ }
+ return;
+ }
+ break;
case INTRIN_MEMMOVE: {
MCLabel forward = mc_label_new(t->mc);
MCLabel done = mc_label_new(t->mc);
@@ -4323,6 +4352,20 @@ static void aa_intrinsic(NativeTarget* t, IntrinKind kind,
}
return;
case INTRIN_PREFETCH:
+ if (narg >= 1u && args[0].kind == NATIVE_LOC_REG) {
+ /* GCC locality 0..3 maps to streaming L1, then L3/L2/L1 keep. PST
+ * differs from PLD by bit 3 of prfop; both are baseline A64 hints. */
+ static const u8 op[4] = {1u, 4u, 2u, 0u};
+ u32 rw = 0u;
+ u32 locality = 3u;
+ if (narg >= 2u && args[1].kind == NATIVE_LOC_IMM)
+ rw = (u32)args[1].v.imm;
+ if (narg >= 3u && args[2].kind == NATIVE_LOC_IMM)
+ locality = (u32)args[2].v.imm;
+ if (locality > 3u) locality = 3u;
+ aa_emit32(t->mc, aa_prfm(op[locality] | (rw ? 8u : 0u),
+ loc_reg(args[0])));
+ }
return;
case INTRIN_TRAP:
aa_trap(t);
diff --git a/src/arch/arm32/arch.c b/src/arch/arm32/arch.c
@@ -435,6 +435,8 @@ static int arm32_supports_intrinsic(const Compiler* c, KitCgIntrinsic intrin) {
case KIT_CG_INTRIN_PREFETCH:
case KIT_CG_INTRIN_EXPECT:
case KIT_CG_INTRIN_ASSUME_ALIGNED:
+ case KIT_CG_INTRIN_SMUL_HIGH:
+ case KIT_CG_INTRIN_UMUL_HIGH:
return 1;
case KIT_CG_INTRIN_READCYCLECOUNTER:
case KIT_CG_INTRIN_SYSCALL:
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -3184,6 +3184,19 @@ static void arm_intrinsic(NativeTarget* t, IntrinKind kind,
return;
}
break;
+ case INTRIN_SMUL_HIGH:
+ case INTRIN_UMUL_HIGH:
+ if (ndst == 1u && narg == 2u) {
+ u32 rd = loc_reg(dsts[0]);
+ u32 ra = loc_reg(args[0]);
+ u32 rb = loc_reg(args[1]);
+ arm_emit_t32(mc, kind == INTRIN_SMUL_HIGH
+ ? arm_smull(ARM_TMP, ARM_SCRATCH, ra, rb)
+ : arm_umull(ARM_TMP, ARM_SCRATCH, ra, rb));
+ arm_emit_t16(mc, arm_mov_hi(rd, ARM_SCRATCH));
+ return;
+ }
+ break;
case INTRIN_FRAME_ADDRESS:
case INTRIN_RETURN_ADDRESS:
/* kit's prologue anchors r7 at the saved pair: [r7]=caller r7,
diff --git a/src/arch/c_target/c_emit.c b/src/arch/c_target/c_emit.c
@@ -2525,6 +2525,42 @@ void c_emit_intrinsic(CTarget* t, IntrinKind k, Operand* dsts, u32 ndst,
c_emit_local_assign_close(t);
return;
}
+ case INTRIN_SMUL_HIGH:
+ case INTRIN_UMUL_HIGH: {
+ u32 width;
+ int is_signed = k == INTRIN_SMUL_HIGH;
+ if (ndst != 1 || narg != 2) {
+ compiler_panic(t->c, loc,
+ "C target: mul-high: bad shape (ndst=%u narg=%u)",
+ (unsigned)ndst, (unsigned)narg);
+ }
+ width = (u32)cg_type_size(t->c, dsts[0].type) * 8u;
+ c_ensure_local(t, dsts[0].v.local, dsts[0].type);
+ c_emit_local_assign_open(t, dsts[0].v.local, (KitCgTypeId)0);
+ if (width == 64u) {
+ cbuf_puts(&t->body, is_signed ? "((int64_t)(((__int128)(int64_t)("
+ : "((uint64_t)(((unsigned __int128)(uint64_t)(");
+ c_emit_operand(t, args[0]);
+ cbuf_puts(&t->body, is_signed ? ") * (__int128)(int64_t)("
+ : ") * (unsigned __int128)(uint64_t)(");
+ c_emit_operand(t, args[1]);
+ cbuf_puts(&t->body, ")) >> 64))");
+ } else {
+ cbuf_puts(&t->body, is_signed ? "((int32_t)(((int64_t)(int32_t)("
+ : "((uint32_t)(((uint64_t)(uint32_t)(");
+ c_emit_operand(t, args[0]);
+ cbuf_puts(&t->body, is_signed ? ") * (int64_t)(int32_t)("
+ : ") * (uint64_t)(uint32_t)(");
+ c_emit_operand(t, args[1]);
+ cbuf_puts(&t->body, is_signed ? ")) >> 32))" : ")) >> 32))");
+ }
+ c_emit_local_assign_close(t);
+ return;
+ }
+ case INTRIN_CPU_YIELD:
+ /* A portable relax hint may be discarded by the downstream C compiler. */
+ cbuf_puts(&t->body, " (void)0;\n");
+ return;
case INTRIN_MEMMOVE: {
cbuf_puts(&t->body, " __builtin_memmove(");
for (u32 i = 0; i < narg; ++i) {
diff --git a/src/arch/riscv/arch.c b/src/arch/riscv/arch.c
@@ -367,6 +367,9 @@ static int rv64_supports_intrinsic(const Compiler* c, KitCgIntrinsic intrin) {
* register intrinsic path can't carry), so it routes to the
* __kit_readcyclecounter libkit_rt helper instead (src/cg/arith.c). */
return c->target.arch == KIT_ARCH_RV64 || c->target.arch == KIT_ARCH_RV32;
+ case KIT_CG_INTRIN_SMUL_HIGH:
+ case KIT_CG_INTRIN_UMUL_HIGH:
+ return 1;
case KIT_CG_INTRIN_SYSCALL:
return c->target.os == KIT_OS_LINUX ||
c->target.os == KIT_OS_FREESTANDING;
diff --git a/src/arch/riscv/native.c b/src/arch/riscv/native.c
@@ -3475,6 +3475,30 @@ static void rv_intrinsic(NativeTarget* t, IntrinKind kind,
}
return;
}
+ case INTRIN_SMUL_HIGH:
+ case INTRIN_UMUL_HIGH: {
+ int is64 = rv_is_64(t, dsts[0].type);
+ int single = is64 || v->xlen == 32u;
+ u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
+ u32 rd = loc_reg(dsts[0]);
+ if (single) {
+ rv64_emit32(mc, kind == INTRIN_SMUL_HIGH ? rv_mulh(rd, ra, rb)
+ : rv_mulhu(rd, ra, rb));
+ } else if (kind == INTRIN_SMUL_HIGH) {
+ rv64_emit32(mc, rv_addiw(RV_TMP2, ra, 0));
+ rv64_emit32(mc, rv_addiw(RV_TMP3, rb, 0));
+ rv64_emit32(mc, rv_mul(RV_TMP2, RV_TMP2, RV_TMP3));
+ rv64_emit32(mc, rv_srai(rd, RV_TMP2, 32));
+ } else {
+ rv64_emit32(mc, rv_slli(RV_TMP2, ra, 32));
+ rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP2, 32));
+ rv64_emit32(mc, rv_slli(RV_TMP3, rb, 32));
+ rv64_emit32(mc, rv_srli(RV_TMP3, RV_TMP3, 32));
+ rv64_emit32(mc, rv_mul(RV_TMP2, RV_TMP2, RV_TMP3));
+ rv64_emit32(mc, rv_srli(rd, RV_TMP2, 32));
+ }
+ return;
+ }
case INTRIN_MEMMOVE: {
u32 dr, sr, n;
if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
diff --git a/src/arch/wasm/arch.c b/src/arch/wasm/arch.c
@@ -90,6 +90,8 @@ static int wasm_supports_intrinsic(const Compiler* c, KitCgIntrinsic intrin) {
case KIT_CG_INTRIN_CTZ:
case KIT_CG_INTRIN_POPCOUNT:
case KIT_CG_INTRIN_BSWAP:
+ case KIT_CG_INTRIN_SMUL_HIGH:
+ case KIT_CG_INTRIN_UMUL_HIGH:
case KIT_CG_INTRIN_SADD_OVERFLOW:
case KIT_CG_INTRIN_UADD_OVERFLOW:
case KIT_CG_INTRIN_SSUB_OVERFLOW:
@@ -100,6 +102,9 @@ static int wasm_supports_intrinsic(const Compiler* c, KitCgIntrinsic intrin) {
case KIT_CG_INTRIN_EXPECT:
case KIT_CG_INTRIN_ASSUME_ALIGNED:
return 1;
+ /* Portable spin-loop hint: wasm has no instruction, so it is a no-op. */
+ case KIT_CG_INTRIN_CPU_YIELD:
+ return 1;
case KIT_CG_INTRIN_SETJMP:
case KIT_CG_INTRIN_LONGJMP:
case KIT_CG_INTRIN_FMA:
@@ -116,7 +121,6 @@ static int wasm_supports_intrinsic(const Compiler* c, KitCgIntrinsic intrin) {
case KIT_CG_INTRIN_DCACHE_CLEAN_INVALIDATE:
case KIT_CG_INTRIN_ICACHE_INVALIDATE:
case KIT_CG_INTRIN_CPU_NOP:
- case KIT_CG_INTRIN_CPU_YIELD:
case KIT_CG_INTRIN_WFI:
case KIT_CG_INTRIN_WFE:
case KIT_CG_INTRIN_SEV:
diff --git a/src/arch/wasm/emit.c b/src/arch/wasm/emit.c
@@ -1571,6 +1571,10 @@ static const char* intrin_name(IntrinKind k) {
return "__builtin_clz";
case INTRIN_BSWAP:
return "__builtin_bswap";
+ case INTRIN_SMUL_HIGH:
+ return "smul_high";
+ case INTRIN_UMUL_HIGH:
+ return "umul_high";
case INTRIN_MEMMOVE:
return "memmove";
case INTRIN_PREFETCH:
@@ -1649,6 +1653,10 @@ void wasm_intrinsic(CGTarget* tg, IntrinKind k, Operand* dst, u32 ndst,
/* No-op hint. */
return;
+ case INTRIN_CPU_YIELD:
+ /* Portable spin-loop hint; wasm has no corresponding core instruction. */
+ return;
+
case INTRIN_EXPECT:
case INTRIN_ASSUME_ALIGNED:
/* Pass-through hint: result = first argument. CG always allocates a
@@ -1755,6 +1763,25 @@ void wasm_intrinsic(CGTarget* tg, IntrinKind k, Operand* dst, u32 ndst,
return;
}
+ case INTRIN_SMUL_HIGH:
+ case INTRIN_UMUL_HIGH: {
+ if (ndst != 1 || nargs != 2 || dst[0].kind != OPK_REG) {
+ compiler_panic(t->c, cur_loc(t),
+ "wasm target: %s requires 2 args + 1 result reg",
+ intrin_name(k));
+ return;
+ }
+ WIR* w = wir_push(t);
+ w->op = WIR_INTRINSIC;
+ w->cgop = (u8)k;
+ w->dst = dst[0].v.reg;
+ w->type = dst[0].type;
+ w->cls = dst[0].cls;
+ wir_capture_operand(w, 0, args[0]);
+ wir_capture_operand(w, 1, args[1]);
+ return;
+ }
+
case INTRIN_SETJMP:
case INTRIN_LONGJMP:
compiler_panic(t->c, cur_loc(t),
@@ -1767,7 +1794,6 @@ void wasm_intrinsic(CGTarget* tg, IntrinKind k, Operand* dst, u32 ndst,
* kit_cg_target_supports_intrinsic reports them false so frontends
* diagnose before reaching here. Fall through to the generic panic. */
case INTRIN_CPU_NOP:
- case INTRIN_CPU_YIELD:
case INTRIN_WFI:
case INTRIN_WFE:
case INTRIN_SEV:
@@ -3086,6 +3112,133 @@ static void emit_intrinsic_bswap(WTarget* t, const WIR* w) {
emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
}
+static void emit_intrinsic_mul_high(WTarget* t, const WIR* w) {
+ IntrinKind k = (IntrinKind)w->cgop;
+ WasmValType vt = type_valtype(t, w->type);
+ u32 a_loc = add_wasm_local(t, vt);
+ u32 b_loc = add_wasm_local(t, vt);
+ emit_push_operand(t, w->imm_kind, w->imm_a, w->a, w->type);
+ emit_insn(t, WASM_INSN_LOCAL_SET, (i64)a_loc);
+ emit_push_operand(t, w->imm_kind_b, w->imm_b, w->b, w->type);
+ emit_insn(t, WASM_INSN_LOCAL_SET, (i64)b_loc);
+
+ if (vt == WASM_VAL_I32) {
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
+ emit_insn(t, k == INTRIN_SMUL_HIGH ? WASM_INSN_I64_EXTEND_I32_S
+ : WASM_INSN_I64_EXTEND_I32_U,
+ 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
+ emit_insn(t, k == INTRIN_SMUL_HIGH ? WASM_INSN_I64_EXTEND_I32_S
+ : WASM_INSN_I64_EXTEND_I32_U,
+ 0);
+ emit_insn(t, WASM_INSN_I64_MUL, 0);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, k == INTRIN_SMUL_HIGH ? WASM_INSN_I64_SHR_S
+ : WASM_INSN_I64_SHR_U,
+ 0);
+ emit_insn(t, WASM_INSN_I32_WRAP_I64, 0);
+ emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
+ return;
+ }
+
+ /* Core wasm has no widening i64 multiply. Split into 32-bit limbs:
+ * high = p11 + (p01>>32) + (p10>>32) +
+ * (((p00>>32) + lo32(p01) + lo32(p10)) >> 32). */
+ {
+ u32 p00 = add_wasm_local(t, WASM_VAL_I64);
+ u32 p01 = add_wasm_local(t, WASM_VAL_I64);
+ u32 p10 = add_wasm_local(t, WASM_VAL_I64);
+ u32 p11 = add_wasm_local(t, WASM_VAL_I64);
+ u32 middle = add_wasm_local(t, WASM_VAL_I64);
+ u32 high = add_wasm_local(t, WASM_VAL_I64);
+
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
+ emit_insn(t, WASM_INSN_I64_AND, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
+ emit_insn(t, WASM_INSN_I64_AND, 0);
+ emit_insn(t, WASM_INSN_I64_MUL, 0);
+ emit_insn(t, WASM_INSN_LOCAL_SET, (i64)p00);
+
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
+ emit_insn(t, WASM_INSN_I64_AND, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, WASM_INSN_I64_SHR_U, 0);
+ emit_insn(t, WASM_INSN_I64_MUL, 0);
+ emit_insn(t, WASM_INSN_LOCAL_SET, (i64)p01);
+
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, WASM_INSN_I64_SHR_U, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
+ emit_insn(t, WASM_INSN_I64_AND, 0);
+ emit_insn(t, WASM_INSN_I64_MUL, 0);
+ emit_insn(t, WASM_INSN_LOCAL_SET, (i64)p10);
+
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, WASM_INSN_I64_SHR_U, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, WASM_INSN_I64_SHR_U, 0);
+ emit_insn(t, WASM_INSN_I64_MUL, 0);
+ emit_insn(t, WASM_INSN_LOCAL_SET, (i64)p11);
+
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p00);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, WASM_INSN_I64_SHR_U, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p01);
+ emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
+ emit_insn(t, WASM_INSN_I64_AND, 0);
+ emit_insn(t, WASM_INSN_I64_ADD, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p10);
+ emit_insn(t, WASM_INSN_I64_CONST, 0xffffffffull);
+ emit_insn(t, WASM_INSN_I64_AND, 0);
+ emit_insn(t, WASM_INSN_I64_ADD, 0);
+ emit_insn(t, WASM_INSN_LOCAL_SET, (i64)middle);
+
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p11);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p01);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, WASM_INSN_I64_SHR_U, 0);
+ emit_insn(t, WASM_INSN_I64_ADD, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)p10);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, WASM_INSN_I64_SHR_U, 0);
+ emit_insn(t, WASM_INSN_I64_ADD, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)middle);
+ emit_insn(t, WASM_INSN_I64_CONST, 32);
+ emit_insn(t, WASM_INSN_I64_SHR_U, 0);
+ emit_insn(t, WASM_INSN_I64_ADD, 0);
+ emit_insn(t, WASM_INSN_LOCAL_SET, (i64)high);
+
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)high);
+ if (k == INTRIN_SMUL_HIGH) {
+ /* signed_high(a,b) = unsigned_high(a,b) - (a<0 ? b : 0)
+ * - (b<0 ? a : 0). */
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 0);
+ emit_insn(t, WASM_INSN_I64_LT_S, 0);
+ emit_insn(t, WASM_INSN_SELECT, 0);
+ emit_insn(t, WASM_INSN_I64_SUB, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)a_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 0);
+ emit_insn(t, WASM_INSN_LOCAL_GET, (i64)b_loc);
+ emit_insn(t, WASM_INSN_I64_CONST, 0);
+ emit_insn(t, WASM_INSN_I64_LT_S, 0);
+ emit_insn(t, WASM_INSN_SELECT, 0);
+ emit_insn(t, WASM_INSN_I64_SUB, 0);
+ }
+ emit_local_set(t, w->dst, w->type, (RegClass)w->cls);
+ }
+}
+
static void emit_intrinsic_overflow(WTarget* t, const WIR* w) {
IntrinKind k = (IntrinKind)w->cgop;
WasmValType vt = type_valtype(t, w->type);
@@ -3236,6 +3389,10 @@ static void emit_intrinsic(WTarget* t, const WIR* w) {
case INTRIN_BSWAP:
emit_intrinsic_bswap(t, w);
return;
+ case INTRIN_SMUL_HIGH:
+ case INTRIN_UMUL_HIGH:
+ emit_intrinsic_mul_high(t, w);
+ return;
case INTRIN_SADD_OVERFLOW:
case INTRIN_UADD_OVERFLOW:
case INTRIN_SSUB_OVERFLOW:
diff --git a/src/arch/x64/arch.c b/src/arch/x64/arch.c
@@ -48,11 +48,12 @@ enum {
X64_FEAT_SSE42,
X64_FEAT_AVX,
X64_FEAT_AVX2,
+ X64_FEAT_POPCNT,
};
static const ArchTargetFeature x64_target_features[] = {
{"sse"}, {"sse2"}, {"sse3"}, {"ssse3"},
- {"sse4.1"}, {"sse4.2"}, {"avx"}, {"avx2"},
+ {"sse4.1"}, {"sse4.2"}, {"avx"}, {"avx2"}, {"popcnt"},
};
static void x64_feature_set(u64* words, u32 nwords, u32 idx) {
@@ -76,6 +77,7 @@ static KitStatus x64_target_feature_apply_isa(const Target* target,
x64_feature_set(words, nwords, X64_FEAT_SSSE3);
x64_feature_set(words, nwords, X64_FEAT_SSE41);
x64_feature_set(words, nwords, X64_FEAT_SSE42);
+ x64_feature_set(words, nwords, X64_FEAT_POPCNT);
return KIT_OK;
}
if (kit_slice_eq_cstr(isa, "x86-64-v3") ||
@@ -89,6 +91,7 @@ static KitStatus x64_target_feature_apply_isa(const Target* target,
x64_feature_set(words, nwords, X64_FEAT_SSE42);
x64_feature_set(words, nwords, X64_FEAT_AVX);
x64_feature_set(words, nwords, X64_FEAT_AVX2);
+ x64_feature_set(words, nwords, X64_FEAT_POPCNT);
return KIT_OK;
}
return KIT_UNSUPPORTED;
@@ -151,6 +154,8 @@ static int x64_supports_intrinsic(const Compiler* c, KitCgIntrinsic intrin) {
case KIT_CG_INTRIN_FRAME_ADDRESS:
case KIT_CG_INTRIN_RETURN_ADDRESS:
case KIT_CG_INTRIN_READCYCLECOUNTER: /* RDTSC */
+ case KIT_CG_INTRIN_SMUL_HIGH:
+ case KIT_CG_INTRIN_UMUL_HIGH:
return 1;
case KIT_CG_INTRIN_SYSCALL:
return c->target.os == KIT_OS_LINUX ||
diff --git a/src/arch/x64/native.c b/src/arch/x64/native.c
@@ -3503,6 +3503,24 @@ static void emit_popcnt(MCEmitter* mc, int w, u32 dst, u32 src) {
}
emit_rm_reg(mc, dst, src);
}
+static void emit_popcount_software(MCEmitter* mc, int w, u32 dst, u32 src) {
+ MCLabel loop = mc_label_new(mc);
+ MCLabel done = mc_label_new(mc);
+ /* Kernighan's bit-count loop. Keep the source in reserved scratch so dst
+ * may legally overlap it, and so the baseline x86-64 profile never executes
+ * POPCNT unless the resolved target says it is available. */
+ emit_mov_rr(mc, w, X64_TMP_INT, src);
+ x64_emit_load_imm(mc, w, dst, 0);
+ mc_label_place(mc, loop);
+ emit_test_self(mc, w, X64_TMP_INT);
+ emit_jcc_rel32(mc, X64_CC_E, done);
+ emit_alu_imm8(mc, w, X64_ALU_SUB_ADD, dst, 1);
+ emit_mov_rr(mc, w, X64_TMP_INT2, X64_TMP_INT);
+ emit_alu_imm8(mc, w, X64_ALU_SUB_SUB, X64_TMP_INT2, 1);
+ emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_TMP_INT, X64_TMP_INT2);
+ emit_jmp_rel32(mc, loop);
+ mc_label_place(mc, done);
+}
static void emit_bs(MCEmitter* mc, int w, u8 opcode2, u32 dst, u32 src) {
emit_rex(mc, w, dst, 0, src);
{
@@ -3574,6 +3592,21 @@ static void x64_intrinsic(NativeTarget* t, IntrinKind kind,
x64_move(t, dsts[0], args[0]);
return;
case INTRIN_PREFETCH:
+ if (narg >= 1u && args[0].kind == NATIVE_LOC_REG) {
+ /* GCC locality 0..3 maps from no-temporal through increasing temporal
+ * locality: PREFETCHNTA, PREFETCHT2, PREFETCHT1, PREFETCHT0. The rw
+ * operand is intentionally ignored: PREFETCHW is not baseline x64. */
+ static const u8 hint[4] = {0u, 3u, 2u, 1u};
+ u32 locality = 3u;
+ u32 base = loc_reg(args[0]);
+ u8 op[2] = {0x0F, 0x18};
+ if (narg >= 3u && args[2].kind == NATIVE_LOC_IMM)
+ locality = (u32)args[2].v.imm;
+ if (locality > 3u) locality = 3u;
+ emit_rex(mc, 0, hint[locality], 0, base);
+ mc_emit_bytes(mc, op, 2);
+ emit_mem_operand(mc, hint[locality], base, 0);
+ }
return;
case INTRIN_TRAP:
emit_ud2(mc);
@@ -3614,10 +3647,14 @@ static void x64_intrinsic(NativeTarget* t, IntrinKind kind,
native_loc_reg(dsts[0].type, NATIVE_REG_INT, X64_RAX));
}
return;
- case INTRIN_POPCOUNT:
- emit_popcnt(mc, x64_is_64(t, args[0].type) ? 1 : 0, loc_reg(dsts[0]),
- loc_reg(args[0]));
+ case INTRIN_POPCOUNT: {
+ int w = x64_is_64(t, args[0].type) ? 1 : 0;
+ if (kit_target_has_feature(t->c->target_ref, KIT_SLICE_LIT("popcnt")))
+ emit_popcnt(mc, w, loc_reg(dsts[0]), loc_reg(args[0]));
+ else
+ emit_popcount_software(mc, w, loc_reg(dsts[0]), loc_reg(args[0]));
return;
+ }
case INTRIN_CTZ:
emit_bs(mc, x64_is_64(t, args[0].type) ? 1 : 0, 0xBC /* bsf */,
loc_reg(dsts[0]), loc_reg(args[0]));
@@ -3702,6 +3739,24 @@ static void x64_intrinsic(NativeTarget* t, IntrinKind kind,
emit_movzx_r32_r8(mc, rovf, rovf);
return;
}
+ case INTRIN_SMUL_HIGH:
+ case INTRIN_UMUL_HIGH: {
+ int w = x64_is_64(t, dsts[0].type) ? 1 : 0;
+ u32 rd = loc_reg(dsts[0]);
+ u32 ra = loc_reg(args[0]);
+ u32 rb = loc_reg(args[1]);
+ if (rb == X64_RAX || rb == X64_RDX) {
+ emit_mov_rr(mc, w, X64_R11, rb);
+ rb = X64_R11;
+ }
+ if (ra != X64_RAX) emit_mov_rr(mc, w, X64_RAX, ra);
+ emit_f7_rm(mc, w,
+ kind == INTRIN_SMUL_HIGH ? X64_F7_SUB_IMUL
+ : X64_F7_SUB_MUL,
+ rb);
+ if (rd != X64_RDX) emit_mov_rr(mc, w, rd, X64_RDX);
+ return;
+ }
case INTRIN_MEMMOVE: {
u32 dr, sr, n, i;
if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
@@ -4278,6 +4333,8 @@ static int x64_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
* and the CPU instruction itself clobbers rcx/r11; the kernel ABI treats
* the integer caller-saved syscall registers as volatile. */
if ((IntrinKind)op->intrin == INTRIN_UMUL_OVERFLOW ||
+ (IntrinKind)op->intrin == INTRIN_SMUL_HIGH ||
+ (IntrinKind)op->intrin == INTRIN_UMUL_HIGH ||
(IntrinKind)op->intrin == INTRIN_READCYCLECOUNTER) {
/* MUL's rdx:rax product / RDTSC's edx:eax counter both write both
* registers; keep live values out of them across the op. */
diff --git a/src/cg/arith.c b/src/cg/arith.c
@@ -2003,10 +2003,12 @@ static const IntrinDesc kIntrinTable[] = {
false, false},
[KIT_CG_INTRIN_READCYCLECOUNTER] = {INTRIN_READCYCLECOUNTER,
"readcyclecounter", false, false},
+ [KIT_CG_INTRIN_SMUL_HIGH] = {INTRIN_SMUL_HIGH, "smul_high", false, false},
+ [KIT_CG_INTRIN_UMUL_HIGH] = {INTRIN_UMUL_HIGH, "umul_high", false, false},
};
_Static_assert(sizeof(kIntrinTable) / sizeof(kIntrinTable[0]) ==
- KIT_CG_INTRIN_READCYCLECOUNTER + 1,
+ KIT_CG_INTRIN_UMUL_HIGH + 1,
"kIntrinTable must have exactly one row per KitCgIntrinsic");
/* Bounds-guarded row lookup: an out-of-range intrinsic falls back to the NONE
@@ -2083,6 +2085,25 @@ void kit_cg_intrinsic(KitCg* g, KitCgIntrinsic intrin, uint32_t nargs,
builtin_id(KIT_CG_BUILTIN_I64), NULL, 0, NULL);
return;
}
+ /* A 64-bit value is a two-register pair on the 32-bit native targets. The
+ * ordinary intrinsic ABI carries one register per operand/result, so use the
+ * rollover-free compiler runtime's limb implementation for high multiply. */
+ if (nargs == 2 &&
+ (intrin == KIT_CG_INTRIN_SMUL_HIGH ||
+ intrin == KIT_CG_INTRIN_UMUL_HIGH) &&
+ api_wide64_stack_top(g, 0) && api_wide64_stack_top(g, 1)) {
+ ApiSValue args_h[2];
+ KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
+ KitCgTypeId ps[2] = {i64, i64};
+ args_h[1] = api_pop(g);
+ args_h[0] = api_pop(g);
+ api_runtime_call_values(g,
+ intrin == KIT_CG_INTRIN_SMUL_HIGH
+ ? "__kit_mulhdi3"
+ : "__kit_umulhdi3",
+ i64, ps, 2, args_h);
+ return;
+ }
/* clz/ctz/popcount/bswap on a split 64-bit value cannot use the backend's
* single-register software sequence. Route them to the compiler-rt __*di2
* helpers, which decompose into 32-bit operations. (32-bit forms still lower
diff --git a/src/cg/cgir.h b/src/cg/cgir.h
@@ -199,6 +199,10 @@ typedef enum IntrinKind {
* is conservatively side-effecting, so it is never hoisted/CSE'd/removed). */
INTRIN_READCYCLECOUNTER,
+ /* High half of a full-width two-operand product. */
+ INTRIN_SMUL_HIGH,
+ INTRIN_UMUL_HIGH,
+
/* Internal stack-protector guard read. No operands; dsts[0] receives one
* pointer-width word. This is emitted only for target ABIs whose guard has
* no linkable object symbol (Linux/Android x86-64 use fs:0x28). */
diff --git a/src/interp/engine.c b/src/interp/engine.c
@@ -1557,6 +1557,17 @@ static u64 ibswap(u64 v, u32 nbytes) {
return r;
}
+static u64 imul_high_u64(u64 a, u64 b) {
+ u64 a0 = (u32)a, a1 = a >> 32;
+ u64 b0 = (u32)b, b1 = b >> 32;
+ u64 p00 = a0 * b0;
+ u64 p01 = a0 * b1;
+ u64 p10 = a1 * b0;
+ u64 p11 = a1 * b1;
+ u64 middle = (p00 >> 32) + (u32)p01 + (u32)p10;
+ return p11 + (p01 >> 32) + (p10 >> 32) + (middle >> 32);
+}
+
static int interp_intrinsic(InterpStack* st, InterpFunc* fn, u64* regs,
u32 mem_off, InterpInsn* in) {
InterpProgram* p = st->prog;
@@ -1590,6 +1601,27 @@ static int interp_intrinsic(InterpStack* st, InterpFunc* fn, u64* regs,
case INTRIN_BSWAP:
regs[DST0] = ibswap(ARGV(0), DWID(0));
return 1;
+ case INTRIN_SMUL_HIGH:
+ case INTRIN_UMUL_HIGH: {
+ u32 w = AWID(0);
+ u64 a = mask_w(ARGV(0), w);
+ u64 b = mask_w(ARGV(1), w);
+ u64 hi;
+ if (w < 8u) {
+ if (aux->kind == INTRIN_SMUL_HIGH)
+ hi = (u64)((i64)sext_w(a, w) * (i64)sext_w(b, w)) >> (w * 8u);
+ else
+ hi = (a * b) >> (w * 8u);
+ } else {
+ hi = imul_high_u64(a, b);
+ if (aux->kind == INTRIN_SMUL_HIGH) {
+ if ((i64)a < 0) hi -= b;
+ if ((i64)b < 0) hi -= a;
+ }
+ }
+ regs[DST0] = mask_w(hi, w);
+ return 1;
+ }
case INTRIN_EXPECT:
regs[DST0] = ARGV(0);
return 1;
diff --git a/src/opt/pass_combine.c b/src/opt/pass_combine.c
@@ -1793,6 +1793,11 @@ static int try_addr_of_cse(CombineCtx* ctx, Inst* in, i32 i) {
continue;
/* A copy of the producer reg into self is a no-op the dst already holds. */
if (same_phys_reg(&in->opnds[0], &e->dst)) continue;
+ /* Native emission may reuse a reserved scratch between MIR instructions
+ * while materializing an immediate or frame operand. Such clobbers are
+ * deliberately invisible to MIR liveness, so extending an address value
+ * held in scratch to this later copy is unsound. */
+ if (is_target_scratch_reg(ctx->f, e->dst.cls, e->dst.v.reg)) continue;
/* Rewrite `addr_of rD, <addr>` into `copy rD, rP`. */
in->op = (u16)IR_COPY;
in->opnds[1] = e->dst;
diff --git a/test/api/target_test.c b/test/api/target_test.c
@@ -184,6 +184,14 @@ static void check_x64_defaults_and_isa(void) {
EXPECT(has(t, "sse"), "x64 default has sse");
EXPECT(has(t, "sse2"), "x64 default has sse2");
EXPECT(!has(t, "avx"), "x64 default lacks avx");
+ EXPECT(!has(t, "popcnt"), "x64 default lacks popcnt");
+ kit_target_free(t);
+
+ t = NULL;
+ EXPECT(make_target(spec, KIT_SLICE_LIT("x86-64-v2"), NULL, 0, &t) == KIT_OK,
+ "x64 v2 target");
+ EXPECT(has(t, "popcnt"), "x86-64-v2 enables popcnt");
+ EXPECT(!has(t, "avx"), "x86-64-v2 lacks avx");
kit_target_free(t);
t = NULL;
@@ -191,6 +199,7 @@ static void check_x64_defaults_and_isa(void) {
KIT_OK,
"x64 isa profile plus feature override");
EXPECT(has(t, "avx"), "x86-64-v3 enables avx");
+ EXPECT(has(t, "popcnt"), "x86-64-v3 enables popcnt");
EXPECT(!has(t, "avx2"), "explicit feature override disables avx2");
kit_target_free(t);
}
diff --git a/test/parse/CORPUS.md b/test/parse/CORPUS.md
@@ -566,6 +566,9 @@ ordinary calls.
| `builtin_24_atomic_lock_free` | ★ | target-aware lock-free folding through `if`, `&&`, and `||`; dead 16-byte atomic arms suppress codegen | 42 |
| `builtin_25_atomic_fetch_nand` | ★ | `__atomic_fetch_nand` lowers to atomic NAND RMW | 42 |
| `builtin_26_sadd_overflow` | ★ | signed/unsigned typed overflow builtins store result and return overflow flag | 42 |
+| `builtin_32_portable_bits` | ★ | popcount, byte swap, and width-specific rotate builtins, including wrapped and zero counts | 42 |
+| `builtin_33_chip_hints_features` | ★ | prefetch, assume-aligned, portable CPU relax, and parser-folded resolved-target feature detection | 42 |
+| `builtin_34_carry_mulhigh` | ★ | add-carry/sub-borrow plus signed/unsigned 32- and 64-bit multiply-high | 42 |
## Variadic coverage
diff --git a/test/parse/cases/builtin_32_portable_bits.c b/test/parse/cases/builtin_32_portable_bits.c
@@ -0,0 +1,30 @@
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long long u64;
+
+int test_main(void) {
+ if (__builtin_popcount(0xf00fu) != 8) return 1;
+ if (__builtin_popcountl((unsigned long)0x80000001ul) != 2) return 2;
+ if (__builtin_popcountll(0xf00000000000000full) != 8) return 3;
+
+ if (__builtin_bswap16((u16)0x1234u) != (u16)0x3412u) return 4;
+ if (__builtin_bswap32(0x12345678u) != 0x78563412u) return 5;
+ if (__builtin_bswap64(0x0123456789abcdefull) !=
+ 0xefcdab8967452301ull)
+ return 6;
+
+ if (__builtin_rotateleft8((u8)0x81u, 1) != (u8)0x03u) return 7;
+ if (__builtin_rotateright16((u16)0x0003u, 1) != (u16)0x8001u) return 8;
+ if (__builtin_rotateleft32(0x12345678u, 36) != 0x23456781u) return 9;
+ if (__builtin_rotateright64(0x0123456789abcdefull, 8) !=
+ 0xef0123456789abcdull)
+ return 10;
+ if (__builtin_rotateleft64(0x55aa55aa55aa55aaull, 0) !=
+ 0x55aa55aa55aa55aaull)
+ return 11;
+ if (__builtin_rotateright8((u8)0x03u, 1) != (u8)0x81u) return 12;
+ if (__builtin_rotateleft16((u16)0x8001u, 1) != (u16)0x0003u) return 13;
+ if (__builtin_rotateright32(0x23456781u, 4) != 0x12345678u) return 14;
+ return 42;
+}
diff --git a/test/parse/cases/builtin_32_portable_bits.expected b/test/parse/cases/builtin_32_portable_bits.expected
@@ -0,0 +1 @@
+42
diff --git a/test/parse/cases/builtin_33_chip_hints_features.c b/test/parse/cases/builtin_33_chip_hints_features.c
@@ -0,0 +1,30 @@
+int test_main(void) {
+ enum {
+ unknown_target_feature =
+ __builtin_kit_target_has("kit-feature-that-does-not-exist")
+ };
+ int value = 7;
+ int values[2] = {1, 7};
+ int* aligned;
+
+ __builtin_prefetch(&value);
+ __builtin_prefetch(&value, 1);
+ __builtin_prefetch(&value, 0, 3);
+ aligned = (int*)__builtin_assume_aligned(&value, _Alignof(int));
+ if (aligned != &value || *aligned != 7) return 1;
+ aligned = (int*)__builtin_assume_aligned(&values[1], 8, sizeof(int));
+ if (aligned != &values[1] || *aligned != 7) return 7;
+
+ __builtin_kit_cpu_relax();
+
+ if (unknown_target_feature) return 2;
+#if defined(__x86_64__)
+ if (!__builtin_kit_target_has("sse2")) return 3;
+ if (__builtin_kit_target_has("popcnt")) return 6;
+#elif defined(__riscv)
+ if (!__builtin_kit_target_has("i")) return 4;
+#elif defined(__wasm__)
+ if (!__builtin_kit_target_has("threads")) return 5;
+#endif
+ return 42;
+}
diff --git a/test/parse/cases/builtin_33_chip_hints_features.expected b/test/parse/cases/builtin_33_chip_hints_features.expected
@@ -0,0 +1 @@
+42
diff --git a/test/parse/cases/builtin_34_carry_mulhigh.c b/test/parse/cases/builtin_34_carry_mulhigh.c
@@ -0,0 +1,44 @@
+typedef unsigned int u32;
+typedef int i32;
+typedef unsigned long ulong;
+typedef unsigned long long u64;
+typedef long long i64;
+
+int test_main(void) {
+ u32 carry32;
+ ulong carryl;
+ u64 carry64;
+
+ if (__builtin_addc(~0u, 1u, 0u, &carry32) != 0u || carry32 != 1u) return 1;
+ if (__builtin_addcll(~0ull, 0ull, 1ull, &carry64) != 0ull ||
+ carry64 != 1ull)
+ return 2;
+ if (__builtin_subc(0u, 0u, 1u, &carry32) != ~0u || carry32 != 1u) return 3;
+ if (__builtin_subcll(0ull, 1ull, 0ull, &carry64) != ~0ull ||
+ carry64 != 1ull)
+ return 4;
+ if (__builtin_addcl(~0ul, 0ul, 1ul, &carryl) != 0ul || carryl != 1ul)
+ return 9;
+ if (__builtin_subcl(0ul, 1ul, 0ul, &carryl) != ~0ul || carryl != 1ul)
+ return 10;
+
+ if (__builtin_kit_umul_high32(~0u, 2u) != 1u) return 5;
+ if (__builtin_kit_umul_high64(~0ull, 2ull) != 1ull) return 6;
+ if (__builtin_kit_smul_high32((i32)-2, (i32)3) != (i32)-1) return 7;
+ if (__builtin_kit_smul_high64((i64)-2, (i64)3) != (i64)-1) return 8;
+ if (__builtin_kit_umul_high32(0x80000000u, 0x80000000u) != 0x40000000u)
+ return 11;
+ if (__builtin_kit_umul_high64(0x8000000000000000ull,
+ 0x8000000000000000ull) !=
+ 0x4000000000000000ull)
+ return 12;
+ if (__builtin_kit_umul_high64(~0ull, ~0ull) != ~1ull) return 13;
+ if (__builtin_kit_smul_high32((i32)0x80000000u, (i32)0x80000000u) !=
+ (i32)0x40000000u)
+ return 14;
+ if (__builtin_kit_smul_high64((i64)0x8000000000000000ull,
+ (i64)0x8000000000000000ull) !=
+ (i64)0x4000000000000000ull)
+ return 15;
+ return 42;
+}
diff --git a/test/parse/cases/builtin_34_carry_mulhigh.expected b/test/parse/cases/builtin_34_carry_mulhigh.expected
@@ -0,0 +1 @@
+42
