wide.c (12525B)
1 #include "cg/internal.h" 2 3 CGLocal api_f128_temp_local(KitCg* g, KitCgTypeId ty) { 4 CGLocalDesc d; 5 memset(&d, 0, sizeof d); 6 d.type = ty; 7 d.size = 16; 8 d.align = 16; 9 d.flags = CG_LOCAL_ADDR_TAKEN | CG_LOCAL_MEMORY_REQUIRED; 10 return g->target->local(g->target, &d); 11 } 12 13 u64 api_u64_from_target_bytes(KitCg* g, const u8* bytes) { 14 u64 v = 0; 15 for (u32 i = 0; i < 8; ++i) { 16 u32 shift = g->c->target.big_endian ? (7u - i) * 8u : i * 8u; 17 v |= (u64)bytes[i] << shift; 18 } 19 return v; 20 } 21 22 void api_wide16_sext_imm_bytes(KitCg* g, i64 imm, u8 bytes[16]) { 23 /* A 16-byte scalar immediate only carries 64 bits in op.v.imm; the full 24 * value is its sign-extension. Fill both lanes accordingly, honoring the 25 * target byte order. */ 26 u64 lo = (u64)imm; 27 u64 hi = imm < 0 ? ~(u64)0 : 0; 28 for (u32 i = 0; i < 8; ++i) { 29 u32 lo_idx = g->c->target.big_endian ? 15u - i : i; 30 u32 hi_idx = g->c->target.big_endian ? 7u - i : 8u + i; 31 bytes[lo_idx] = (u8)(lo >> (i * 8u)); 32 bytes[hi_idx] = (u8)(hi >> (i * 8u)); 33 } 34 } 35 36 ApiSValue api_make_wide16_int_const(KitCg* g, i64 value, KitCgTypeId ty) { 37 u8 bytes[16]; 38 CGLocal local = api_f128_temp_local(g, ty); 39 api_wide16_sext_imm_bytes(g, value, bytes); 40 api_store_f128_bytes(g, local, ty, bytes); 41 /* i128/f128 are scalar VALUEs (Track 7.3), not places: the constant lives in 42 * `local` and flows as a value. Returning an lvalue here made the O1 ABI path 43 * pass the constant by-reference and deref a value slot (a null-deref crash 44 * on i128->bool compares); a value backed by the local is the correct form 45 * and matches how api_push_call_result represents an i128 result. */ 46 return api_make_sv(api_op_local(local, ty), ty); 47 } 48 49 ApiSValue api_make_wide16_int_const_bits(KitCg* g, u64 lo, u64 hi, 50 KitCgTypeId ty) { 51 u8 bytes[16]; 52 CGLocal local; 53 for (u32 i = 0; i < 8; ++i) { 54 u32 lo_idx = g->c->target.big_endian ? 15u - i : i; 55 u32 hi_idx = g->c->target.big_endian ? 7u - i : 8u + i; 56 bytes[lo_idx] = (u8)(lo >> (i * 8u)); 57 bytes[hi_idx] = (u8)(hi >> (i * 8u)); 58 } 59 local = api_f128_temp_local(g, ty); 60 api_store_f128_bytes(g, local, ty, bytes); 61 return api_make_sv(api_op_local(local, ty), ty); 62 } 63 64 void api_store_f128_bytes(KitCg* g, CGLocal local, KitCgTypeId ty, 65 const u8 bytes[16]) { 66 KitCgTypeId i64_ty = builtin_id(KIT_CG_BUILTIN_I64); 67 KitCgTypeId ptr_ty = cg_type_ptr_to(g->c, ty); 68 CGLocal ar = api_alloc_temp_local(g, ptr_ty); 69 Operand base = api_op_local(ar, ptr_ty); 70 MemAccess ma; 71 memset(&ma, 0, sizeof ma); 72 ma.type = i64_ty; 73 ma.size = 8; 74 ma.align = 8; 75 g->target->addr_of(g->target, base, api_op_local(local, ty)); 76 g->target->store(g->target, api_op_indirect(ar, 0, i64_ty), 77 api_op_imm((i64)api_u64_from_target_bytes(g, bytes), i64_ty), 78 ma); 79 g->target->store( 80 g->target, api_op_indirect(ar, 8, i64_ty), 81 api_op_imm((i64)api_u64_from_target_bytes(g, bytes + 8), i64_ty), ma); 82 } 83 84 void api_encode_binary128_from_double(KitCg* g, double value, u8 out[16]) { 85 union { 86 double d; 87 u64 u; 88 } in; 89 u64 lo = 0; 90 u64 hi = 0; 91 u64 frac; 92 u32 sign; 93 u32 exp; 94 in.d = value; 95 sign = (u32)(in.u >> 63); 96 exp = (u32)((in.u >> 52) & 0x7ffu); 97 frac = in.u & 0x000fffffffffffffull; 98 if (sign) hi |= 1ull << 63; 99 if (exp == 0x7ffu) { 100 hi |= (u64)0x7fffu << 48; 101 if (frac) { 102 lo |= (frac & 0xfu) << 60; 103 hi |= frac >> 4; 104 hi |= 1ull << 47; 105 } 106 } else if (exp != 0 || frac != 0) { 107 i32 e; 108 u64 sig; 109 if (exp == 0) { 110 e = -1022; 111 sig = frac; 112 while ((sig & (1ull << 52)) == 0) { 113 sig <<= 1; 114 --e; 115 } 116 frac = sig & 0x000fffffffffffffull; 117 } else { 118 e = (i32)exp - 1023; 119 } 120 hi |= (u64)(u32)(e + 16383) << 48; 121 lo |= (frac & 0xfu) << 60; 122 hi |= frac >> 4; 123 } 124 for (u32 i = 0; i < 16; ++i) { 125 if (g->c->target.big_endian) { 126 u64 lane = i < 8u ? hi : lo; 127 u32 shift = (7u - (i & 7u)) * 8u; 128 out[i] = (u8)(lane >> shift); 129 } else { 130 u64 lane = i < 8u ? lo : hi; 131 u32 shift = (i & 7u) * 8u; 132 out[i] = (u8)(lane >> shift); 133 } 134 } 135 } 136 137 ApiSValue api_make_f128_const(KitCg* g, double value, KitCgTypeId ty) { 138 u8 bytes[16]; 139 CGLocal local; 140 api_encode_binary128_from_double(g, value, bytes); 141 local = api_f128_temp_local(g, ty); 142 api_store_f128_bytes(g, local, ty, bytes); 143 return api_make_lv(api_op_local(local, ty), ty); 144 } 145 146 /* ============================================================ 147 * wide8 — 8-byte scalar split into two 4-byte lanes 148 * 149 * Some 32-bit ABIs represent long long / int64_t, and sometimes soft double, 150 * as two machine words. Like the wide16 (i128/f128) scalars above it is 151 * memory-resident (api_is_wide8_scalar_type forces CG_LOCAL_MEMORY_REQUIRED), 152 * but its arithmetic is done INLINE as 2-word lane sequences (src/cg/arith.c) 153 * rather than via a runtime call, because compiler-rt has no 64-bit 154 * add/sub/and/or/xor helper. The lane size is 4 bytes; the low word is at 155 * offset 0 on a little-endian target. These primitives are the inline analogue 156 * of api_store_f128_bytes / api_i128_addr / api_i128_load_lane. 157 * ============================================================ */ 158 159 /* Allocate an 8-byte memory-resident, address-taken scalar temp. */ 160 CGLocal api_wide8_temp_local(KitCg* g, KitCgTypeId ty) { 161 CGLocalDesc d; 162 memset(&d, 0, sizeof d); 163 d.type = ty; 164 d.size = 8; 165 d.align = (u32)abi_cg_alignof(g->c->abi, ty); 166 if (!d.align) d.align = 8; 167 d.flags = CG_LOCAL_ADDR_TAKEN | CG_LOCAL_MEMORY_REQUIRED; 168 return g->target->local(g->target, &d); 169 } 170 171 /* Byte offset of the low / high 32-bit lane within an 8-byte scalar. */ 172 static i32 api_wide8_lo_off(KitCg* g) { 173 return g->c->target.big_endian ? 4 : 0; 174 } 175 static i32 api_wide8_hi_off(KitCg* g) { 176 return g->c->target.big_endian ? 0 : 4; 177 } 178 179 /* Materialize a 64-bit constant bit pattern into a fresh memory-resident 180 * scalar, storing its two 32-bit lanes, and return the value backed by that 181 * local. Used for both i64 immediates (bits = (u64)imm) and soft-double 182 * constants (bits = the IEEE-754 binary64 encoding). */ 183 ApiSValue api_make_wide8_const_bits(KitCg* g, u64 bits, KitCgTypeId ty) { 184 KitCgTypeId i32_ty = builtin_id(KIT_CG_BUILTIN_I32); 185 KitCgTypeId ptr_ty = cg_type_ptr_to(g->c, ty); 186 CGLocal local = api_wide8_temp_local(g, ty); 187 CGLocal ar = api_alloc_temp_local(g, ptr_ty); 188 Operand base = api_op_local(ar, ptr_ty); 189 MemAccess ma; 190 memset(&ma, 0, sizeof ma); 191 ma.type = i32_ty; 192 ma.size = 4; 193 ma.align = 4; 194 g->target->addr_of(g->target, base, api_op_local(local, ty)); 195 g->target->store(g->target, api_op_indirect(ar, api_wide8_lo_off(g), i32_ty), 196 api_op_imm((i64)(i32)(u32)(bits & 0xffffffffu), i32_ty), ma); 197 g->target->store(g->target, api_op_indirect(ar, api_wide8_hi_off(g), i32_ty), 198 api_op_imm((i64)(i32)(u32)(bits >> 32), i32_ty), ma); 199 return api_make_sv(api_op_local(local, ty), ty); 200 } 201 202 ApiSValue api_make_wide8_int_const(KitCg* g, i64 value, KitCgTypeId ty) { 203 return api_make_wide8_const_bits(g, (u64)value, ty); 204 } 205 206 /* Materialize an 8-byte value as an lvalue and return a pointer local to it. 207 * An immediate is first lowered to a 2-lane memory constant. */ 208 Operand api_wide8_addr(KitCg* g, ApiSValue* v, KitCgTypeId ty) { 209 ApiSValue lv; 210 if (api_sv_op_is(v, OPK_IMM)) { 211 lv = api_make_wide8_int_const(g, v->op.v.imm, ty); 212 } else { 213 lv = *v; 214 } 215 /* A delayed value (SV_CMP/SV_ARITH) routed here through the wide64 helpers is 216 * not yet a place. Materialize it first: api_ensure_local lowers it into a 217 * memory-resident wide8 temp (api_alloc_temp_local forces 218 * CG_LOCAL_MEMORY_REQUIRED for an 8-byte scalar), which is a real addressable 219 * home. Materialization, however, clears 220 * sv.lvalue (fold.c), so we must set the flag AFTER it runs — otherwise the 221 * lvalue check in api_lvalue_addr fails ("addr operand is not an lvalue"). 222 * Doing this before api_lvalue_addr also makes its own api_ensure_local a 223 * no-op (kind is now SV_OPERAND), so the flag survives. An operand that is 224 * already a place is left untouched by api_ensure_local and flows through as 225 * before. */ 226 if (api_sv_kind(&lv) != SV_OPERAND) api_ensure_local(g, &lv); 227 lv.type = ty; 228 lv.op.type = ty; 229 api_sv_set_lvalue(&lv, 1); 230 return api_lvalue_addr(g, &lv, cg_type_ptr_to(g->c, ty)); 231 } 232 233 /* Load a 32-bit lane (at byte offset `off`) of the scalar addressed by `addr` 234 * into a fresh i32 temp; returns the temp operand. */ 235 Operand api_wide8_load_lane(KitCg* g, Operand addr, i32 off) { 236 KitCgTypeId i32_ty = builtin_id(KIT_CG_BUILTIN_I32); 237 CGLocal rr = api_alloc_temp_local(g, i32_ty); 238 Operand dst = api_op_local(rr, i32_ty); 239 MemAccess ma; 240 memset(&ma, 0, sizeof ma); 241 ma.type = i32_ty; 242 ma.size = 4; 243 ma.align = 4; 244 g->target->load(g->target, dst, api_op_indirect(addr.v.local, off, i32_ty), 245 ma); 246 return dst; 247 } 248 249 /* Store an i32 `val` into the 32-bit lane (byte offset `off`) addressed by 250 * `addr`. */ 251 void api_wide8_store_lane(KitCg* g, Operand addr, i32 off, Operand val) { 252 KitCgTypeId i32_ty = builtin_id(KIT_CG_BUILTIN_I32); 253 MemAccess ma; 254 memset(&ma, 0, sizeof ma); 255 ma.type = i32_ty; 256 ma.size = 4; 257 ma.align = 4; 258 g->target->store(g->target, api_op_indirect(addr.v.local, off, i32_ty), val, 259 ma); 260 } 261 262 ApiSValue api_wide16_materialize_lvalue(KitCg* g, ApiSValue* v, 263 KitCgTypeId ty) { 264 if (v->op.kind == OPK_LOCAL) { 265 api_sv_set_lvalue(v, 1); 266 return *v; 267 } 268 if (v->op.kind == OPK_INDIRECT) { 269 ApiSValue out = *v; 270 out.type = ty; 271 out.op.type = ty; 272 api_sv_set_lvalue(&out, 1); 273 return out; 274 } 275 if (v->op.kind == OPK_GLOBAL) { 276 CGLocal local = api_f128_temp_local(g, ty); 277 Operand dst_lv = api_op_local(local, ty); 278 Operand dst_addr; 279 Operand src_addr; 280 AggregateAccess agg; 281 ApiSValue tmp = api_make_lv(dst_lv, ty); 282 ApiSValue src = api_make_lv(v->op, ty); 283 dst_addr = api_lvalue_addr(g, &tmp, cg_type_ptr_to(g->c, ty)); 284 src_addr = api_lvalue_addr(g, &src, cg_type_ptr_to(g->c, ty)); 285 memset(&agg, 0, sizeof agg); 286 agg.size = 16; 287 agg.align = 16; 288 g->target->copy_bytes(g->target, dst_addr, src_addr, agg); 289 return api_make_lv(dst_lv, ty); 290 } 291 if (v->op.kind == OPK_IMM) { 292 return api_make_wide16_int_const(g, v->op.v.imm, ty); 293 } 294 CG_BUG(g, "KitCg: 16-byte scalar value is not addressable (kind %u, op %u)", 295 (unsigned)api_sv_kind(v), (unsigned)v->op.kind); 296 return *v; 297 } 298 299 KitCgSym api_runtime_helper(KitCg* g, const char* name, KitCgTypeId ret, 300 const KitCgTypeId* params, u32 nparams) { 301 KitCgFuncParam ps[3]; 302 KitCgFuncResult result; 303 KitCgFuncSig sig; 304 KitCgDecl decl; 305 if (nparams > 3) return KIT_CG_SYM_NONE; 306 memset(ps, 0, sizeof ps); 307 for (u32 i = 0; i < nparams; ++i) ps[i].type = params[i]; 308 memset(&sig, 0, sizeof sig); 309 /* Runtime helpers always return a single value. */ 310 memset(&result, 0, sizeof result); 311 result.type = ret; 312 sig.result = result; 313 sig.params = ps; 314 sig.nparams = nparams; 315 sig.call_conv = KIT_CG_CC_TARGET_C; 316 memset(&decl, 0, sizeof decl); 317 decl.kind = KIT_CG_DECL_FUNC; 318 decl.linkage_name = kit_cg_c_linkage_name( 319 (KitCompiler*)g->c, 320 pool_intern_slice(g->c->global, slice_from_cstr(name))); 321 decl.display_name = decl.linkage_name; 322 decl.type = kit_cg_type_func((KitCompiler*)g->c, sig); 323 decl.sym.bind = KIT_SB_GLOBAL; 324 decl.sym.visibility = KIT_CG_VIS_DEFAULT; 325 return kit_cg_decl(g, decl); 326 } 327 328 void api_runtime_call_values(KitCg* g, const char* name, KitCgTypeId ret, 329 const KitCgTypeId* params, u32 nparams, 330 ApiSValue* args) { 331 KitCgCallAttrs attrs; 332 if (api_unevaluated(g)) { 333 (void)name; 334 (void)params; 335 for (u32 i = 0; i < nparams; ++i) 336 if (args) api_release(g, &args[i]); 337 if (ret != KIT_CG_TYPE_NONE) { 338 api_push(g, api_uneval_value(g, ret)); 339 api_const_set_top(g, api_const_unknown(ret)); 340 } 341 return; 342 } 343 KitCgSym sym = api_runtime_helper(g, name, ret, params, nparams); 344 memset(&attrs, 0, sizeof attrs); 345 for (u32 i = 0; i < nparams; ++i) api_push(g, args[i]); 346 api_call_symbol_common(g, sym, nparams, attrs); 347 } 348 349 /* ============================================================ 350 * Locals and params 351 * ============================================================ */