memory.c (34142B)
1 #include "cg/internal.h" 2 3 void kit_cg_push_int(KitCg* g, uint64_t value, KitCgTypeId type) { 4 KitCgTypeId ty; 5 ApiConstValue cv; 6 if (!g) return; 7 ty = resolve_type(g->c, type); 8 if (!ty) return; 9 cv = api_const_int_result(g, ty, value, 0, 0); 10 if (api_unevaluated(g)) { 11 api_push(g, api_uneval_value(g, ty)); 12 api_const_set_top(g, cv); 13 return; 14 } 15 /* A 16-byte scalar immediate cannot be represented by the 64-bit op.v.imm 16 * alone; materialize it into addressable storage with both lanes 17 * sign-extended so no downstream consumer sees an undefined high half. */ 18 if (api_is_wide16_scalar_type(g->c, ty)) { 19 api_push(g, api_make_wide16_int_const(g, (i64)value, ty)); 20 api_const_set_top(g, cv); 21 return; 22 } 23 /* Split-lane 8-byte int: the 64-bit value fits in op.v.imm, but the value is 24 * memory-resident, so materialize it as two 32-bit lanes. */ 25 if (api_is_wide8_scalar_type(g->c, ty)) { 26 api_push(g, api_make_wide8_int_const(g, (i64)value, ty)); 27 api_const_set_top(g, cv); 28 return; 29 } 30 api_push(g, api_make_sv(api_op_imm((i64)value, ty), ty)); 31 api_const_set_top(g, cv); 32 } 33 34 void kit_cg_push_const_int(KitCg* g, KitCgTypeId type, 35 const KitCgConstInt* value) { 36 KitCgTypeId ty; 37 ApiConstValue cv; 38 if (!g) return; 39 ty = resolve_type(g->c, type); 40 if (!ty) return; 41 cv = api_const_for_push(g, ty, value); 42 if (api_unevaluated(g)) { 43 api_push(g, api_uneval_value(g, ty)); 44 api_const_set_top(g, cv); 45 return; 46 } 47 if (cv.value.known && cv.value.width > 64u) { 48 api_push(g, 49 api_make_wide16_int_const_bits(g, cv.value.lo, cv.value.hi, ty)); 50 api_const_set_top(g, cv); 51 return; 52 } 53 kit_cg_push_int(g, cv.value.known ? cv.value.lo : 0, ty); 54 api_const_set_top(g, cv); 55 } 56 57 void kit_cg_push_float(KitCg* g, double value, KitCgTypeId type) { 58 KitCgTypeId ty; 59 CgTarget* T; 60 ConstBytes cb; 61 union { 62 double d; 63 float f; 64 uint8_t b[8]; 65 } u; 66 CGLocal r; 67 Operand dst; 68 if (!g) return; 69 ty = resolve_type(g->c, type); 70 if (!ty) return; 71 if (api_unevaluated(g)) { 72 api_push(g, api_uneval_value(g, ty)); 73 api_const_set_top(g, api_const_unknown(ty)); 74 return; 75 } 76 if (api_is_f128_type(g->c, ty)) { 77 api_push(g, api_make_f128_const(g, value, ty)); 78 return; 79 } 80 /* Split-lane double: the 8-byte value is memory-resident, so materialize the 81 * IEEE-754 binary64 pattern as two 32-bit lanes. */ 82 if (api_is_wide8_scalar_type(g->c, ty)) { 83 union { 84 double d; 85 u64 u; 86 } bits; 87 bits.d = value; 88 api_push(g, api_make_wide8_const_bits(g, bits.u, ty)); 89 return; 90 } 91 T = g->target; 92 cb.type = ty; 93 cb.size = (u32)abi_cg_sizeof(g->c->abi, type); 94 cb.align = (u32)abi_cg_alignof(g->c->abi, type); 95 if (ty == builtin_id(KIT_CG_BUILTIN_F32)) 96 u.f = (float)value; 97 else 98 u.d = value; 99 cb.bytes = u.b; 100 r = api_alloc_temp_local(g, ty); 101 dst = api_op_local(r, ty); 102 T->load_const(T, dst, cb); 103 api_push(g, api_make_sv(dst, ty)); 104 } 105 106 void kit_cg_push_null(KitCg* g, KitCgTypeId ptr_type) { 107 KitCgTypeId ty; 108 ApiConstValue cv; 109 if (!g) return; 110 ty = resolve_type(g->c, ptr_type); 111 if (!ty) return; 112 cv = api_const_int_result(g, ty, 0, 0, 0); 113 api_push(g, api_make_sv(api_op_imm(0, ty), ty)); 114 api_const_set_top(g, cv); 115 } 116 117 static int api_const_data_can_defer(const KitCg* g) { 118 if (!g || g->opt_level < 1) return 0; 119 if (g->fn_ret_type == KIT_CG_TYPE_NONE) return 0; 120 if (g->data_sym != OBJ_SYM_NONE || g->data_sec != OBJ_SEC_NONE) return 0; 121 if (g->data_local_static_target || g->data_discard || g->data_tls_collect) { 122 return 0; 123 } 124 return g->target && g->target->local_static_data_begin && 125 g->target->local_static_data_write && g->target->local_static_data_end; 126 } 127 128 static int api_const_data_emit_deferred(KitCg* g, ObjSymId sym, 129 KitCgTypeId type, const uint8_t* data, 130 size_t len, uint32_t align) { 131 CGLocalStaticDataDesc desc; 132 KitCgDataDefAttrs data_attrs; 133 if (!api_const_data_can_defer(g)) return 0; 134 memset(&data_attrs, 0, sizeof data_attrs); 135 data_attrs.flags = KIT_CG_DATADEF_FUNCTION_LOCAL | KIT_CG_DATADEF_READONLY; 136 data_attrs.align = align; 137 memset(&desc, 0, sizeof desc); 138 desc.sym = sym; 139 desc.type = type; 140 desc.attrs = data_attrs; 141 desc.align = align; 142 if (!g->target->local_static_data_begin(g->target, &desc)) return 0; 143 if (len) g->target->local_static_data_write(g->target, data, (u64)len); 144 g->target->local_static_data_end(g->target); 145 return 1; 146 } 147 148 KitCgSym kit_cg_const_data(KitCg* g, const uint8_t* data, size_t len, 149 uint32_t align, KitCgTypeId pointee_type) { 150 Compiler* c; 151 ObjBuilder* ob; 152 KitCgTypeId pty; 153 Sym sec_name; 154 ObjSecId sec; 155 u32 base; 156 char name_buf[32]; 157 StrBuf name_sb; 158 Sym anon_name; 159 ObjSymId sym; 160 KitCgDecl attrs; 161 int defer; 162 if (!g) return KIT_CG_SYM_NONE; 163 if (api_unevaluated(g)) return KIT_CG_SYM_NONE; 164 c = g->c; 165 ob = g->obj; 166 pty = resolve_type(c, pointee_type); 167 if (!pty) return KIT_CG_SYM_NONE; 168 align = align ? align : (u32)abi_cg_alignof(c->abi, pointee_type); 169 sec_name = pool_intern_slice(c->global, SLICE_LIT(".rodata")); 170 strbuf_init(&name_sb, name_buf, sizeof(name_buf)); 171 strbuf_put_slice(&name_sb, SLICE_LIT(".Lkit_ro.")); 172 strbuf_put_u64(&name_sb, g->rodata_counter++); 173 anon_name = pool_intern_slice( 174 c->global, 175 (Slice){.s = strbuf_cstr(&name_sb), .len = strbuf_len(&name_sb)}); 176 defer = api_const_data_can_defer(g); 177 sym = defer ? obj_symbol_defer(ob, anon_name, SB_LOCAL, SV_DEFAULT, SK_OBJ, 178 (u64)len) 179 : obj_symbol(ob, anon_name, SB_LOCAL, SK_OBJ, OBJ_SEC_NONE, 0, 180 (u64)len); 181 if (sym == OBJ_SYM_NONE) 182 compiler_panic(c, g->cur_loc, "kit_cg_const_data: symbol failed"); 183 memset(&attrs, 0, sizeof(attrs)); 184 attrs.kind = KIT_CG_DECL_OBJECT; 185 attrs.sym.bind = KIT_SB_LOCAL; 186 attrs.sym.visibility = KIT_CG_VIS_DEFAULT; 187 attrs.as.object.flags = KIT_CG_OBJ_READONLY; 188 api_remember_sym(g, sym, pty, attrs); 189 if (defer && api_const_data_emit_deferred(g, sym, pty, data, len, align)) 190 return (KitCgSym)sym; 191 sec = obj_section(ob, sec_name, SEC_RODATA, SF_ALLOC, align); 192 base = obj_align_to(ob, sec, align); 193 obj_write(ob, sec, data, len); 194 obj_symbol_define_live(ob, sym, sec, base, (u64)len); 195 return (KitCgSym)sym; 196 } 197 198 void api_push_local_lvalue(KitCg* g, CGLocal local, KitCgTypeId type) { 199 if (!g) return; 200 api_push(g, api_make_lv(api_op_local(local, type), type)); 201 api_const_set_top(g, api_const_unknown(type)); 202 } 203 204 void api_push_source_local_lvalue(KitCg* g, KitCgLocal source_local, 205 CGLocal storage, KitCgTypeId type) { 206 ApiSValue sv; 207 if (!g) return; 208 sv = api_make_lv(api_op_local(storage, type), type); 209 sv.source_local = source_local; 210 api_push(g, sv); 211 } 212 213 void kit_cg_push_local(KitCg* g, KitCgLocal local) { 214 ApiSourceLocal* rec; 215 if (!g) return; 216 rec = api_local_from_handle(g, local); 217 if (!rec) return; 218 api_push_source_local_lvalue(g, local, rec->storage, rec->type); 219 } 220 221 void kit_cg_push_local_addr(KitCg* g, KitCgLocal local) { 222 kit_cg_push_local(g, local); 223 kit_cg_addr(g); 224 } 225 226 void kit_cg_push_symbol_addr(KitCg* g, KitCgSym sym, int64_t addend) { 227 KitCgTypeId ty; 228 KitCgTypeId ptr_ty; 229 if (!g) return; 230 ty = api_sym_type(g, sym); 231 if (!ty) ty = builtin_id(KIT_CG_BUILTIN_VOID); 232 ptr_ty = cg_type_ptr_to(g->c, ty); 233 if (api_unevaluated(g)) { 234 api_push(g, api_uneval_value(g, ptr_ty)); 235 api_const_set_top(g, api_const_unknown(ptr_ty)); 236 return; 237 } 238 if (api_sym_is_tls(g, sym)) { 239 CGLocal r = api_alloc_temp_local(g, ptr_ty); 240 Operand dst = api_op_local(r, ptr_ty); 241 g->target->tls_addr_of(g->target, dst, (ObjSymId)sym, addend); 242 api_push(g, api_make_sv(dst, ptr_ty)); 243 } else { 244 api_push(g, 245 api_make_sv(api_op_global((ObjSymId)sym, addend, ptr_ty), ptr_ty)); 246 } 247 } 248 249 static int api_sv_local_storage_is_aggregate(KitCg* g, const ApiSValue* sv) { 250 ApiSourceLocal* rec; 251 if (!sv || sv->op.kind != OPK_LOCAL || 252 sv->source_local == KIT_CG_LOCAL_NONE) { 253 return 0; 254 } 255 rec = api_local_from_handle(g, sv->source_local); 256 return rec && rec->storage == sv->op.v.local && 257 cg_type_is_aggregate(g->c, rec->type); 258 } 259 260 /* Retype an already-addressable place operand to the access type — the way the 261 * old offset-0 EA fold did. GLOBAL/INDIRECT carry the access type into the 262 * operand; a bare LOCAL keeps its own type (the memop's MemAccess carries the 263 * access width). */ 264 static Operand place_operand_for_access(Operand op, KitCgTypeId access_ty) { 265 switch (op.kind) { 266 case OPK_GLOBAL: 267 return api_op_global(op.v.global.sym, op.v.global.addend, access_ty); 268 case OPK_INDIRECT: 269 return api_op_indirect_indexed(op.v.ind.base, op.v.ind.index, 270 op.v.ind.log2_scale, op.v.ind.ofs, 271 access_ty); 272 default: 273 return op; 274 } 275 } 276 277 static KitCgMemAccess api_load_access_with_hints(KitCg* g, 278 KitCgMemAccess access, 279 KitCgTypeId access_ty, 280 int is_bitfield) { 281 if (!is_bitfield && (access.flags & KIT_CG_MEM_SOURCE_SIGNED) && 282 cg_type_is_int(g->c, access_ty) && 283 (u32)api_mem_type_size_resolved(g, access_ty, "load") < 4u) { 284 access.flags |= KIT_CG_MEM_SEXT_LOAD; 285 } 286 return access; 287 } 288 289 /* Load a VALUE from the PLACE on TOS. The place encodes the full address (built 290 * by push_local / deref / field / elem); there is no EA rider. Strict: the 291 * operand must be a PLACE — a pointer VALUE must be `deref`'d first. */ 292 void kit_cg_load(KitCg* g, KitCgMemAccess access) { 293 ApiSValue base; 294 CgTarget* T; 295 KitCgTypeId ty; 296 KitCgTypeId access_ty; 297 KitCgTypeId base_ty; 298 Operand mem_op; 299 CGLocal dst_r; 300 Operand dst; 301 int is_bitfield; 302 if (!g) return; 303 T = g->target; 304 if (access.flags & KIT_CG_MEM_VOLATILE) api_local_const_memory_boundary(g); 305 306 base = api_pop(g); 307 CG_REQUIRE(g, api_is_lvalue_sv(&base), 308 "KitCg: load requires a place; deref the pointer first"); 309 base_ty = api_sv_type(&base); 310 is_bitfield = api_sv_is_bitfield(&base); 311 312 /* Aggregate place: an aggregate-typed access returns the place itself; a 313 * scalar access reads a scalar sub-object and falls through. */ 314 if (!is_bitfield && cg_type_is_aggregate(g->c, base_ty)) { 315 ty = api_mem_access_type(g, access, base_ty, "load"); 316 if (cg_type_is_aggregate(g->c, ty)) { 317 u32 access_size = api_mem_type_size_resolved(g, ty, "load"); 318 u32 lvalue_size = api_mem_type_size_resolved(g, base_ty, "load"); 319 CG_REQUIRE(g, access_size == lvalue_size, 320 "KitCg: load aggregate type/size mismatch: access " 321 "size %u, lvalue size %u", 322 (unsigned)access_size, (unsigned)lvalue_size); 323 base.lang_type = NULL; 324 base.lang_flags = 0; 325 api_push(g, base); 326 return; 327 } 328 /* scalar access from an aggregate place: fall through */ 329 } 330 331 ty = api_mem_access_type(g, access, base_ty, "load"); 332 access_ty = ty; 333 334 if (!is_bitfield) 335 (void)api_require_scalar_mem_type_resolved(g, "load", access_ty); 336 access = api_load_access_with_hints(g, access, access_ty, is_bitfield); 337 338 if (api_unevaluated(g)) { 339 api_release(g, &base); 340 api_push(g, api_uneval_value(g, access_ty)); 341 api_const_set_top(g, api_const_unknown(access_ty)); 342 return; 343 } 344 345 /* A volatile load is an externally observable access: it must emit a real 346 * memory read even when its result is unused, and it must never be served 347 * from the local's register residency or its const shadow. The shadow was 348 * already cleared by the boundary above (so api_local_const_load declines), 349 * but a register-resident scalar local (RES_FIXED_LOCAL) would otherwise be 350 * handed back below with no memory access at all -- the optimizer then DCEs 351 * the would-be load and folds the constant through it. Force the local to a 352 * real memory home (materializing a frame slot, exactly as address-taking 353 * does) and load through it so the access survives. */ 354 if (base.source_local != KIT_CG_LOCAL_NONE && 355 (access.flags & KIT_CG_MEM_VOLATILE) && base.op.kind == OPK_LOCAL) { 356 KitCgTypeId pty = cg_type_ptr_to(g->c, base_ty); 357 Operand addr = api_lvalue_addr(g, &base, pty); 358 /* api_lvalue_addr does not consume the place (it forces the home and 359 * returns its address); retarget base to load through that home. The 360 * general load path below issues the single api_release(&base). */ 361 base = api_make_lv(api_op_indirect(addr.v.local, 0, base_ty), base_ty); 362 } 363 364 /* Source-local constant load. */ 365 if (!is_bitfield && base.source_local != KIT_CG_LOCAL_NONE && 366 api_local_const_load(g, base.source_local, access, &dst)) { 367 api_release(g, &base); 368 api_push(g, api_make_sv(dst, dst.type)); 369 return; 370 } 371 372 /* Scalar local place: the value already lives in the local; hand it back 373 * directly without a memory access. Decode each id's predicate bitset once 374 * and compare exact operational ids. */ 375 if (!is_bitfield && base.source_local != KIT_CG_LOCAL_NONE && 376 base.op.kind == OPK_LOCAL && 377 !api_sv_local_storage_is_aggregate(g, &base)) { 378 u8 base_bits = api_type_pred_bits(g->c, base_ty); 379 u8 ty_bits = api_type_pred_bits(g->c, ty); 380 if (!(base_bits & API_PRED_AGGREGATE) && !(ty_bits & API_PRED_AGGREGATE) && 381 base_ty == ty) { 382 api_sv_set_lvalue(&base, 0); 383 api_sv_set_res(&base, RES_FIXED_LOCAL); 384 api_push(g, base); 385 return; 386 } 387 } 388 389 /* Resolve the place into a single backend memop operand. */ 390 if (!api_operand_can_address(&base.op)) { 391 KitCgTypeId pty = cg_type_ptr_to(g->c, base_ty); 392 Operand addr = api_lvalue_addr(g, &base, pty); 393 mem_op = api_op_indirect(addr.v.local, 0, access_ty); 394 } else { 395 mem_op = place_operand_for_access(base.op, access_ty); 396 } 397 398 if (base.source_local != KIT_CG_LOCAL_NONE) { 399 api_local_const_clear(api_local_from_handle(g, base.source_local)); 400 } 401 402 dst_r = api_alloc_temp_local(g, access_ty); 403 dst = api_op_local(dst_r, access_ty); 404 if (is_bitfield) { 405 /* A bit-field load rides the generic `load` with a bit-field MemAccess; the 406 * CgTarget impl extracts/extends within the storage unit. */ 407 T->load(T, dst, mem_op, api_mem_for_bitfield(g, &base, &mem_op, access_ty)); 408 } else { 409 T->load(T, dst, mem_op, 410 api_mem_from_access_resolved(g, &mem_op, access, access_ty)); 411 } 412 413 api_release(g, &base); 414 api_push(g, api_make_sv(dst, access_ty)); 415 } 416 417 /* VALUE(ptr) -> PLACE: the explicit pointer->place transition. The produced 418 * place is *(ptr + offset bytes). Strict on kind: the operand must be a pointer 419 * VALUE, never a PLACE — the caller turns a place into a pointer with `addr` 420 * first. */ 421 void kit_cg_deref(KitCg* g, int64_t offset) { 422 ApiSValue v; 423 KitCgTypeId pty; 424 KitCgTypeId pointee; 425 Operand ptr; 426 if (!g) return; 427 v = api_pop(g); 428 pty = api_sv_type(&v); 429 CG_REQUIRE(g, !api_is_lvalue_sv(&v) && cg_type_is_ptr(g->c, pty), 430 "KitCg: deref requires a pointer value, not a place"); 431 pointee = cg_type_pointee(g->c, pty); 432 if (!pointee) pointee = builtin_id(KIT_CG_BUILTIN_VOID); 433 if (api_unevaluated(g)) { 434 api_release(g, &v); 435 api_push(g, api_uneval_place(g, pointee)); 436 api_const_set_top(g, api_const_unknown(pointee)); 437 return; 438 } 439 /* A symbol address derefs to a global place, preserving direct 440 * (PC-relative/absolute) addressing rather than materializing the address. */ 441 if (api_sv_kind(&v) == SV_OPERAND && v.op.kind == OPK_GLOBAL) { 442 api_push(g, 443 api_make_lv(api_op_global(v.op.v.global.sym, 444 v.op.v.global.addend + offset, pointee), 445 pointee)); 446 return; 447 } 448 ptr = api_force_local(g, &v, pty); 449 if (offset >= INT32_MIN && offset <= INT32_MAX) { 450 api_push(g, api_make_lv(api_op_indirect(ptr.v.local, (i32)offset, pointee), 451 pointee)); 452 } else { 453 CGLocal r = api_alloc_temp_local(g, pty); 454 Operand p2 = api_op_local(r, pty); 455 g->target->binop(g->target, BO_IADD, p2, ptr, api_op_imm(offset, pty)); 456 api_push(g, api_make_lv(api_op_indirect(r, 0, pointee), pointee)); 457 } 458 } 459 460 void kit_cg_addr(KitCg* g) { 461 ApiSValue v; 462 KitCgTypeId pty; 463 Operand dst; 464 if (!g) return; 465 v = api_pop(g); 466 pty = cg_type_ptr_to(g->c, api_sv_type(&v)); 467 if (v.source_local != KIT_CG_LOCAL_NONE) 468 api_local_const_address_taken(g, v.source_local); 469 if (api_unevaluated(g)) { 470 api_release(g, &v); 471 api_push(g, api_uneval_value(g, pty)); 472 api_const_set_top(g, api_const_unknown(pty)); 473 return; 474 } 475 dst = api_lvalue_addr(g, &v, pty); 476 api_release(g, &v); 477 api_push(g, api_make_sv(dst, pty)); 478 } 479 480 static void api_push_kept_store_value(KitCg* g, const ApiSValue* rv, 481 Operand src, KitCgTypeId ty, 482 ApiConstValue cv) { 483 ApiSValue out = api_make_sv_with_local_ownership( 484 src, ty, api_sv_owns_operand_local(rv, &src)); 485 out.lang_type = rv->lang_type; 486 out.lang_flags = rv->lang_flags; 487 api_push(g, out); 488 api_const_copy_top_from(g, cv); 489 } 490 491 /* Store the VALUE on TOS into the PLACE beneath it. keep_result selects 492 * [place,value] -> [value] for assignment expressions without forcing delayed 493 * RHS values before the scalar-local store fast path can use them. */ 494 static void api_cg_store_impl(KitCg* g, KitCgMemAccess access, 495 int keep_result) { 496 ApiSValue base, rv; 497 ApiConstValue rv_const; 498 CgTarget* T; 499 KitCgTypeId ty; 500 KitCgTypeId access_ty; 501 KitCgTypeId base_ty; 502 KitCgTypeId rv_ty; 503 Operand src; 504 Operand mem_op; 505 int is_lvalue; 506 int is_bitfield; 507 int ty_is_agg; 508 if (!g) return; 509 T = g->target; 510 if (access.flags & KIT_CG_MEM_VOLATILE) api_local_const_memory_boundary(g); 511 512 /* Stack: [base, value] - pop value, then base. */ 513 rv_const = api_const_at(g, 0); 514 rv = api_pop(g); 515 base = api_pop(g); 516 rv_ty = api_sv_type(&rv); 517 base_ty = api_sv_type(&base); 518 is_lvalue = api_is_lvalue_sv(&base); 519 520 CG_REQUIRE(g, is_lvalue, 521 "KitCg: store requires a place destination; deref first"); 522 is_bitfield = api_sv_is_bitfield(&base); 523 524 ty = api_mem_access_type(g, access, base_ty, "store"); 525 access_ty = ty; 526 if (api_unevaluated(g)) { 527 api_release(g, &base); 528 if (keep_result) { 529 api_push(g, rv); 530 api_const_copy_top_from(g, rv_const); 531 } else { 532 api_release(g, &rv); 533 } 534 return; 535 } 536 537 /* Aggregate store: memcpy through the source place. `ty` is stable here, so 538 * decode its aggregate predicate once and reuse it below; likewise decode the 539 * source value's id once for its aggregate/ptr predicate bits. */ 540 ty_is_agg = cg_type_is_aggregate(g->c, ty); 541 u8 rv_bits = api_type_pred_bits(g->c, rv_ty); 542 if (!is_bitfield && (ty_is_agg || (rv_bits & API_PRED_AGGREGATE))) { 543 KitCgTypeId ptr_ty; 544 Operand dst_addr, src_addr; 545 int src_ptr_rvalue; 546 AggregateAccess agg; 547 u32 src_size; 548 u32 dst_size = ty_is_agg ? api_mem_type_size_resolved(g, ty, "store") 549 : api_mem_type_size_resolved(g, base_ty, "store"); 550 u32 access_size = 551 ty_is_agg ? api_mem_type_size_resolved(g, ty, "store") : dst_size; 552 src_ptr_rvalue = !api_is_lvalue_sv(&rv) && (rv_bits & API_PRED_PTR); 553 src_size = src_ptr_rvalue ? access_size 554 : api_mem_type_size_resolved(g, rv_ty, "store"); 555 CG_REQUIRE(g, api_is_lvalue_sv(&rv) || src_ptr_rvalue, 556 "KitCg: aggregate store source is not an lvalue"); 557 CG_REQUIRE(g, access_size == dst_size && access_size == src_size, 558 "KitCg: store aggregate type/size mismatch: access " 559 "size %u, destination size %u, value size %u", 560 (unsigned)access_size, (unsigned)dst_size, (unsigned)src_size); 561 if (base.source_local != KIT_CG_LOCAL_NONE) { 562 api_local_const_clear(api_local_from_handle(g, base.source_local)); 563 } else if (base.op.kind == OPK_INDIRECT || base.op.kind == OPK_GLOBAL || 564 (access.flags & KIT_CG_MEM_VOLATILE)) { 565 api_local_const_memory_boundary(g); 566 } 567 ptr_ty = cg_type_ptr_to(g->c, ty); 568 dst_addr = api_lvalue_addr(g, &base, ptr_ty); 569 if (src_ptr_rvalue) { 570 src_addr = api_force_local(g, &rv, rv_ty); 571 } else { 572 src_addr = api_lvalue_addr(g, &rv, ptr_ty); 573 } 574 memset(&agg, 0, sizeof agg); 575 agg.size = access_size; 576 agg.align = access.align ? access.align : abi_cg_alignof(g->c->abi, ty); 577 T->copy_bytes(T, dst_addr, src_addr, agg); 578 api_release(g, &base); 579 if (keep_result) { 580 api_push(g, rv); 581 api_const_copy_top_from(g, rv_const); 582 return; 583 } 584 api_release(g, &rv); 585 return; 586 } 587 588 if (!is_bitfield) api_validate_memory_value_resolved(g, "store", ty, rv_ty); 589 590 /* A 16-byte scalar immediate (an i128 small constant) only carries 64 bits in 591 * op.v.imm; materialize it into both sign-extended lanes so the general store 592 * path moves a correct 16-byte value rather than load_imm'ing the low lane 593 * and leaving the high half as garbage. */ 594 if (!is_bitfield && api_sv_op_is(&rv, OPK_IMM) && 595 api_is_wide16_scalar_type(g->c, ty)) { 596 rv = api_make_wide16_int_const(g, rv.op.v.imm, ty); 597 } 598 /* Same for a split-lane 8-byte immediate: lower it to a 2-lane memory value 599 * so the store moves a full 64-bit value rather than only the low word. */ 600 if (!is_bitfield && api_sv_op_is(&rv, OPK_IMM) && 601 api_is_wide8_scalar_type(g->c, ty)) { 602 rv = api_make_wide8_int_const(g, rv.op.v.imm, ty); 603 } 604 605 /* Does this store land a scalar value straight into a local's own storage 606 * (a plain `x = <expr>` / `int x = <expr>`, not 607 * bit-field/aggregate/indirect)? Single-decode: `ty`'s aggregate bit was 608 * already taken (ty_is_agg); decode base's id once for its aggregate bit and 609 * compare exact operational ids. */ 610 int scalar_local_place = 611 !is_bitfield && base.source_local != KIT_CG_LOCAL_NONE && 612 base.op.kind == OPK_LOCAL && 613 !api_sv_local_storage_is_aggregate(g, &base) && !ty_is_agg; 614 if (scalar_local_place) { 615 u8 base_bits = api_type_pred_bits(g->c, base_ty); 616 scalar_local_place = !(base_bits & API_PRED_AGGREGATE) && base_ty == ty; 617 } 618 619 /* A still-delayed arith/cmp value going into a scalar local: emit the op 620 * DIRECTLY into the local instead of materializing it into a temp and then 621 * copying that in. Kills the temp->local routing mov for the common 622 * `x = <expr>` — the producer targets the consumer's storage. (Equivalence: 623 * api_materialize_*_to with dst=base.op emits the same op the old temp got, 624 * just landing in base's storage; binops read both sources before writing the 625 * dst, so a self-referential RHS like `b = b + c` stays correct.) */ 626 if (scalar_local_place && 627 (api_sv_kind(&rv) == SV_ARITH || api_sv_kind(&rv) == SV_CMP)) { 628 Operand dst = base.op; 629 if (api_sv_kind(&rv) == SV_ARITH) 630 api_materialize_arith_to(g, &rv, dst); 631 else 632 api_materialize_cmp_to(g, &rv, dst); 633 api_local_const_clear(api_local_from_handle(g, base.source_local)); 634 if (keep_result) { 635 ApiSValue out = api_make_sv(dst, ty); 636 out.lang_type = rv.lang_type; 637 out.lang_flags = rv.lang_flags; 638 api_sv_set_res(&out, RES_FIXED_LOCAL); 639 api_release(g, &base); 640 api_release(g, &rv); 641 api_push(g, out); 642 api_const_copy_top_from(g, rv_const); 643 return; 644 } 645 api_release(g, &base); 646 api_release(g, &rv); 647 return; 648 } 649 650 /* General scalar / bit-field store. Compute the source operand first so its 651 * local lifetime doesn't overlap any addressing arith. */ 652 api_ensure_local(g, &rv); 653 if (api_sv_op_is_local_or_imm(&rv)) { 654 src = rv.op; 655 } else { 656 src = api_force_local(g, &rv, rv_ty); 657 } 658 659 /* Scalar local-resident place, plain store: copy into the local. */ 660 if (scalar_local_place) { 661 Operand dst = base.op; 662 if (src.kind == OPK_IMM) { 663 T->load_imm(T, dst, src.v.imm); 664 if (base.source_local != KIT_CG_LOCAL_NONE) 665 api_local_const_store(g, base.source_local, access, src.v.imm); 666 } else { 667 if (src.kind != OPK_LOCAL) src = api_force_local(g, &rv, ty); 668 if (src.v.local != dst.v.local) { 669 /* `x = <value in a dead transient>`: flag the source dead so the -O0 670 * backend renames its register to x instead of emitting a routing mov 671 * (rv, its last reference, was popped above). */ 672 if (!keep_result && api_coalesce_on(g) && src.kind == OPK_LOCAL && 673 api_temp_dead(g, src.v.local)) 674 src.flags |= OPK_FLAG_KILL; 675 T->copy(T, dst, src); 676 } 677 if (base.source_local != KIT_CG_LOCAL_NONE) 678 api_local_const_clear(api_local_from_handle(g, base.source_local)); 679 } 680 if (keep_result) { 681 src.flags &= (uint8_t)~OPK_FLAG_KILL; 682 api_push_kept_store_value(g, &rv, src, ty, rv_const); 683 } 684 api_release(g, &base); 685 api_release(g, &rv); 686 return; 687 } 688 689 /* Resolve the place into a single backend memop operand. */ 690 if (!api_operand_can_address(&base.op)) { 691 KitCgTypeId pty = cg_type_ptr_to(g->c, base_ty); 692 Operand addr = api_lvalue_addr(g, &base, pty); 693 mem_op = api_op_indirect(addr.v.local, 0, access_ty); 694 } else { 695 mem_op = place_operand_for_access(base.op, access_ty); 696 } 697 698 /* Source-local tracking: only a plain scalar-to-scalar store can fold into a 699 * tracked constant; everything else clears tracking. */ 700 if (base.source_local != KIT_CG_LOCAL_NONE) { 701 if (!is_bitfield && src.kind == OPK_IMM) { 702 api_local_const_store(g, base.source_local, access, src.v.imm); 703 } else { 704 api_local_const_clear(api_local_from_handle(g, base.source_local)); 705 } 706 } else if (base.op.kind == OPK_INDIRECT || base.op.kind == OPK_GLOBAL || 707 (access.flags & KIT_CG_MEM_VOLATILE)) { 708 api_local_const_memory_boundary(g); 709 } 710 711 /* Flag a dead-transient store value so the -O0 backend drops it after the 712 * store instead of spilling it at the next barrier (eager dead-operand drop). 713 * mem_op is the memory place (OPK_INDIRECT/GLOBAL, never a scalar local — 714 * that is the scalar_local_place fast path above), so the dst self-guard is a 715 * no-op and the store has already read the address before the drop. */ 716 if (!keep_result) src = api_op_kill_if_dead(g, src, mem_op); 717 if (is_bitfield) { 718 /* A bit-field store rides the generic `store` with a bit-field MemAccess; 719 * the CgTarget impl does the read-modify-write insert. */ 720 T->store(T, mem_op, src, 721 api_mem_for_bitfield(g, &base, &mem_op, access_ty)); 722 } else { 723 T->store(T, mem_op, src, 724 api_mem_from_access_resolved(g, &mem_op, access, access_ty)); 725 } 726 727 if (keep_result) { 728 src.flags &= (uint8_t)~OPK_FLAG_KILL; 729 api_push_kept_store_value(g, &rv, src, ty, rv_const); 730 } 731 api_release(g, &base); 732 api_release(g, &rv); 733 } 734 735 void kit_cg_store(KitCg* g, KitCgMemAccess access) { 736 api_cg_store_impl(g, access, 0); 737 } 738 739 void kit_cg_store_keep(KitCg* g, KitCgMemAccess access) { 740 api_cg_store_impl(g, access, 1); 741 } 742 743 /* ============================================================ 744 * Stack manipulation 745 * ============================================================ */ 746 747 void kit_cg_dup(KitCg* g) { 748 ApiSValue v, dup; 749 ApiSValue* top; 750 ApiConstValue cv; 751 KitCgTypeId ty; 752 CGLocal r; 753 Operand dst; 754 if (!g || g->sp == 0) return; 755 top = &g->stack[g->sp - 1]; 756 cv = api_const_at(g, 0); 757 if (api_unevaluated(g)) { 758 api_push(g, *top); 759 api_const_copy_top_from(g, cv); 760 return; 761 } 762 api_ensure_local(g, top); 763 v = *top; 764 if (api_sv_res(&v) != RES_LOCAL) { 765 if (api_sv_res(&v) == RES_FIXED_LOCAL && !api_is_lvalue_sv(&v) && 766 v.op.kind == OPK_LOCAL) { 767 ty = api_owned_local_type(g, &v); 768 r = api_alloc_temp_local(g, ty); 769 dst = api_op_local(r, ty); 770 g->target->copy(g->target, dst, 771 api_op_local((CGLocal)api_local_of_sv(&v), ty)); 772 dup = v; 773 api_set_owned_local(&dup, r); 774 api_sv_set_res(&dup, RES_LOCAL); 775 api_sv_set_pinned(&dup, 0); 776 dup.source_local = KIT_CG_LOCAL_NONE; 777 g->stack[g->sp - 1] = dup; 778 api_push(g, v); 779 api_const_copy_top_from(g, cv); 780 return; 781 } 782 api_push(g, v); 783 api_const_copy_top_from(g, cv); 784 return; 785 } 786 if (api_coalesce_on(g)) { 787 /* Lazy dup: push a second reference to the owned temp instead of copying it 788 * into a fresh one. Both entries only ever read the write-once value temp, 789 * so they can share it; the value-stack refcount (now 2) keeps 790 * finer-reclaim and copy-adoption from recycling the temp while it is still 791 * shared, and it is freed once both references are gone (Fix A at the 792 * statement boundary, or finer reclaim when the count returns to 0). 793 * Eliminates the common assignment-result dup whose value is stored once 794 * and then discarded. */ 795 dup = v; 796 api_sv_set_pinned(&dup, 0); 797 api_push(g, dup); 798 api_const_copy_top_from(g, cv); 799 return; 800 } 801 api_sv_set_pinned(top, 1); 802 ty = api_owned_local_type(g, &v); 803 r = api_alloc_temp_local(g, ty); 804 dst = api_op_local(r, ty); 805 g->target->copy(g->target, dst, 806 api_op_local((CGLocal)api_local_of_sv(&v), ty)); 807 api_sv_set_pinned(&g->stack[g->sp - 1], 0); 808 dup = v; 809 api_set_owned_local(&dup, r); 810 api_sv_set_res(&dup, RES_LOCAL); 811 api_sv_set_pinned(&dup, 0); 812 api_push(g, dup); 813 api_const_copy_top_from(g, cv); 814 } 815 816 /* Duplicate the top two value-stack entries. The lower of the two is the deeper 817 * element; the higher is TOS. After dup2, the stack contains [a, b, a, b] 818 * where TOS was [..., a, b]. Used to support compound assignment through a 819 * scaled-index lvalue: the frontend duplicates [base, index] so it can 820 * read-modify-write with a single EA expression each side. 821 * 822 * The current implementation duplicates the two entries one at a time using 823 * kit_cg_dup with a rot3 between them so local/operand sharing stays 824 * correct under the per-entry machinery. */ 825 void kit_cg_dup2(KitCg* g) { 826 if (!g || g->sp < 2) return; 827 /* Stack: [..., a, b] 828 * Step 1: dup the lower (a). We push under TOS by first swapping. */ 829 kit_cg_swap(g); /* [..., b, a] */ 830 kit_cg_dup(g); /* [..., b, a, a] */ 831 kit_cg_rot3(g); /* [..., a, a, b] */ 832 kit_cg_dup(g); /* [..., a, a, b, b] */ 833 /* Now: [..., a, a, b, b]; we want [..., a, b, a, b]. */ 834 /* swap middle two: this is the [..., x, a, b, y]-shaped rotation. We 835 * implement it by rot3 then swap. */ 836 /* Current: ..., a, a, b, b indices (from top): 0=b, 1=b, 2=a, 3=a 837 * 838 * Want: ..., a, b, a, b. Difference: positions 1 (b) and 2 (a) should 839 * swap. We accomplish that by: 840 * rot3 : [..., a, b, b, a] (rotate top 3 forward) 841 * swap : [..., a, b, a, b] 842 */ 843 kit_cg_rot3(g); 844 kit_cg_swap(g); 845 } 846 847 void kit_cg_swap(KitCg* g) { 848 ApiSValue tmp; 849 ApiConstValue ctmp; 850 if (!g || g->sp < 2) return; 851 tmp = g->stack[g->sp - 1]; 852 g->stack[g->sp - 1] = g->stack[g->sp - 2]; 853 g->stack[g->sp - 2] = tmp; 854 ctmp = g->const_stack[g->sp - 1]; 855 g->const_stack[g->sp - 1] = g->const_stack[g->sp - 2]; 856 g->const_stack[g->sp - 2] = ctmp; 857 } 858 859 void kit_cg_drop(KitCg* g) { 860 ApiSValue v; 861 if (!g) return; 862 v = api_pop(g); 863 api_release(g, &v); 864 } 865 866 static KitCgSlotInfo api_slot_info_from_sv(const ApiSValue* sv) { 867 return (KitCgSlotInfo){.cg_type = api_sv_type(sv), 868 .lang_type = sv->lang_type, 869 .lang_flags = sv->lang_flags}; 870 } 871 872 KitCgSlotInfo kit_cg_slot_info(KitCg* g, uint32_t depth_from_top) { 873 if (!g || depth_from_top >= g->sp) return (KitCgSlotInfo){0}; 874 return api_slot_info_from_sv(&g->stack[g->sp - 1u - depth_from_top]); 875 } 876 877 KitCgTypeId kit_cg_slot_cg_type(KitCg* g, uint32_t depth_from_top) { 878 if (!g || depth_from_top >= g->sp) return KIT_CG_TYPE_NONE; 879 return api_sv_type(&g->stack[g->sp - 1u - depth_from_top]); 880 } 881 882 const void* kit_cg_slot_lang_type(KitCg* g, uint32_t depth_from_top) { 883 if (!g || depth_from_top >= g->sp) return NULL; 884 return g->stack[g->sp - 1u - depth_from_top].lang_type; 885 } 886 887 uint16_t kit_cg_slot_lang_flags(KitCg* g, uint32_t depth_from_top) { 888 if (!g || depth_from_top >= g->sp) return 0; 889 return g->stack[g->sp - 1u - depth_from_top].lang_flags; 890 } 891 892 void kit_cg_retag_at(KitCg* g, uint32_t depth_from_top, const void* lang_type, 893 uint16_t lang_flags) { 894 ApiSValue* sv; 895 if (!g || depth_from_top >= g->sp) return; 896 sv = &g->stack[g->sp - 1u - depth_from_top]; 897 sv->lang_type = lang_type; 898 sv->lang_flags = lang_flags; 899 } 900 901 void kit_cg_retag_top(KitCg* g, const void* lang_type, uint16_t lang_flags) { 902 kit_cg_retag_at(g, 0, lang_type, lang_flags); 903 } 904 905 void kit_cg_set_top_flags(KitCg* g, uint16_t set, uint16_t clear) { 906 ApiSValue* sv; 907 if (!g || !g->sp) return; 908 sv = &g->stack[g->sp - 1u]; 909 sv->lang_flags = (uint16_t)((sv->lang_flags | set) & (uint16_t)~clear); 910 } 911 912 uint32_t kit_cg_stack_depth(KitCg* g) { return g ? g->sp : 0u; } 913 914 void kit_cg_rot3(KitCg* g) { 915 ApiSValue a, b, c; 916 ApiConstValue ca, cb, cc; 917 if (!g || g->sp < 3) return; 918 a = g->stack[g->sp - 3]; 919 b = g->stack[g->sp - 2]; 920 c = g->stack[g->sp - 1]; 921 ca = g->const_stack[g->sp - 3]; 922 cb = g->const_stack[g->sp - 2]; 923 cc = g->const_stack[g->sp - 1]; 924 g->stack[g->sp - 3] = b; 925 g->stack[g->sp - 2] = c; 926 g->stack[g->sp - 1] = a; 927 g->const_stack[g->sp - 3] = cb; 928 g->const_stack[g->sp - 2] = cc; 929 g->const_stack[g->sp - 1] = ca; 930 } 931 932 /* ============================================================ 933 * Arithmetic / compare / convert 934 * ============================================================ */ 935 936 const char* api_i128_binop_helper(BinOp op); 937 int api_i128_cmp_is_unsigned(CmpOp op); 938 void api_cg_cmp(KitCg* g, CmpOp cop); 939 void api_f128_call_unary(KitCg* g, const char* name, KitCgTypeId ret, 940 KitCgTypeId param);