data.c (27845B)
1 #include "cg/internal.h" 2 #include "core/vec.h" 3 #include "obj/symresolve.h" 4 5 static void api_data_tls_write_zero(KitCg* g, uint64_t size); 6 7 static SymAttrs api_data_sym_attrs(const ObjSym* s) { 8 SymAttrs a; 9 memset(&a, 0, sizeof a); 10 if (!s) return a; 11 a.bind = s->bind; 12 a.kind = s->kind; 13 a.size = s->size; 14 a.common_align = (s->kind == SK_COMMON) ? (u32)s->common_align : 0u; 15 a.in_comdat = 0; 16 return a; 17 } 18 19 static SymAttrs api_data_decl_attrs(Compiler* c, const KitCgDecl* decl, 20 uint64_t size, uint32_t common_align) { 21 SymAttrs a; 22 memset(&a, 0, sizeof a); 23 if (!decl) return a; 24 a.bind = api_map_bind(decl->sym.bind); 25 a.kind = (decl->as.object.flags & KIT_CG_OBJ_TLS) ? SK_TLS : SK_OBJ; 26 a.size = size; 27 a.common_align = common_align; 28 a.in_comdat = 0; 29 (void)c; 30 return a; 31 } 32 33 static void api_data_clear_state(KitCg* g) { 34 if (!g) return; 35 g->data_sec = OBJ_SEC_NONE; 36 g->data_sym = OBJ_SYM_NONE; 37 g->data_base = 0; 38 g->data_size = 0; 39 g->data_atomize = 0; 40 g->data_retain = 0; 41 g->data_local_static_target = 0; 42 g->data_discard = 0; 43 } 44 45 static void api_data_discard_begin(KitCg* g, ObjSymId sym) { 46 if (!g) return; 47 g->data_sec = OBJ_SEC_NONE; 48 g->data_sym = sym; 49 g->data_base = 0; 50 g->data_size = 0; 51 g->data_atomize = 0; 52 g->data_retain = 0; 53 g->data_local_static_target = 0; 54 g->data_discard = 1; 55 } 56 57 static int api_data_section_is_isolated(const Section* sec, const ObjSym* sym) { 58 if (!sec || !sym || sym->section_id == OBJ_SEC_NONE || sym->value != 0) 59 return 0; 60 if (sec->kind == SEC_BSS || sec->sem == SSEM_NOBITS) 61 return sec->bss_size == sym->size; 62 return sec->bytes.total == sym->size; 63 } 64 65 static void api_data_remove_existing_if_isolated(KitCg* g, const ObjSym* sym) { 66 const Section* sec; 67 if (!g || !sym || sym->section_id == OBJ_SEC_NONE) return; 68 sec = obj_section_get(g->obj, sym->section_id); 69 if (api_data_section_is_isolated(sec, sym)) 70 obj_section_remove(g->obj, sym->section_id); 71 } 72 73 static void api_data_apply_symbol_attrs(KitCg* g, ObjSymId sym, 74 const KitCgDecl* decl) { 75 ObjSym* osym; 76 if (!g || sym == OBJ_SYM_NONE || !decl) return; 77 osym = (ObjSym*)obj_symbol_get(g->obj, sym); 78 if (!osym) return; 79 osym->bind = api_map_bind(decl->sym.bind); 80 osym->vis = api_map_vis(decl->sym.visibility); 81 osym->kind = (decl->as.object.flags & KIT_CG_OBJ_TLS) ? SK_TLS : SK_OBJ; 82 osym->common_align = 0; 83 } 84 85 /* A symbol already defined by the *current* source unit is a same-TU 86 * re-definition — legal C tentative-definition coalescing (`int g; int g;`, 87 * `int g; int g = 5;`, `int arr[]; int arr[3];`). Those re-emit through the 88 * legacy last-writer-wins path; only a definition contributed by a *different* 89 * unit (cross-TU LTO staging) is resolved via symresolve_merge. */ 90 static int api_data_defined_this_unit(const KitCg* g, ObjSymId sym) { 91 if (!g || g->cur_unit_seq == 0 || sym == OBJ_SYM_NONE) return 0; 92 if (sym >= g->sym_def_seq_cap) return 0; 93 return g->sym_def_seq[sym] == g->cur_unit_seq; 94 } 95 96 static void api_data_mark_defined_unit(KitCg* g, ObjSymId sym) { 97 Heap* h; 98 u32* na; 99 u32 cap; 100 if (!g || g->cur_unit_seq == 0 || sym == OBJ_SYM_NONE) return; 101 if (sym >= g->sym_def_seq_cap) { 102 h = g->c->ctx->heap; 103 cap = g->sym_def_seq_cap ? g->sym_def_seq_cap : 16u; 104 while (cap <= sym) cap *= 2u; 105 na = (u32*)h->alloc(h, sizeof(*na) * cap, _Alignof(u32)); 106 if (!na) return; 107 memset(na, 0, sizeof(*na) * cap); 108 if (g->sym_def_seq) { 109 memcpy(na, g->sym_def_seq, sizeof(*na) * g->sym_def_seq_cap); 110 h->free(h, g->sym_def_seq, sizeof(*g->sym_def_seq) * g->sym_def_seq_cap); 111 } 112 g->sym_def_seq = na; 113 g->sym_def_seq_cap = cap; 114 } 115 g->sym_def_seq[sym] = g->cur_unit_seq; 116 } 117 118 static void api_data_debug_object_def(KitCg* g, ObjSymId sym, 119 const KitCgDecl* decl, KitCgTypeId type) { 120 DebugVarLoc loc; 121 DebugTypeId dbg_type; 122 Sym name; 123 if (!g || !g->debug || sym == OBJ_SYM_NONE || !decl || 124 decl->kind != KIT_CG_DECL_OBJECT) { 125 return; 126 } 127 name = (Sym)(decl->display_name ? decl->display_name : decl->linkage_name); 128 if (!name) return; 129 dbg_type = api_debug_type(g, decl->as.object.debug_type, type); 130 if (dbg_type == DEBUG_TYPE_NONE) return; 131 memset(&loc, 0, sizeof loc); 132 loc.kind = DVL_GLOBAL; 133 loc.v.global = sym; 134 if (g->fn_ret_type != KIT_CG_TYPE_NONE) { 135 debug_local(g->debug, name, dbg_type, g->cur_loc, loc); 136 } else { 137 debug_global(g->debug, name, dbg_type, g->cur_loc, loc, 138 decl->sym.bind != KIT_SB_LOCAL); 139 } 140 } 141 142 static void api_data_tls_ensure_materialized(KitCg* g) { 143 if (!g || !g->data_tls_collect || !g->data_tls_zero_fill) return; 144 if (g->data_size) api_data_tls_write_zero(g, g->data_size); 145 g->data_tls_zero_fill = 0; 146 } 147 148 static void api_data_tls_write(KitCg* g, const void* data, size_t len) { 149 if (!g || !len) return; 150 api_data_tls_ensure_materialized(g); 151 buf_write(&g->data_tls_bytes, data, len); 152 g->data_size += len; 153 } 154 155 static void api_data_tls_write_zero(KitCg* g, uint64_t size) { 156 u8 pad[64]; 157 if (!g || !size) return; 158 memset(pad, 0, sizeof pad); 159 while (size >= sizeof pad) { 160 buf_write(&g->data_tls_bytes, pad, sizeof pad); 161 size -= sizeof pad; 162 } 163 if (size) buf_write(&g->data_tls_bytes, pad, (size_t)size); 164 } 165 166 static void api_data_tls_reloc(KitCg* g, KitCgSym target, int64_t addend, 167 RelocKind kind) { 168 Heap* h; 169 ObjTlsReloc* r; 170 if (!g || kind == R_NONE) return; 171 api_data_tls_ensure_materialized(g); 172 h = (Heap*)g->c->ctx->heap; 173 if (VEC_GROW(h, g->data_tls_relocs, g->data_tls_relocs_cap, 174 g->data_tls_nrelocs + 1u)) { 175 compiler_panic(g->c, api_no_loc(), "KitCg: oom on TLS data relocs"); 176 } 177 r = &g->data_tls_relocs[g->data_tls_nrelocs++]; 178 r->offset = (u32)g->data_size; 179 r->kind = kind; 180 r->target = (ObjSymId)target; 181 r->addend = addend; 182 } 183 184 static void api_data_encode_addend(KitCg* g, u8* out, u32 width, 185 int64_t addend) { 186 uint64_t v; 187 if (!g || !out) return; 188 v = (uint64_t)addend; 189 for (u32 i = 0; i < width; ++i) { 190 u32 shift = g->c->target.big_endian ? (width - 1u - i) * 8u : i * 8u; 191 out[i] = (u8)(v >> shift); 192 } 193 } 194 195 void kit_cg_data_begin(KitCg* g, KitCgSym cg_sym, KitCgDataDefAttrs attrs) { 196 Compiler* c; 197 ObjBuilder* ob; 198 ObjSymId sym; 199 KitCgTypeId ty; 200 u32 align; 201 SecKind sec_kind; 202 u16 sec_flags; 203 Sym sec_name_sym; 204 Slice split_base; 205 ObjSecId sec; 206 KitCgDecl decl_attrs; 207 int atomize = 0; 208 if (!g) return; 209 c = g->c; 210 ob = g->obj; 211 sym = (ObjSymId)cg_sym; 212 ty = api_sym_type(g, cg_sym); 213 if (!ty) return; 214 decl_attrs = api_sym_attrs(g, cg_sym); 215 align = 216 attrs.align ? attrs.align : (u32)abi_cg_alignof(c->abi, decl_attrs.type); 217 if (sym != OBJ_SYM_NONE && !api_data_defined_this_unit(g, sym)) { 218 const ObjSym* existing = obj_symbol_get(ob, sym); 219 if (symresolve_sym_is_def(existing)) { 220 SymAttrs old_attrs = api_data_sym_attrs(existing); 221 SymAttrs new_attrs = 222 api_data_decl_attrs(c, &decl_attrs, abi_cg_sizeof(c->abi, ty), 0); 223 SymMergeResult mr = symresolve_merge(old_attrs, new_attrs); 224 switch (mr.kind) { 225 case SYM_MERGE_REPLACE: 226 api_data_remove_existing_if_isolated(g, existing); 227 obj_symbol_set_bind(ob, sym, (SymBind)new_attrs.bind); 228 break; 229 case SYM_MERGE_KEEP_EXISTING: 230 case SYM_MERGE_COMDAT_DISCARD: 231 case SYM_MERGE_COMMON: 232 api_data_discard_begin(g, sym); 233 return; 234 case SYM_MERGE_ODR_ERROR: 235 compiler_panic(c, g->cur_loc, "duplicate definition of symbol"); 236 } 237 } 238 } 239 240 if ((attrs.flags & KIT_CG_DATADEF_FUNCTION_LOCAL) && g->target && 241 g->target->local_static_data_begin) { 242 CGLocalStaticDataDesc desc; 243 memset(&desc, 0, sizeof desc); 244 desc.sym = sym; 245 desc.type = ty; 246 desc.attrs = attrs; 247 desc.align = align; 248 if (g->target->local_static_data_begin(g->target, &desc)) { 249 g->data_sec = OBJ_SEC_NONE; 250 g->data_sym = sym; 251 g->data_base = 0; 252 g->data_size = 0; 253 g->data_local_static_target = 1; 254 api_data_debug_object_def(g, sym, &decl_attrs, ty); 255 return; 256 } 257 } 258 259 if ((decl_attrs.as.object.flags & KIT_CG_OBJ_TLS) && 260 obj_format_tls_via_descriptor(c)) { 261 g->data_sec = OBJ_SEC_NONE; 262 g->data_sym = sym; 263 g->data_base = 0; 264 g->data_size = 0; 265 g->data_tls_collect = 1; 266 g->data_tls_zero_fill = (attrs.flags & KIT_CG_DATADEF_ZERO_FILL) ? 1u : 0u; 267 g->data_tls_align = align ? align : 1u; 268 g->data_tls_nrelocs = 0; 269 buf_init(&g->data_tls_bytes, (Heap*)c->ctx->heap); 270 return; 271 } 272 273 if (!attrs.section && decl_attrs.as.object.section) { 274 attrs.section = decl_attrs.as.object.section; 275 } 276 split_base = SLICE_NULL; 277 if ((decl_attrs.as.object.flags & KIT_CG_OBJ_TLS) && 278 (attrs.flags & KIT_CG_DATADEF_ZERO_FILL)) { 279 sec_kind = SEC_BSS; 280 sec_flags = SF_ALLOC | SF_WRITE | SF_TLS; 281 if (attrs.section) { 282 sec_name_sym = (Sym)attrs.section; 283 } else { 284 sec_name_sym = obj_secname_tbss(c); 285 split_base = pool_slice(c->global, sec_name_sym); 286 } 287 } else if (attrs.flags & KIT_CG_DATADEF_ZERO_FILL) { 288 sec_kind = SEC_BSS; 289 sec_flags = SF_ALLOC | SF_WRITE; 290 if (attrs.section) { 291 sec_name_sym = (Sym)attrs.section; 292 } else { 293 split_base = SLICE_LIT(".bss"); 294 sec_name_sym = pool_intern_slice(c->global, split_base); 295 } 296 } else if (attrs.section) { 297 sec_name_sym = (Sym)attrs.section; 298 if (attrs.flags & KIT_CG_DATADEF_READONLY) { 299 sec_kind = SEC_RODATA; 300 sec_flags = SF_ALLOC; 301 } else { 302 sec_kind = SEC_OTHER; 303 sec_flags = SF_ALLOC | SF_WRITE; 304 } 305 } else if ((attrs.flags & KIT_CG_DATADEF_READONLY) || 306 (decl_attrs.as.object.flags & KIT_CG_OBJ_READONLY)) { 307 sec_kind = SEC_RODATA; 308 sec_flags = SF_ALLOC; 309 split_base = SLICE_LIT(".rodata"); 310 sec_name_sym = pool_intern_slice(c->global, split_base); 311 } else if (decl_attrs.as.object.flags & KIT_CG_OBJ_TLS) { 312 sec_kind = SEC_DATA; 313 sec_flags = SF_ALLOC | SF_WRITE | SF_TLS; 314 sec_name_sym = obj_secname_tdata(c); 315 split_base = pool_slice(c->global, sec_name_sym); 316 } else { 317 sec_kind = SEC_DATA; 318 sec_flags = SF_ALLOC | SF_WRITE; 319 split_base = SLICE_LIT(".data"); 320 sec_name_sym = pool_intern_slice(c->global, split_base); 321 } 322 if (!attrs.section && g->data_sections && split_base.len) { 323 atomize = obj_format_split_sections_as_atoms(c); 324 if (!atomize) { 325 Sym split_name = 326 api_cg_symbol_section_name(g, split_base, decl_attrs.linkage_name); 327 if (split_name) sec_name_sym = split_name; 328 } 329 } 330 if (attrs.flags & KIT_CG_DATADEF_RETAIN) sec_flags |= SF_RETAIN; 331 if (attrs.flags & KIT_CG_DATADEF_MERGE) sec_flags |= SF_MERGE; 332 if (attrs.flags & KIT_CG_DATADEF_STRINGS) sec_flags |= SF_STRINGS; 333 if ((attrs.flags & KIT_CG_DATADEF_ZERO_FILL) && atomize && 334 obj_format_split_sections_as_atoms(c)) { 335 sec = obj_section(ob, sec_name_sym, sec_kind, sec_flags, align); 336 } else if (attrs.flags & KIT_CG_DATADEF_ZERO_FILL) { 337 sec = obj_section_ex(ob, sec_name_sym, sec_kind, SSEM_NOBITS, sec_flags, 338 align, 0, OBJ_SEC_NONE, 0); 339 } else if (attrs.entsize) { 340 sec = obj_section_ex(ob, sec_name_sym, sec_kind, SSEM_PROGBITS, sec_flags, 341 align, attrs.entsize, OBJ_SEC_NONE, 0); 342 } else { 343 sec = obj_section(ob, sec_name_sym, sec_kind, sec_flags, align); 344 } 345 g->data_sec = sec; 346 g->data_sym = sym; 347 g->data_base = obj_align_to(ob, sec, align); 348 g->data_size = 0; 349 g->data_atomize = atomize ? 1u : 0u; 350 g->data_retain = (attrs.flags & KIT_CG_DATADEF_RETAIN) ? 1u : 0u; 351 if (sym != OBJ_SYM_NONE) { 352 api_data_apply_symbol_attrs(g, sym, &decl_attrs); 353 obj_symbol_define(ob, sym, sec, (u64)g->data_base, 354 (u64)abi_cg_sizeof(c->abi, decl_attrs.type)); 355 api_data_mark_defined_unit(g, sym); 356 api_data_debug_object_def(g, sym, &decl_attrs, ty); 357 } 358 } 359 360 void kit_cg_data_common(KitCg* g, KitCgSym cg_sym, uint64_t size, 361 uint32_t align) { 362 ObjSym* osym; 363 ObjSymId sym; 364 KitCgDecl decl_attrs; 365 if (!g || cg_sym == KIT_CG_SYM_NONE) return; 366 sym = (ObjSymId)cg_sym; 367 osym = (ObjSym*)obj_symbol_get(g->obj, sym); 368 if (!osym) return; 369 decl_attrs = api_sym_attrs(g, cg_sym); 370 if (symresolve_sym_is_def(osym) && !api_data_defined_this_unit(g, sym)) { 371 SymAttrs old_attrs = api_data_sym_attrs(osym); 372 SymAttrs new_attrs = api_data_decl_attrs(g->c, &decl_attrs, size, align); 373 SymMergeResult mr; 374 new_attrs.kind = SK_COMMON; 375 mr = symresolve_merge(old_attrs, new_attrs); 376 switch (mr.kind) { 377 case SYM_MERGE_COMMON: 378 osym->bind = new_attrs.bind; 379 osym->vis = api_map_vis(decl_attrs.sym.visibility); 380 osym->kind = SK_COMMON; 381 osym->section_id = OBJ_SEC_NONE; 382 osym->value = 0; 383 osym->size = size; 384 osym->common_align = mr.merged_align; 385 return; 386 case SYM_MERGE_REPLACE: 387 break; 388 case SYM_MERGE_KEEP_EXISTING: 389 case SYM_MERGE_COMDAT_DISCARD: 390 return; 391 case SYM_MERGE_ODR_ERROR: 392 compiler_panic(g->c, g->cur_loc, "duplicate definition of symbol"); 393 } 394 } 395 osym->bind = api_map_bind(decl_attrs.sym.bind); 396 osym->vis = api_map_vis(decl_attrs.sym.visibility); 397 osym->kind = SK_COMMON; 398 osym->section_id = OBJ_SEC_NONE; 399 osym->value = 0; 400 osym->size = size; 401 osym->common_align = align; 402 api_data_mark_defined_unit(g, sym); 403 api_data_debug_object_def(g, sym, &decl_attrs, decl_attrs.type); 404 } 405 406 void kit_cg_data_align(KitCg* g, uint32_t align) { 407 if (g && g->data_discard) return; 408 if (g && g->data_local_static_target) { 409 u32 a = align ? align : 1u; 410 u64 base = (g->data_size + (a - 1u)) & ~(u64)(a - 1u); 411 u64 pad = base - g->data_size; 412 if (pad) kit_cg_data_zero(g, pad); 413 return; 414 } 415 if (g && g->data_tls_collect) { 416 u32 a = align ? align : 1u; 417 u64 base = (g->data_size + (a - 1u)) & ~(u64)(a - 1u); 418 u64 pad = base - g->data_size; 419 if (pad) { 420 if (g->data_tls_zero_fill) { 421 g->data_size += pad; 422 } else { 423 api_data_tls_write_zero(g, pad); 424 g->data_size += pad; 425 } 426 } 427 return; 428 } 429 if (!g || g->data_sec == OBJ_SEC_NONE || !align) return; 430 g->data_size = obj_align_to(g->obj, g->data_sec, align) - g->data_base; 431 } 432 433 void kit_cg_data_pad(KitCg* g, uint64_t size, uint8_t value) { 434 u8 pad[64]; 435 if (!g || !size) return; 436 if (g->data_discard) return; 437 if (g->data_local_static_target) { 438 if (value == 0) { 439 kit_cg_data_zero(g, size); 440 return; 441 } 442 memset(pad, value, sizeof pad); 443 while (size >= sizeof pad) { 444 g->target->local_static_data_write(g->target, pad, sizeof pad); 445 size -= sizeof pad; 446 g->data_size += sizeof pad; 447 } 448 if (size) { 449 g->target->local_static_data_write(g->target, pad, size); 450 g->data_size += size; 451 } 452 return; 453 } 454 if (g->data_tls_collect) { 455 if (value == 0 && g->data_tls_zero_fill) { 456 g->data_size += size; 457 return; 458 } 459 api_data_tls_ensure_materialized(g); 460 memset(pad, value, sizeof pad); 461 while (size >= sizeof pad) { 462 buf_write(&g->data_tls_bytes, pad, sizeof pad); 463 size -= sizeof pad; 464 g->data_size += sizeof pad; 465 } 466 if (size) { 467 buf_write(&g->data_tls_bytes, pad, (size_t)size); 468 g->data_size += size; 469 } 470 return; 471 } 472 memset(pad, value, sizeof(pad)); 473 while (size >= sizeof(pad)) { 474 obj_write(g->obj, g->data_sec, pad, sizeof(pad)); 475 size -= sizeof(pad); 476 g->data_size += sizeof(pad); 477 } 478 if (size) { 479 obj_write(g->obj, g->data_sec, pad, (size_t)size); 480 g->data_size += size; 481 } 482 } 483 484 void kit_cg_data_int(KitCg* g, uint64_t value, KitCgTypeId type) { 485 KitCgTypeId ty; 486 u32 size; 487 u8 bytes[8]; 488 if (!g) return; 489 if (g->data_discard) return; 490 ty = resolve_type(g->c, type); 491 if (!ty) return; 492 size = (u32)abi_cg_sizeof(g->c->abi, type); 493 if (size > sizeof(bytes)) return; 494 for (u32 i = 0; i < size; ++i) { 495 u32 shift = g->c->target.big_endian ? (size - 1u - i) * 8u : i * 8u; 496 bytes[i] = (u8)(value >> shift); 497 } 498 kit_cg_data_bytes(g, bytes, size); 499 } 500 501 void kit_cg_data_float(KitCg* g, double value, KitCgTypeId type) { 502 KitCgTypeId ty; 503 union { 504 float f; 505 double d; 506 u8 b[8]; 507 } u; 508 if (!g) return; 509 if (g->data_discard) return; 510 ty = resolve_type(g->c, type); 511 if (!ty) return; 512 if (api_is_f128_type(g->c, ty)) { 513 u8 bytes[16]; 514 api_encode_binary128_from_double(g, value, bytes); 515 kit_cg_data_bytes(g, bytes, sizeof bytes); 516 return; 517 } 518 if (ty == builtin_id(KIT_CG_BUILTIN_F32)) { 519 u.f = (float)value; 520 if (g->c->target.big_endian) { 521 u8 t = u.b[0]; 522 u.b[0] = u.b[3]; 523 u.b[3] = t; 524 t = u.b[1]; 525 u.b[1] = u.b[2]; 526 u.b[2] = t; 527 } 528 kit_cg_data_bytes(g, u.b, 4); 529 } else if (ty == builtin_id(KIT_CG_BUILTIN_F64)) { 530 u.d = value; 531 if (g->c->target.big_endian) { 532 for (u32 i = 0; i < 4; ++i) { 533 u8 t = u.b[i]; 534 u.b[i] = u.b[7u - i]; 535 u.b[7u - i] = t; 536 } 537 } 538 kit_cg_data_bytes(g, u.b, 8); 539 } 540 } 541 542 void kit_cg_data_bytes(KitCg* g, const uint8_t* data, size_t len) { 543 if (!g || !len) return; 544 if (g->data_discard) return; 545 if (g->data_local_static_target) { 546 g->target->local_static_data_write(g->target, data, (u64)len); 547 g->data_size += len; 548 return; 549 } 550 if (g->data_tls_collect) { 551 api_data_tls_write(g, data, len); 552 return; 553 } 554 obj_write(g->obj, g->data_sec, data, len); 555 g->data_size += len; 556 } 557 558 void kit_cg_data_zero(KitCg* g, uint64_t size) { 559 const Section* sec; 560 if (!g || !size) return; 561 if (g->data_discard) return; 562 if (g->data_local_static_target) { 563 g->target->local_static_data_write(g->target, NULL, size); 564 g->data_size += size; 565 return; 566 } 567 if (g->data_tls_collect) { 568 if (g->data_tls_zero_fill) { 569 g->data_size += size; 570 } else { 571 api_data_tls_write_zero(g, size); 572 g->data_size += size; 573 } 574 return; 575 } 576 sec = obj_section_get(g->obj, g->data_sec); 577 if (sec && (sec->kind == SEC_BSS || sec->sem == SSEM_NOBITS)) { 578 obj_reserve_bss(g->obj, g->data_sec, 579 g->data_base + (u32)(g->data_size + size), 0); 580 g->data_size += size; 581 return; 582 } 583 { 584 u8 pad[64]; 585 memset(pad, 0, sizeof pad); 586 u64 remaining = size; 587 while (remaining >= sizeof pad) { 588 obj_write(g->obj, g->data_sec, pad, sizeof pad); 589 remaining -= sizeof pad; 590 } 591 if (remaining) obj_write(g->obj, g->data_sec, pad, (size_t)remaining); 592 } 593 g->data_size += size; 594 } 595 596 void api_cg_data_reloc(KitCg* g, KitCgSym target, int64_t addend, 597 uint32_t width, int pcrel) { 598 ObjBuilder* ob; 599 RelocKind rk; 600 u8 pad[8]; 601 if (!g || !width || width > sizeof(pad)) return; 602 if (g->data_discard) return; 603 ob = g->obj; 604 rk = api_data_reloc_kind(pcrel, width); 605 if (rk == R_NONE) return; 606 memset(pad, 0, sizeof pad); 607 api_data_encode_addend(g, pad, width, addend); 608 if (g->data_tls_collect) { 609 api_data_tls_reloc(g, target, addend, rk); 610 api_data_tls_write(g, pad, width); 611 return; 612 } 613 obj_write(ob, g->data_sec, pad, width); 614 obj_reloc(ob, g->data_sec, g->data_base + (u32)g->data_size, rk, 615 (ObjSymId)target, addend); 616 g->data_size += width; 617 } 618 619 void kit_cg_data_addr(KitCg* g, KitCgSym target, int64_t addend, uint32_t width, 620 uint32_t address_space) { 621 (void)address_space; 622 if (g && g->data_local_static_target) { 623 compiler_panic(g->c, g->cur_loc, 624 "kit_cg_data_addr: function-local static data address " 625 "relocations are not yet supported by this target"); 626 return; 627 } 628 if (g && g->data_discard) return; 629 api_cg_data_reloc(g, target, addend, width, 0); 630 } 631 632 void kit_cg_data_label_addr(KitCg* g, KitCgLabel target, int64_t addend, 633 uint32_t width, uint32_t address_space) { 634 u8 pad[8]; 635 (void)target; 636 (void)addend; 637 (void)address_space; 638 if (!g) return; 639 if (g->data_discard) return; 640 CG_REQUIRE(g, width && width <= sizeof(pad), 641 "kit_cg_data_label_addr: width must be 1..%u, got %u", 642 (unsigned)sizeof(pad), (unsigned)width); 643 if (g->data_local_static_target) { 644 g->target->local_static_data_label_addr(g->target, (Label)target, addend, 645 width, address_space); 646 g->data_size += width; 647 return; 648 } 649 if (g->target && g->target->data_label_addr_unsupported_msg) { 650 const char* msg = g->target->data_label_addr_unsupported_msg(g->target); 651 if (msg) { 652 compiler_panic(g->c, g->cur_loc, "%s", msg); 653 return; 654 } 655 } 656 compiler_panic(g->c, g->cur_loc, 657 "kit_cg_data_label_addr: target does not support " 658 "intra-function label addresses in data sections"); 659 } 660 661 void kit_cg_data_pcrel(KitCg* g, KitCgSym target, int64_t addend, 662 uint32_t width) { 663 if (g && g->data_local_static_target) { 664 compiler_panic(g->c, g->cur_loc, 665 "kit_cg_data_pcrel: function-local static pcrel data is " 666 "not yet supported by this target"); 667 return; 668 } 669 if (g && g->data_discard) return; 670 api_cg_data_reloc(g, target, addend, width, 1); 671 } 672 673 void kit_cg_data_symdiff(KitCg* g, KitCgSym lhs, KitCgSym rhs, int64_t addend, 674 uint32_t width) { 675 u8 pad[8]; 676 RelocKind add_kind; 677 RelocKind sub_kind; 678 const ObjSym* lhs_sym; 679 const ObjSym* rhs_sym; 680 if (!g || width > sizeof(pad)) return; 681 if (g->data_discard) return; 682 if (g->data_local_static_target) { 683 compiler_panic(g->c, g->cur_loc, 684 "kit_cg_data_symdiff: function-local static symdiff data " 685 "is not yet supported by this target"); 686 return; 687 } 688 lhs_sym = obj_symbol_get(g->obj, (ObjSymId)lhs); 689 rhs_sym = obj_symbol_get(g->obj, (ObjSymId)rhs); 690 if (lhs_sym && rhs_sym && lhs_sym->section_id != OBJ_SEC_NONE && 691 lhs_sym->section_id == rhs_sym->section_id) { 692 u64 value = lhs_sym->value - rhs_sym->value + (u64)addend; 693 for (u32 i = 0; i < width; ++i) pad[i] = (u8)(value >> (i * 8u)); 694 if (g->data_tls_collect) { 695 api_data_tls_write(g, pad, width); 696 } else { 697 obj_write(g->obj, g->data_sec, pad, width); 698 g->data_size += width; 699 } 700 return; 701 } 702 switch (width) { 703 case 1: 704 add_kind = R_ADD8; 705 sub_kind = R_SUB8; 706 break; 707 case 2: 708 add_kind = R_ADD16; 709 sub_kind = R_SUB16; 710 break; 711 case 4: 712 add_kind = R_ADD32; 713 sub_kind = R_SUB32; 714 break; 715 case 8: 716 add_kind = R_ADD64; 717 sub_kind = R_SUB64; 718 break; 719 default: 720 return; 721 } 722 memset(pad, 0, sizeof(pad)); 723 if (g->data_tls_collect) { 724 api_data_tls_reloc(g, lhs, addend, add_kind); 725 api_data_tls_reloc(g, rhs, 0, sub_kind); 726 api_data_tls_write(g, pad, width); 727 return; 728 } 729 obj_write(g->obj, g->data_sec, pad, width); 730 obj_reloc(g->obj, g->data_sec, g->data_base + (u32)g->data_size, add_kind, 731 (ObjSymId)lhs, addend); 732 obj_reloc(g->obj, g->data_sec, g->data_base + (u32)g->data_size, sub_kind, 733 (ObjSymId)rhs, 0); 734 g->data_size += width; 735 } 736 737 void kit_cg_data_end(KitCg* g) { 738 Heap* h; 739 u8* flat; 740 if (!g) return; 741 if (g->data_discard) { 742 api_data_clear_state(g); 743 return; 744 } 745 if (g->data_local_static_target) { 746 g->target->local_static_data_end(g->target); 747 api_data_clear_state(g); 748 return; 749 } 750 if (g->data_tls_collect) { 751 KitCgDecl decl_attrs = api_sym_attrs(g, (KitCgSym)g->data_sym); 752 h = (Heap*)g->c->ctx->heap; 753 flat = NULL; 754 if (!g->data_tls_zero_fill && g->data_size) { 755 flat = (u8*)h->alloc(h, (size_t)g->data_size, 1); 756 if (!flat) 757 compiler_panic(g->c, api_no_loc(), "KitCg: oom on TLS data bytes"); 758 buf_flatten(&g->data_tls_bytes, flat); 759 } 760 api_data_apply_symbol_attrs(g, g->data_sym, &decl_attrs); 761 obj_define_tls(g->c, g->obj, g->data_sym, 762 g->data_tls_zero_fill ? NULL : flat, (u32)g->data_size, 763 g->data_tls_zero_fill ? 0 : 1, g->data_tls_align, 764 g->data_tls_relocs, g->data_tls_nrelocs); 765 api_data_mark_defined_unit(g, g->data_sym); 766 api_data_debug_object_def(g, g->data_sym, &decl_attrs, decl_attrs.type); 767 if (flat) h->free(h, flat, (size_t)g->data_size); 768 buf_fini(&g->data_tls_bytes); 769 if (g->data_tls_relocs) 770 h->free(h, g->data_tls_relocs, 771 sizeof(*g->data_tls_relocs) * g->data_tls_relocs_cap); 772 g->data_tls_relocs = NULL; 773 g->data_tls_relocs_cap = 0; 774 g->data_tls_nrelocs = 0; 775 g->data_tls_collect = 0; 776 g->data_tls_zero_fill = 0; 777 g->data_tls_align = 0; 778 api_data_clear_state(g); 779 return; 780 } 781 if (g->data_sym != OBJ_SYM_NONE) { 782 KitCgDecl decl_attrs = api_sym_attrs(g, (KitCgSym)g->data_sym); 783 api_data_apply_symbol_attrs(g, g->data_sym, &decl_attrs); 784 obj_symbol_define(g->obj, g->data_sym, g->data_sec, g->data_base, 785 g->data_size); 786 } 787 if (g->data_atomize) { 788 obj_atom_define(g->obj, g->data_sec, g->data_base, (u32)g->data_size, 789 g->data_sym, g->data_retain ? OBJ_ATOM_RETAIN : 0u); 790 } 791 api_data_clear_state(g); 792 } 793 794 /* Source targets with a native switch form should override target->switch_. 795 * The old machine-code jump-table path is intentionally not part of the 796 * semantic CgTarget cutover. */ 797 ObjSymId api_emit_label_table(KitCg* g, const Label* labels, u32 n) { 798 Compiler* c; 799 ObjSymId sym; 800 KitCgTypeId void_ptr_ty; 801 KitCgTypeId arr_ty; 802 CGLocalStaticDataDesc desc; 803 KitCgDataDefAttrs attrs; 804 KitCgDecl decl; 805 char name_buf[32]; 806 StrBuf name_sb; 807 Sym name; 808 if (!g || !labels || !n) return OBJ_SYM_NONE; 809 if (!g->target || !g->target->local_static_data_begin || 810 !g->target->local_static_data_write || 811 !g->target->local_static_data_label_addr || 812 !g->target->local_static_data_end) { 813 compiler_panic(g->c, g->cur_loc, 814 "api_emit_label_table: target does not support semantic " 815 "label tables"); 816 } 817 c = g->c; 818 strbuf_init(&name_sb, name_buf, sizeof name_buf); 819 strbuf_put_slice(&name_sb, SLICE_LIT(".Lkit_jt.")); 820 strbuf_put_u64(&name_sb, g->rodata_counter++); 821 name = pool_intern_slice(c->global, (Slice){.s = strbuf_cstr(&name_sb), 822 .len = strbuf_len(&name_sb)}); 823 sym = obj_symbol_defer(g->obj, name, SB_LOCAL, SV_DEFAULT, SK_OBJ, 824 (u64)n * (u64)c->target.ptr_size); 825 if (sym == OBJ_SYM_NONE) 826 compiler_panic(c, g->cur_loc, "api_emit_label_table: symbol failed"); 827 828 void_ptr_ty = cg_type_ptr_to(c, builtin_id(KIT_CG_BUILTIN_VOID)); 829 arr_ty = kit_cg_type_array((KitCompiler*)c, void_ptr_ty, n); 830 memset(&decl, 0, sizeof decl); 831 decl.kind = KIT_CG_DECL_OBJECT; 832 decl.sym.bind = KIT_SB_LOCAL; 833 decl.sym.visibility = KIT_CG_VIS_DEFAULT; 834 decl.as.object.flags = KIT_CG_OBJ_READONLY; 835 api_remember_sym(g, sym, arr_ty, decl); 836 837 memset(&attrs, 0, sizeof attrs); 838 attrs.flags = KIT_CG_DATADEF_FUNCTION_LOCAL | KIT_CG_DATADEF_READONLY; 839 attrs.align = (u32)c->target.ptr_align; 840 memset(&desc, 0, sizeof desc); 841 desc.sym = sym; 842 desc.type = arr_ty; 843 desc.attrs = attrs; 844 desc.align = attrs.align; 845 if (!g->target->local_static_data_begin(g->target, &desc)) { 846 compiler_panic(c, g->cur_loc, 847 "api_emit_label_table: target rejected label table"); 848 } 849 for (u32 i = 0; i < n; ++i) { 850 g->target->local_static_data_label_addr(g->target, labels[i], 0, 851 (u32)c->target.ptr_size, 0); 852 } 853 g->target->local_static_data_end(g->target); 854 return sym; 855 }