link_reloc_layout.c (36505B)
1 /* link_reloc_layout.c — post-section-placement passes: 2 * link_assign_symbol_vaddrs — symbol→vaddr binding (pass 3) 3 * link_emit_array_boundaries — __init_array_start/end etc. 4 * link_emit_tls_boundaries — __tdata_start/end, __tbss_size 5 * link_emit_encoding_section_boundaries — __start_<X>/__stop_<X> 6 * link_layout_jit_stubs — AArch64 JIT CALL26/JUMP26 stubs 7 * link_layout_got — static-PIC .got 8 * link_layout_iplt — STT_GNU_IFUNC trampoline (.iplt etc.) 9 * link_emit_relocations — emit LinkRelocApply records (pass 4) 10 * link_resolve_entry — entry symbol lookup 11 */ 12 13 #include <kit/core.h> 14 #include <string.h> 15 16 #include "core/buf.h" 17 #include "core/bytes.h" 18 #include "core/heap.h" 19 #include "core/pool.h" 20 #include "core/slice.h" 21 #include "core/util.h" 22 #include "core/vec.h" 23 #include "link/link.h" 24 #include "link/link_arch.h" 25 #include "link/link_internal.h" 26 #include "link/link_reloc_desc.h" 27 #include "obj/format.h" 28 29 /* ---- pass 3: assign symbol vaddrs ---- */ 30 31 void link_assign_symbol_vaddrs(Linker* l, LinkImage* img) { 32 u32 ii; 33 for (ii = 0; ii < LinkInputs_count(&l->inputs); ++ii) { 34 LinkInput* in = LinkInputs_at(&l->inputs, ii); 35 ObjBuilder* ob = in->obj; 36 InputMap* m = &img->input_maps[ii]; 37 ObjSymIter* it; 38 ObjSymEntry e; 39 if (in->kind == LINK_INPUT_DSO_BYTES) continue; 40 it = obj_symiter_new(ob); 41 while (obj_symiter_next(it, &e)) { 42 LinkSymId lsid = m->sym[e.id]; 43 LinkSymbol* ls; 44 if (lsid == LINK_SYM_NONE) continue; 45 ls = LinkSyms_at(&img->syms, lsid - 1); 46 if (!ls->defined) continue; 47 if (ls->kind == SK_ABS && ls->vaddr != 0) continue; 48 if (e.sym->section_id == OBJ_SEC_NONE) continue; 49 if (ls->input_id != LinkInputs_at(&l->inputs, ii)->id) continue; 50 ls->section_id = link_input_symbol_section(m, e.sym, e.id); 51 } 52 obj_symiter_free(it); 53 } 54 { 55 u32 i; 56 for (i = 0; i < LinkSyms_count(&img->syms); ++i) { 57 LinkSymbol* s = LinkSyms_at(&img->syms, i); 58 if (s->kind == SK_ABS && s->vaddr != 0) continue; 59 if (!s->defined) continue; 60 if (s->section_id == LINK_SEC_NONE) continue; 61 { 62 const LinkSection* ls = &img->sections[s->section_id - 1]; 63 s->vaddr = ls->vaddr + (s->value - ls->obj_offset); 64 } 65 } 66 } 67 { 68 u32 i; 69 for (i = 0; i < LinkSyms_count(&img->syms); ++i) { 70 LinkSymbol* s = LinkSyms_at(&img->syms, i); 71 if (s->defined) continue; 72 if (s->name == 0) continue; 73 { 74 LinkSymId hit = symhash_get(&img->globals, s->name); 75 if (hit != LINK_SYM_NONE && hit != s->id) { 76 LinkSymbol* def = LinkSyms_at(&img->syms, hit - 1); 77 if (def->defined) { 78 s->section_id = def->section_id; 79 s->value = def->value; 80 s->vaddr = def->vaddr; 81 s->kind = def->kind; 82 s->defined = 1; 83 } 84 } 85 } 86 } 87 } 88 } 89 90 /* ---- pass 3b: boundary symbols ---- */ 91 92 void link_emit_array_boundaries(Linker* l, LinkImage* img) { 93 u32 ii, j; 94 u64 init_start = (u64)-1, init_end = 0; 95 u64 fini_start = (u64)-1, fini_end = 0; 96 u64 preinit_start = (u64)-1, preinit_end = 0; 97 98 for (ii = 0; ii < LinkInputs_count(&l->inputs); ++ii) { 99 ObjBuilder* ob = LinkInputs_at(&l->inputs, ii)->obj; 100 InputMap* m = &img->input_maps[ii]; 101 for (j = 1; j < obj_section_count(ob); ++j) { 102 const Section* s = obj_section_get(ob, j); 103 LinkSectionId ls_id; 104 const LinkSection* ls; 105 u64 start, end; 106 if (!s) continue; 107 if (s->sem != SSEM_INIT_ARRAY && s->sem != SSEM_FINI_ARRAY && 108 s->sem != SSEM_PREINIT_ARRAY) 109 continue; 110 ls_id = m->section[j]; 111 if (ls_id == LINK_SEC_NONE) continue; 112 ls = &img->sections[ls_id - 1]; 113 start = ls->vaddr; 114 end = ls->vaddr + ls->size; 115 if (s->sem == SSEM_INIT_ARRAY) { 116 if (start < init_start) init_start = start; 117 if (end > init_end) init_end = end; 118 } else if (s->sem == SSEM_FINI_ARRAY) { 119 if (start < fini_start) fini_start = start; 120 if (end > fini_end) fini_end = end; 121 } else { 122 if (start < preinit_start) preinit_start = start; 123 if (end > preinit_end) preinit_end = end; 124 } 125 } 126 } 127 128 { 129 u32 i; 130 for (i = 0; i < img->nsections; ++i) { 131 const LinkSection* ls = &img->sections[i]; 132 u64 start, end; 133 if (ls->input_id != LINK_INPUT_NONE) continue; 134 if (ls->sem != SSEM_INIT_ARRAY && ls->sem != SSEM_FINI_ARRAY && 135 ls->sem != SSEM_PREINIT_ARRAY) 136 continue; 137 start = ls->vaddr; 138 end = ls->vaddr + ls->size; 139 if (ls->sem == SSEM_INIT_ARRAY) { 140 if (start < init_start) init_start = start; 141 if (end > init_end) init_end = end; 142 } else if (ls->sem == SSEM_FINI_ARRAY) { 143 if (start < fini_start) fini_start = start; 144 if (end > fini_end) fini_end = end; 145 } else { 146 if (start < preinit_start) preinit_start = start; 147 if (end > preinit_end) preinit_end = end; 148 } 149 } 150 } 151 152 if (init_start == (u64)-1) { 153 init_start = 0; 154 init_end = 0; 155 } 156 if (fini_start == (u64)-1) { 157 fini_start = 0; 158 fini_end = 0; 159 } 160 if (preinit_start == (u64)-1) { 161 preinit_start = 0; 162 preinit_end = 0; 163 } 164 165 link_emit_boundary_sym(l, img, "__init_array_start", init_start); 166 link_emit_boundary_sym(l, img, "__init_array_end", init_end); 167 link_emit_boundary_sym(l, img, "__fini_array_start", fini_start); 168 link_emit_boundary_sym(l, img, "__fini_array_end", fini_end); 169 link_emit_boundary_sym(l, img, "__preinit_array_start", preinit_start); 170 link_emit_boundary_sym(l, img, "__preinit_array_end", preinit_end); 171 } 172 173 void link_emit_tls_boundaries(Linker* l, LinkImage* img) { 174 u64 tdata_start = img->tls_vaddr; 175 u64 tdata_end = img->tls_vaddr + img->tls_filesz; 176 u64 tbss_size = img->tls_memsz - img->tls_filesz; 177 Sym sym_size = pool_intern_slice(l->c->global, SLICE_LIT("__tbss_size")); 178 LinkSymId id; 179 LinkSymbol rec; 180 181 link_emit_boundary_sym(l, img, "__tdata_start", tdata_start); 182 link_emit_boundary_sym(l, img, "__tdata_end", tdata_end); 183 184 id = symhash_get(&img->globals, sym_size); 185 memset(&rec, 0, sizeof(rec)); 186 rec.name = sym_size; 187 rec.kind = SK_ABS; 188 rec.bind = SB_GLOBAL; 189 rec.defined = 1; 190 rec.vaddr = tbss_size; 191 if (id != LINK_SYM_NONE) { 192 *LinkSyms_at(&img->syms, id - 1) = rec; 193 LinkSyms_at(&img->syms, id - 1)->id = id; 194 } else { 195 LinkSymId fresh = link_append_symbol(img, &rec); 196 symhash_insert(&img->globals, sym_size, fresh, &id); 197 } 198 } 199 200 void link_emit_encoding_section_boundaries(Linker* l, LinkImage* img) { 201 u32 i, ii, j; 202 for (i = 0; i < LinkSyms_count(&img->syms); ++i) { 203 LinkSymbol* sym = LinkSyms_at(&img->syms, i); 204 const char* nm; 205 size_t namelen, off, ilen; 206 int is_start; 207 Sym secname; 208 u64 lo = (u64)-1; 209 u64 hi = 0; 210 int found = 0; 211 if (sym->defined) continue; 212 if (sym->name == 0) continue; 213 { 214 Slice nm_s = pool_slice(l->c->global, sym->name); 215 nm = nm_s.s; 216 namelen = nm_s.len; 217 } 218 if (!link_gc_split_start_stop(nm, namelen, &off, &ilen, &is_start)) 219 continue; 220 secname = 221 pool_intern_slice(l->c->global, (Slice){.s = nm + off, .len = ilen}); 222 for (ii = 0; ii < LinkInputs_count(&l->inputs); ++ii) { 223 ObjBuilder* ob = LinkInputs_at(&l->inputs, ii)->obj; 224 InputMap* m = &img->input_maps[ii]; 225 for (j = 1; j < obj_section_count(ob); ++j) { 226 const Section* s = obj_section_get(ob, j); 227 LinkSectionId ls_id; 228 const LinkSection* ls; 229 u64 start, end; 230 if (!s || s->name != secname) continue; 231 ls_id = m->section[j]; 232 if (ls_id == LINK_SEC_NONE) continue; 233 ls = &img->sections[ls_id - 1]; 234 start = ls->vaddr; 235 end = ls->vaddr + ls->size; 236 if (start < lo) lo = start; 237 if (end > hi) hi = end; 238 found = 1; 239 } 240 } 241 if (!found) continue; 242 sym->kind = SK_OBJ; 243 sym->bind = SB_GLOBAL; 244 sym->defined = 1; 245 sym->vaddr = is_start ? lo : hi; 246 } 247 } 248 249 /* ---- iplt alloc helpers (used by layout_jit_call_stubs too) ---- */ 250 251 u32 link_iplt_alloc_segments(LinkImage* img, u32 nseg) { 252 Heap* h = img->heap; 253 u32 base = img->nsegments; 254 u32 new_nseg = base + nseg; 255 LinkSegment* nsegs = (LinkSegment*)h->realloc( 256 h, img->segments, sizeof(*img->segments) * img->nsegments, 257 sizeof(*img->segments) * new_nseg, _Alignof(LinkSegment)); 258 u8** nsbufs = (u8**)h->realloc( 259 h, img->segment_bytes, sizeof(*img->segment_bytes) * img->nsegments, 260 sizeof(*img->segment_bytes) * new_nseg, _Alignof(u8*)); 261 size_t* nscaps = (size_t*)h->realloc( 262 h, img->segment_bytes_cap, 263 sizeof(*img->segment_bytes_cap) * img->nsegments, 264 sizeof(*img->segment_bytes_cap) * new_nseg, _Alignof(size_t)); 265 if (!nsegs || !nsbufs || !nscaps) 266 compiler_panic(img->c, SRCLOC_NONE, "link: oom on iplt segments"); 267 img->segments = nsegs; 268 img->segment_bytes = nsbufs; 269 img->segment_bytes_cap = nscaps; 270 return base; 271 } 272 273 u32 link_iplt_alloc_sections(LinkImage* img, u32 nsec) { 274 Heap* h = img->heap; 275 u32 base = img->nsections; 276 u32 new_nsec = base + nsec; 277 LinkSection* nsections = (LinkSection*)h->realloc( 278 h, img->sections, sizeof(*img->sections) * img->nsections, 279 sizeof(*img->sections) * new_nsec, _Alignof(LinkSection)); 280 if (!nsections) 281 compiler_panic(img->c, SRCLOC_NONE, "link: oom on iplt sections"); 282 img->sections = nsections; 283 return base; 284 } 285 286 /* ---- synthetic region builder ---- 287 * 288 * The GOT, JIT-call-stub, and IPLT passes all need the same shape: a 289 * fresh page-aligned segment carrying a single fixed-size section with a 290 * zero-filled byte buffer. This is the one place that grows the segment / 291 * section arrays for those regions, fills both records, and allocates the 292 * buffer; callers vary only by name / perms / sem / size / section align. 293 * Sequential calls chain naturally — each placement scans the current 294 * max segment end, so a region appended after another lands just past it 295 * (page-aligned), matching the hand-rolled ALIGN_UP(prev_end, page). */ 296 LinkSectionId link_synth_region(LinkImage* img, Linker* l, Sym name, u16 perms, 297 u16 sem, u64 size, u32 sec_align, 298 u64* out_vaddr, u8** out_bytes) { 299 Heap* h = img->heap; 300 u64 page = link_layout_page_size(l); 301 u64 base_vaddr = 0; 302 u32 seg_idx, sec_idx, i; 303 LinkSegment* seg; 304 LinkSection* sec; 305 u8* bytes; 306 307 for (i = 0; i < img->nsegments; ++i) { 308 u64 end = img->segments[i].vaddr + img->segments[i].mem_size; 309 if (end > base_vaddr) base_vaddr = end; 310 } 311 base_vaddr = ALIGN_UP(base_vaddr, (u64)page); 312 313 seg_idx = link_iplt_alloc_segments(img, 1u); 314 seg = &img->segments[seg_idx]; 315 memset(seg, 0, sizeof(*seg)); 316 seg->id = (LinkSegmentId)(seg_idx + 1u); 317 seg->flags = perms; 318 seg->file_offset = base_vaddr; 319 seg->vaddr = base_vaddr; 320 seg->file_size = size; 321 seg->mem_size = size; 322 seg->align = (u32)page; 323 seg->nsections = 1; 324 bytes = (u8*)h->alloc(h, (size_t)size, 16); 325 img->segment_bytes[seg_idx] = bytes; 326 img->segment_bytes_cap[seg_idx] = (size_t)size; 327 if (!bytes) 328 compiler_panic(img->c, SRCLOC_NONE, "link: oom on synth region bytes"); 329 memset(bytes, 0, (size_t)size); 330 img->nsegments += 1u; 331 332 sec_idx = link_iplt_alloc_sections(img, 1u); 333 sec = &img->sections[sec_idx]; 334 memset(sec, 0, sizeof(*sec)); 335 sec->id = (LinkSectionId)(sec_idx + 1u); 336 sec->input_id = LINK_INPUT_NONE; 337 sec->obj_section_id = OBJ_SEC_NONE; 338 sec->segment_id = seg->id; 339 sec->input_offset = 0; 340 sec->file_offset = base_vaddr; 341 sec->vaddr = base_vaddr; 342 sec->size = size; 343 sec->flags = perms; 344 sec->align = sec_align; 345 sec->name = name; 346 sec->sem = sem; 347 img->nsections += 1u; 348 349 if (out_vaddr) *out_vaddr = base_vaddr; 350 if (out_bytes) *out_bytes = bytes; 351 return sec->id; 352 } 353 354 /* One fixed 8-byte ABS64 reloc-apply record for a synthetic slot. */ 355 void link_emit_internal_abs64(LinkImage* img, LinkSectionId lsid, u32 offset, 356 u64 write_vaddr, LinkSymId target) { 357 LinkRelocApply rrec; 358 memset(&rrec, 0, sizeof(rrec)); 359 rrec.input_id = LINK_INPUT_NONE; 360 rrec.section_id = OBJ_SEC_NONE; 361 rrec.link_section_id = lsid; 362 rrec.offset = offset; 363 rrec.width = 8; 364 rrec.write_vaddr = write_vaddr; 365 rrec.write_file_offset = write_vaddr; 366 rrec.kind = R_ABS64; 367 rrec.target = target; 368 rrec.addend = 0; 369 *link_append_reloc_slot(img) = rrec; 370 } 371 372 /* Record the per-stub reloc-apply slots the arch's emit_iplt_stub asks 373 * for: each entry's kind / width / offset_in_stub is variable, so this 374 * mirrors the records the JIT-call-stub and IPLT passes build verbatim. 375 * `stub_base_offset` is the stub's byte offset within its section, 376 * `stub_vaddr` its image-relative base; every record points at the same 377 * `slot_target` GOT slot. */ 378 static void emit_stub_apply_relocs(LinkImage* img, LinkSectionId stub_lsid, 379 u32 stub_base_offset, u64 stub_vaddr, 380 const LinkArchIPltReloc* relocs, u32 nrelocs, 381 LinkSymId slot_target) { 382 u32 ri; 383 for (ri = 0; ri < nrelocs; ++ri) { 384 LinkRelocApply rrec; 385 memset(&rrec, 0, sizeof(rrec)); 386 rrec.input_id = LINK_INPUT_NONE; 387 rrec.section_id = OBJ_SEC_NONE; 388 rrec.link_section_id = stub_lsid; 389 rrec.offset = stub_base_offset + relocs[ri].offset_in_stub; 390 rrec.width = relocs[ri].width; 391 rrec.write_vaddr = stub_vaddr + relocs[ri].offset_in_stub; 392 rrec.write_file_offset = rrec.write_vaddr; 393 rrec.kind = relocs[ri].kind; 394 rrec.target = slot_target; 395 rrec.addend = 0; 396 *link_append_reloc_slot(img) = rrec; 397 } 398 } 399 400 /* ---- pass: JIT call stubs ---- */ 401 402 void link_layout_jit_stubs(Linker* l, LinkImage* img, u32 map_size, 403 LinkSymId** stub_map_out) { 404 Heap* h = img->heap; 405 const LinkArchDesc* arch; 406 LinkSymId* stub_map; 407 LinkSymId* targets = NULL; 408 u32 ntarget = 0, tcap = 0; 409 u32 ii, k, i; 410 u64 stubs_vaddr, slots_vaddr; 411 u64 stubs_size, slots_size; 412 LinkSectionId stubs_sec_id, slots_sec_id; 413 u8* stubs_bytes; 414 415 *stub_map_out = NULL; 416 arch = link_arch_desc_for(l->c); 417 if (l->emit_static_exe) return; 418 if (!arch) return; 419 420 stub_map = (LinkSymId*)h->alloc(h, sizeof(*stub_map) * map_size, 421 _Alignof(LinkSymId)); 422 if (!stub_map) compiler_panic(img->c, SRCLOC_NONE, "link: oom on stub map"); 423 memset(stub_map, 0, sizeof(*stub_map) * map_size); 424 425 for (ii = 0; ii < LinkInputs_count(&l->inputs); ++ii) { 426 ObjBuilder* ob = LinkInputs_at(&l->inputs, ii)->obj; 427 InputMap* m = &img->input_maps[ii]; 428 u32 total = obj_reloc_total(ob); 429 if (!total) continue; 430 for (k = 0; k < total; ++k) { 431 const Reloc* r = obj_reloc_at(ob, k); 432 const Section* s = obj_section_get(ob, r->section_id); 433 LinkSymId target; 434 const LinkSymbol* tgt; 435 if (!s || !link_section_kept(s)) continue; 436 if (link_input_reloc_section(m, r, k) == LINK_SEC_NONE) continue; 437 if (!reloc_kind_is_branch(l->c, r->kind)) continue; 438 if (r->sym == OBJ_SYM_NONE || r->sym >= m->nsym) continue; 439 target = m->sym[r->sym]; 440 if (target == LINK_SYM_NONE) continue; 441 tgt = LinkSyms_at(&img->syms, target - 1); 442 if (!tgt || tgt->kind != SK_ABS) continue; 443 if (stub_map[target] != LINK_SYM_NONE) continue; 444 if (VEC_GROW(h, targets, tcap, ntarget + 1u)) 445 compiler_panic(img->c, SRCLOC_NONE, "link: oom on stub target list"); 446 targets[ntarget] = target; 447 stub_map[target] = (LinkSymId)(ntarget + 1u); 448 ntarget++; 449 } 450 } 451 452 if (ntarget == 0) { 453 if (targets) h->free(h, targets, sizeof(*targets) * tcap); 454 h->free(h, stub_map, sizeof(*stub_map) * map_size); 455 return; 456 } 457 for (i = 0; i < ntarget; ++i) stub_map[targets[i]] = LINK_SYM_NONE; 458 459 stubs_size = (u64)ntarget * (u64)arch->iplt_stub_size; 460 slots_size = (u64)ntarget * 8u; 461 462 stubs_sec_id = link_synth_region( 463 img, l, pool_intern_slice(l->c->global, SLICE_LIT(".kit_jit_call_stubs")), 464 SF_ALLOC | SF_EXEC, SSEM_PROGBITS, stubs_size, 4, &stubs_vaddr, 465 &stubs_bytes); 466 slots_sec_id = link_synth_region( 467 img, l, pool_intern_slice(l->c->global, SLICE_LIT(".kit_jit_call_slots")), 468 SF_ALLOC | SF_WRITE, SSEM_PROGBITS, slots_size, 8, &slots_vaddr, NULL); 469 470 for (i = 0; i < ntarget; ++i) { 471 LinkSymId orig = targets[i]; 472 LinkSymbol* orig_sym = LinkSyms_at(&img->syms, orig - 1); 473 u64 stub_vaddr = stubs_vaddr + (u64)i * (u64)arch->iplt_stub_size; 474 u64 slot_vaddr = slots_vaddr + (u64)i * 8u; 475 LinkSymbol slot_rec, resolver_rec, stub_rec; 476 LinkSymId slot_id, resolver_id, stub_id; 477 LinkArchIPltReloc stub_relocs[2]; 478 u32 nstub_relocs; 479 u8* stub_dst = stubs_bytes + (size_t)i * (size_t)arch->iplt_stub_size; 480 481 nstub_relocs = 482 arch->emit_iplt_stub(stub_dst, stub_vaddr, slot_vaddr, stub_relocs); 483 484 memset(&slot_rec, 0, sizeof(slot_rec)); 485 slot_rec.kind = SK_OBJ; 486 slot_rec.bind = SB_LOCAL; 487 slot_rec.defined = 1; 488 slot_rec.section_id = slots_sec_id; 489 slot_rec.vaddr = slot_vaddr; 490 slot_rec.size = 8; 491 slot_id = link_append_symbol(img, &slot_rec); 492 493 memset(&resolver_rec, 0, sizeof(resolver_rec)); 494 resolver_rec.kind = SK_ABS; 495 resolver_rec.bind = SB_LOCAL; 496 resolver_rec.defined = 1; 497 resolver_rec.vaddr = orig_sym->vaddr; 498 resolver_id = link_append_symbol(img, &resolver_rec); 499 500 memset(&stub_rec, 0, sizeof(stub_rec)); 501 stub_rec.kind = SK_FUNC; 502 stub_rec.bind = SB_LOCAL; 503 stub_rec.defined = 1; 504 stub_rec.section_id = stubs_sec_id; 505 stub_rec.vaddr = stub_vaddr; 506 stub_rec.size = arch->iplt_stub_size; 507 stub_id = link_append_symbol(img, &stub_rec); 508 stub_map[orig] = stub_id; 509 510 emit_stub_apply_relocs(img, stubs_sec_id, (u32)(i * arch->iplt_stub_size), 511 stub_vaddr, stub_relocs, nstub_relocs, slot_id); 512 513 link_emit_internal_abs64(img, slots_sec_id, (u32)(i * 8u), slot_vaddr, 514 resolver_id); 515 } 516 517 if (targets) h->free(h, targets, sizeof(*targets) * tcap); 518 *stub_map_out = stub_map; 519 } 520 521 /* ---- pass 3c: GOT layout ---- */ 522 523 void link_require_local_tls(Compiler* c, const LinkSymbol* tgt) { 524 if (!tgt) return; 525 if (!tgt->imported && tgt->kind == SK_TLS) return; 526 { 527 Slice nm = tgt->name ? pool_slice(c->global, tgt->name) : SLICE_NULL; 528 const char* why = tgt->imported 529 ? "it is imported from a shared object" 530 : "it resolved to a definition that is not " 531 "thread-local"; 532 compiler_panic(c, SRCLOC_NONE, 533 "link: cannot resolve thread-local access to '%.*s': %s. " 534 "kit emits local-exec TLS only, so every thread-local must " 535 "be defined in the image being linked (initial-exec / " 536 "global-dynamic against a shared object are not supported)", 537 (int)nm.len, nm.s ? nm.s : "", why); 538 } 539 } 540 541 /* Fill a TLS Initial-Exec GOT slot with the target's TP-relative offset. 542 * Emitted as an internal raw-64-bit tpoff reloc so apply_all_relocs computes 543 * the offset in the same coordinate system it uses for ordinary local-exec 544 * sites via the neutral R_TPOFF64: x86_64 stores variant II (X - tls_memsz), 545 * AArch64/RISC-V store variant I ((X - tls_vaddr) + TCB). The slot is then 546 * loaded by the GOTTPOFF / GOTTPREL site. */ 547 static void link_emit_internal_tpoff64(LinkImage* img, Linker* l, 548 LinkSectionId lsid, u32 offset, 549 u64 write_vaddr, LinkSymId target) { 550 LinkRelocApply rrec; 551 (void)l; 552 /* The slot holds a tp-relative offset, valid only for a thread-local 553 * defined in this image (see link_require_local_tls). A TLS-IE access 554 * against an imported or non-thread-local target would otherwise bake a 555 * bogus offset into the slot. */ 556 link_require_local_tls(img->c, LinkSyms_at(&img->syms, target - 1)); 557 memset(&rrec, 0, sizeof(rrec)); 558 rrec.input_id = LINK_INPUT_NONE; 559 rrec.section_id = OBJ_SEC_NONE; 560 rrec.link_section_id = lsid; 561 rrec.offset = offset; 562 rrec.width = 8; 563 rrec.write_vaddr = write_vaddr; 564 rrec.write_file_offset = write_vaddr; 565 rrec.kind = R_TPOFF64; 566 rrec.target = target; 567 rrec.addend = 0; 568 *link_append_reloc_slot(img) = rrec; 569 } 570 571 void link_layout_got(Linker* l, LinkImage* img, u32 map_size, 572 LinkSymId** got_map_out) { 573 Heap* h = img->heap; 574 LinkSymId* got_map; 575 LinkSymId* slot_targets = NULL; 576 u8* slot_is_tls = NULL; 577 u32 slot_cap = 0; 578 u32 tls_cap = 0; 579 u32 nslot = 0; 580 u32 ii, k; 581 u64 base_vaddr; 582 u64 got_size; 583 LinkSectionId got_sec_id; 584 u32 si; 585 586 *got_map_out = NULL; 587 588 got_map = 589 (LinkSymId*)h->alloc(h, sizeof(*got_map) * map_size, _Alignof(LinkSymId)); 590 if (!got_map) compiler_panic(img->c, SRCLOC_NONE, "link: oom on got map"); 591 memset(got_map, 0, sizeof(*got_map) * map_size); 592 593 for (ii = 0; ii < LinkInputs_count(&l->inputs); ++ii) { 594 ObjBuilder* ob = LinkInputs_at(&l->inputs, ii)->obj; 595 InputMap* m = &img->input_maps[ii]; 596 u32 total = obj_reloc_total(ob); 597 if (!total) continue; 598 for (k = 0; k < total; ++k) { 599 const Reloc* r = obj_reloc_at(ob, k); 600 const Section* s = obj_section_get(ob, r->section_id); 601 LinkSymId target; 602 if (!s || !link_section_kept(s)) continue; 603 if (link_input_reloc_section(m, r, k) == LINK_SEC_NONE) continue; 604 if (!reloc_kind_uses_got(l->c, r->kind)) continue; 605 if (r->sym == OBJ_SYM_NONE || r->sym >= m->nsym) continue; 606 target = m->sym[r->sym]; 607 if (target == LINK_SYM_NONE) continue; 608 if (got_map[target] != LINK_SYM_NONE) { 609 /* A later reloc on the same target may reveal it is a TLS slot 610 * even if the slot was created by a non-TLS reference first. */ 611 if (reloc_kind_is_tls_got(l->c, r->kind)) 612 slot_is_tls[got_map[target] - 1u] = 1u; 613 continue; 614 } 615 if (VEC_GROW(h, slot_targets, slot_cap, nslot + 1u)) 616 compiler_panic(img->c, SRCLOC_NONE, "link: oom on got slot list"); 617 if (VEC_GROW(h, slot_is_tls, tls_cap, nslot + 1u)) 618 compiler_panic(img->c, SRCLOC_NONE, "link: oom on got slot tls map"); 619 slot_targets[nslot] = target; 620 slot_is_tls[nslot] = reloc_kind_is_tls_got(l->c, r->kind) ? 1u : 0u; 621 got_map[target] = (LinkSymId)(nslot + 1u); 622 nslot++; 623 } 624 } 625 626 if (nslot == 0) { 627 if (slot_targets) 628 h->free(h, slot_targets, sizeof(*slot_targets) * slot_cap); 629 if (slot_is_tls) h->free(h, slot_is_tls, tls_cap); 630 h->free(h, got_map, sizeof(*got_map) * map_size); 631 return; 632 } 633 634 for (si = 0; si < nslot; ++si) got_map[slot_targets[si]] = LINK_SYM_NONE; 635 636 got_size = (u64)nslot * 8u; 637 got_sec_id = link_synth_region( 638 img, l, pool_intern_slice(img->c->global, SLICE_LIT(".got")), 639 SF_ALLOC | SF_WRITE, SSEM_PROGBITS, got_size, 8, &base_vaddr, NULL); 640 641 for (si = 0; si < nslot; ++si) { 642 LinkSymId orig = slot_targets[si]; 643 u64 slot_vaddr = base_vaddr + (u64)si * 8u; 644 LinkSymbol sym_rec; 645 LinkSymId slot_id; 646 647 memset(&sym_rec, 0, sizeof(sym_rec)); 648 sym_rec.name = 0; 649 sym_rec.kind = SK_OBJ; 650 sym_rec.bind = SB_LOCAL; 651 sym_rec.defined = 1; 652 sym_rec.section_id = got_sec_id; 653 sym_rec.vaddr = slot_vaddr; 654 sym_rec.size = 8; 655 slot_id = link_append_symbol(img, &sym_rec); 656 got_map[orig] = slot_id; 657 658 if (slot_is_tls[si]) 659 link_emit_internal_tpoff64(img, l, got_sec_id, (u32)(si * 8u), slot_vaddr, 660 orig); 661 else 662 link_emit_internal_abs64(img, got_sec_id, (u32)(si * 8u), slot_vaddr, 663 orig); 664 } 665 666 if (slot_targets) h->free(h, slot_targets, sizeof(*slot_targets) * slot_cap); 667 if (slot_is_tls) h->free(h, slot_is_tls, tls_cap); 668 669 *got_map_out = got_map; 670 } 671 672 /* ---- pass 3d: STT_GNU_IFUNC trampoline ---- */ 673 674 void link_layout_iplt(Linker* l, LinkImage* img) { 675 Heap* h = img->heap; 676 u32 i; 677 u32 nifunc = 0; 678 u64 iplt_vaddr, igot_vaddr, pairs_vaddr; 679 u64 iplt_size, igot_size, pairs_size; 680 u64 init_vaddr = 0, init_size = 0; 681 LinkSectionId iplt_sec_id, igot_sec_id, pairs_sec_id, init_sec_id = 0; 682 u8* iplt_bytes; 683 u32 slot_idx; 684 /* FreeBSD's crt resolves static IFUNCs by walking [__rela_iplt_start, 685 * __rela_iplt_end) of R_*_IRELATIVE relocs in __libc_start1, *before* 686 * _init_tls -- earlier than any (pre)init ctor can run, which matters 687 * because _init_tls allocates through malloc -> getenv -> IFUNC string 688 * ops. Emit that standard table (also the glibc mechanism) instead of the 689 * ctor-based __kit_ifunc_init path for hosted FreeBSD static links. Other 690 * targets (musl/freestanding) keep the ctor, whose crt does not walk 691 * __rela_iplt. */ 692 int use_rela_iplt = 693 l->emit_static_exe && obj_format_static_ifunc_via_rela_iplt(l->c); 694 int emit_init_array = l->emit_static_exe && !use_rela_iplt; 695 LinkSectionId rela_iplt_sec_id = 0; 696 u64 rela_iplt_vaddr = 0, rela_iplt_size = 0; 697 u8* rela_iplt_bytes = NULL; 698 u32 irelative_type = 699 use_rela_iplt ? obj_format_static_ifunc_irelative_type(l->c) : 0u; 700 LinkSymId ifunc_init_sym = LINK_SYM_NONE; 701 Sym ifunc_init_name = 0; 702 const LinkArchDesc* arch = link_arch_desc_for(l->c); 703 if (!arch) 704 compiler_panic(img->c, SRCLOC_NONE, 705 "link: layout_iplt: no arch descriptor"); 706 707 for (i = 0; i < LinkSyms_count(&img->syms); ++i) { 708 const LinkSymbol* s = LinkSyms_at(&img->syms, i); 709 if (s->kind != SK_IFUNC || !s->defined) continue; 710 if (s->name != 0) { 711 LinkSymId canonical = symhash_get(&img->globals, s->name); 712 if (canonical != LINK_SYM_NONE && canonical != s->id) continue; 713 } 714 ++nifunc; 715 } 716 if (nifunc == 0) return; 717 718 iplt_size = (u64)nifunc * (u64)arch->iplt_stub_size; 719 igot_size = (u64)nifunc * 8u; 720 pairs_size = (u64)nifunc * 16u; 721 722 /* Validate the ctor target before mutating the image: the panic must 723 * fire ahead of any synthetic region being appended. */ 724 if (emit_init_array) { 725 ifunc_init_name = 726 pool_intern_slice(l->c->global, SLICE_LIT("__kit_ifunc_init")); 727 ifunc_init_sym = symhash_get(&img->globals, ifunc_init_name); 728 if (ifunc_init_sym == LINK_SYM_NONE || 729 !LinkSyms_at(&img->syms, ifunc_init_sym - 1)->defined) { 730 compiler_panic(img->c, SRCLOC_NONE, 731 "link: STT_GNU_IFUNC requires '__kit_ifunc_init' " 732 "to be defined (link in libkit_rt.a or provide " 733 "your own implementation)"); 734 } 735 init_size = 8u; 736 } 737 738 iplt_sec_id = link_synth_region( 739 img, l, pool_intern_slice(l->c->global, SLICE_LIT(".iplt")), 740 SF_ALLOC | SF_EXEC, SSEM_PROGBITS, iplt_size, 4, &iplt_vaddr, 741 &iplt_bytes); 742 igot_sec_id = link_synth_region( 743 img, l, pool_intern_slice(l->c->global, SLICE_LIT(".igot.plt")), 744 SF_ALLOC | SF_WRITE, SSEM_PROGBITS, igot_size, 8, &igot_vaddr, NULL); 745 pairs_sec_id = link_synth_region( 746 img, l, pool_intern_slice(l->c->global, SLICE_LIT(".iplt.pairs")), 747 SF_ALLOC | SF_WRITE, SSEM_PROGBITS, pairs_size, 8, &pairs_vaddr, NULL); 748 if (emit_init_array) 749 init_sec_id = link_synth_region(img, l, obj_secname_preinit_array(l->c), 750 SF_ALLOC | SF_WRITE, SSEM_PREINIT_ARRAY, 751 init_size, 8, &init_vaddr, NULL); 752 753 link_emit_section_boundary_sym(l, img, "__start_iplt_pairs", pairs_sec_id, 0); 754 link_emit_section_boundary_sym(l, img, "__stop_iplt_pairs", pairs_sec_id, 755 pairs_size); 756 757 if (use_rela_iplt) { 758 /* One Elf64_Rela (24 bytes) per IFUNC: r_offset = igot slot, r_info = 759 * IRELATIVE, r_addend = resolver. crt walks the bracketed range. */ 760 rela_iplt_size = (u64)nifunc * 24u; 761 rela_iplt_sec_id = link_synth_region( 762 img, l, pool_intern_slice(l->c->global, SLICE_LIT(".rela.plt")), 763 SF_ALLOC, SSEM_PROGBITS, rela_iplt_size, 8, &rela_iplt_vaddr, 764 &rela_iplt_bytes); 765 link_emit_boundary_sym(l, img, "__rela_iplt_start", rela_iplt_vaddr); 766 link_emit_boundary_sym(l, img, "__rela_iplt_end", 767 rela_iplt_vaddr + rela_iplt_size); 768 } 769 770 img->iplt_pairs = (u64*)h->alloc( 771 h, sizeof(*img->iplt_pairs) * 2u * (size_t)nifunc, _Alignof(u64)); 772 if (!img->iplt_pairs) 773 compiler_panic(img->c, SRCLOC_NONE, "link: oom on iplt pairs"); 774 img->niplt = nifunc; 775 776 slot_idx = 0; 777 778 for (i = 0; i < LinkSyms_count(&img->syms); ++i) { 779 LinkSymbol* s = LinkSyms_at(&img->syms, i); 780 u64 stub_vaddr; 781 u64 slot_vaddr; 782 u64 pair_vaddr; 783 u64 resolver_vaddr; 784 LinkSectionId resolver_section; 785 u64 resolver_value; 786 LinkSymbol slot_rec; 787 LinkSymbol resolver_rec; 788 LinkSymId slot_id; 789 LinkSymId resolver_id; 790 LinkArchIPltReloc iplt_relocs[2]; 791 u32 niplt_relocs; 792 u8* stub_dst; 793 794 if (s->kind != SK_IFUNC || !s->defined) continue; 795 if (s->name != 0) { 796 LinkSymId canonical = symhash_get(&img->globals, s->name); 797 if (canonical != LINK_SYM_NONE && canonical != s->id) continue; 798 } 799 800 stub_vaddr = iplt_vaddr + (u64)slot_idx * 12u; 801 slot_vaddr = igot_vaddr + (u64)slot_idx * 8u; 802 pair_vaddr = pairs_vaddr + (u64)slot_idx * 16u; 803 resolver_vaddr = s->vaddr; 804 resolver_section = s->section_id; 805 resolver_value = s->value; 806 807 img->iplt_pairs[2u * slot_idx + 0] = resolver_vaddr; 808 img->iplt_pairs[2u * slot_idx + 1] = slot_vaddr; 809 810 stub_dst = iplt_bytes + (size_t)slot_idx * (size_t)arch->iplt_stub_size; 811 niplt_relocs = 812 arch->emit_iplt_stub(stub_dst, stub_vaddr, slot_vaddr, iplt_relocs); 813 814 memset(&slot_rec, 0, sizeof(slot_rec)); 815 slot_rec.name = 0; 816 slot_rec.kind = SK_OBJ; 817 slot_rec.bind = SB_LOCAL; 818 slot_rec.defined = 1; 819 slot_rec.section_id = igot_sec_id; 820 slot_rec.vaddr = slot_vaddr; 821 slot_rec.size = 8; 822 slot_id = link_append_symbol(img, &slot_rec); 823 824 memset(&resolver_rec, 0, sizeof(resolver_rec)); 825 resolver_rec.name = 0; 826 resolver_rec.kind = SK_FUNC; 827 resolver_rec.bind = SB_LOCAL; 828 resolver_rec.defined = 1; 829 resolver_rec.section_id = resolver_section; 830 resolver_rec.value = resolver_value; 831 resolver_rec.vaddr = resolver_vaddr; 832 resolver_rec.size = 0; 833 resolver_id = link_append_symbol(img, &resolver_rec); 834 835 emit_stub_apply_relocs(img, iplt_sec_id, 836 (u32)(slot_idx * arch->iplt_stub_size), stub_vaddr, 837 iplt_relocs, niplt_relocs, slot_id); 838 839 link_emit_internal_abs64(img, pairs_sec_id, (u32)(slot_idx * 16u), 840 pair_vaddr, resolver_id); 841 link_emit_internal_abs64(img, pairs_sec_id, (u32)(slot_idx * 16u + 8u), 842 pair_vaddr + 8u, slot_id); 843 844 if (use_rela_iplt) { 845 /* Elf64_Rela: r_offset(+0)=slot, r_info(+8)=IRELATIVE, r_addend(+16)= 846 * resolver. r_info is a fixed constant (sym index 0); r_offset and 847 * r_addend are filled by internal ABS64 relocs so they pick up the 848 * final (post-shift) vaddrs. */ 849 u64 rela_off = (u64)slot_idx * 24u; 850 u64 rela_ent_vaddr = rela_iplt_vaddr + rela_off; 851 if (rela_iplt_bytes) 852 wr_u64_le(rela_iplt_bytes + rela_off + 8u, (u64)irelative_type); 853 link_emit_internal_abs64(img, rela_iplt_sec_id, (u32)rela_off, 854 rela_ent_vaddr, slot_id); 855 link_emit_internal_abs64(img, rela_iplt_sec_id, (u32)(rela_off + 16u), 856 rela_ent_vaddr + 16u, resolver_id); 857 } 858 859 s->kind = SK_FUNC; 860 s->section_id = iplt_sec_id; 861 s->value = (u64)slot_idx * (u64)arch->iplt_stub_size; 862 s->vaddr = stub_vaddr; 863 s->size = arch->iplt_stub_size; 864 865 ++slot_idx; 866 } 867 868 if (emit_init_array) 869 link_emit_internal_abs64(img, init_sec_id, 0, init_vaddr, ifunc_init_sym); 870 871 { 872 u32 n = LinkSyms_count(&img->syms); 873 for (i = 0; i < n; ++i) { 874 LinkSymbol* s = LinkSyms_at(&img->syms, i); 875 LinkSymId canonical; 876 LinkSymbol* def; 877 if (s->name == 0) continue; 878 canonical = symhash_get(&img->globals, s->name); 879 if (canonical == LINK_SYM_NONE || canonical == s->id) continue; 880 def = LinkSyms_at(&img->syms, canonical - 1); 881 if (def->section_id != iplt_sec_id) continue; 882 s->section_id = def->section_id; 883 s->value = def->value; 884 s->vaddr = def->vaddr; 885 s->kind = def->kind; 886 s->size = def->size; 887 s->defined = 1; 888 } 889 } 890 } 891 892 /* ---- entry symbol ---- */ 893 894 void link_resolve_entry(Linker* l, LinkImage* img) { 895 LinkSymId id; 896 LinkSymbol* s; 897 if (l->entry_name == 0) return; 898 id = symhash_get(&img->globals, l->entry_name); 899 if (id == LINK_SYM_NONE) { 900 Slice nm_s = pool_slice(l->c->global, l->entry_name); 901 const char* nm = nm_s.s; 902 size_t namelen = nm_s.len; 903 compiler_panic(l->c, SRCLOC_NONE, "link: entry symbol '%.*s' not defined", 904 (int)namelen, nm); 905 } 906 s = LinkSyms_at(&img->syms, id - 1); 907 if (!s->defined) { 908 Slice nm_s = pool_slice(l->c->global, l->entry_name); 909 const char* nm = nm_s.s; 910 size_t namelen = nm_s.len; 911 compiler_panic(l->c, SRCLOC_NONE, "link: entry symbol '%.*s' is undefined", 912 (int)namelen, nm); 913 } 914 img->entry_sym = id; 915 } 916 917 /* ---- pass 4: emit reloc records ---- */ 918 919 void link_emit_relocations(Linker* l, LinkImage* img, const LinkSymId* got_map, 920 const LinkSymId* stub_map) { 921 u32 ii; 922 for (ii = 0; ii < LinkInputs_count(&l->inputs); ++ii) { 923 ObjBuilder* ob = LinkInputs_at(&l->inputs, ii)->obj; 924 InputMap* m = &img->input_maps[ii]; 925 u32 total = obj_reloc_total(ob); 926 u32 k; 927 if (total == 0) continue; 928 for (k = 0; k < total; ++k) { 929 const Reloc* r = obj_reloc_at(ob, k); 930 const Section* s = obj_section_get(ob, r->section_id); 931 LinkSymId target; 932 LinkSection* ls; 933 LinkRelocApply rec; 934 if (!s || (!link_section_kept(s) && !link_section_kept_fileonly(s))) 935 continue; 936 if (link_input_reloc_section(m, r, k) == LINK_SEC_NONE) continue; 937 if (r->kind == R_RV_RELAX || r->kind == R_RV_TPREL_ADD || 938 r->kind == R_RV_ALIGN) 939 continue; 940 if (r->sym == OBJ_SYM_NONE || r->sym >= m->nsym) 941 compiler_panic(l->c, SRCLOC_NONE, 942 "link: reloc references unknown symbol"); 943 target = m->sym[r->sym]; 944 if (target == LINK_SYM_NONE) 945 compiler_panic(l->c, SRCLOC_NONE, 946 "link: reloc references unmapped symbol"); 947 if (got_map && reloc_kind_uses_got(l->c, r->kind)) { 948 LinkSymId slot = got_map[target]; 949 if (slot == LINK_SYM_NONE) 950 compiler_panic(l->c, SRCLOC_NONE, 951 "link: GOT slot missing for symbol"); 952 target = slot; 953 } 954 if (stub_map && reloc_kind_is_branch(l->c, r->kind)) { 955 LinkSymId stub = stub_map[target]; 956 if (stub != LINK_SYM_NONE) target = stub; 957 } 958 { 959 LinkSectionId src_lsid = link_input_reloc_section(m, r, k); 960 if (src_lsid == LINK_SEC_NONE) continue; 961 ls = &img->sections[src_lsid - 1]; 962 } 963 memset(&rec, 0, sizeof(rec)); 964 rec.input_id = LinkInputs_at(&l->inputs, ii)->id; 965 rec.section_id = r->section_id; 966 rec.link_section_id = ls->id; 967 rec.offset = r->offset - (u32)ls->obj_offset; 968 rec.width = reloc_kind_width(l->c, (RelocKind)r->kind); 969 rec.write_vaddr = ls->vaddr + rec.offset; 970 rec.write_file_offset = ls->file_offset + rec.offset; 971 rec.kind = (RelocKind)r->kind; 972 rec.target = target; 973 rec.addend = r->addend; 974 if (rec.width == 0) 975 compiler_panic(l->c, SRCLOC_NONE, "link: unsupported reloc kind %u", 976 (unsigned)r->kind); 977 /* The ULEB128 SET/SUB relocs are variable-width (their true length is 978 * read from the bytes at apply time, so rec.width is only a sentinel). 979 * Guard the offset against the section end here — where the size is 980 * known — so the apply-time scan starts in-bounds even for a malformed 981 * external object. */ 982 if ((rec.kind == R_SET_ULEB128 || rec.kind == R_SUB_ULEB128) && 983 rec.offset >= ls->size) 984 compiler_panic(l->c, SRCLOC_NONE, 985 "link: ULEB128 reloc offset past section end"); 986 *link_append_reloc_slot(img) = rec; 987 } 988 } 989 }