commit f8832785ff35d8f461594f58a894dc1d234411d0
parent 277327a487828145938a8eeca9946f9f28d76256
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Tue, 9 Jun 2026 15:19:59 -0700
jit: abstract the reloc/relaxation path over arch and object format
The in-process JIT linker carried two near-identical relocation loops --
kit_jit_from_image (bulk) and jit_apply_one_reloc (incremental append) -- that
each inlined arch-specific instruction byte-patching and had drifted: the
append loop held a stale, partial Mach-O TLV rewrite, and the bulk loop's TLV
relax hard-coded x16 as the descriptor-thunk scratch register and panicked on
clang objects that pick x8 (the Apple TLV ABI fixes only x0, in/out).
Unify both into one shared jit_apply_reloc() and push every arch/format
identity behind hooks and reloc-descriptor flags, so link_jit.c's reloc path
names zero RelocKind, object-format, or target.arch/obj identity:
- LinkArchDesc.jit_reloc_relax (src/arch/<arch>/reloc.c): Mach-O TLV-descriptor
collapse (any scratch register), GOT-load collapse on the append path (via
JitRelaxCtx.got_relaxed), and weak-undef zeroing on aarch64; PCREL_LO12
paired-AUIPC displacement recompute on riscv (link_jit owns the LinkImage
scan via a resolve_pair callback); NULL on x86-64.
- TLS Local-Exec classification is now flag-only: RELOC_IS_TLS_LE (ELF tp
idiom plus the aarch64 COFF SECREL12A pair) and a new RELOC_IS_SECREL
(x86-64 COFF SECREL, a TLS access when its target is SK_TLS). A new
RELOC_IS_PCREL_ANCHOR marks the riscv HI20 anchors. RelocDesc.flags widened
u8 -> u16 (the byte was full).
- The Windows _tls_index drop moves to obj_format_jit_drops_symbol_ref in
obj_tls.c, beside obj_format_tls_model, so the format-specific pseudo-symbol
name stays out of src/link.
Behavior-preserving: the new flags are read only by the JIT, and the frozen
reloc oracles pass unchanged (reloc_desc_test 2949, reloc_apply_test 50 -- they
check width plus the got/branch/tlvp/direct_page predicates, never the new
flags). Verified green: test-link (124), test-macho (80), test-coff (78),
test-toy (1388), jit_tls_relax_test (206), and riscv64/riscv32 jit image build.
Diffstat:
11 files changed, 300 insertions(+), 217 deletions(-)
diff --git a/src/arch/aa64/link.c b/src/arch/aa64/link.c
@@ -178,6 +178,7 @@ void aa64_emit_macho_stub(u8* out, u64 stub_vaddr, u64 got_slot_vaddr) {
const RelocDesc* aa64_reloc_desc(RelocKind);
int aa64_reloc_apply_insn(Compiler*, RelocKind, u8*, u64, i64, u64);
void aa64_jit_tls_le_relax(Compiler*, RelocKind, u8*, u64, u64);
+int aa64_jit_reloc_relax(Compiler*, RelocKind, const JitRelaxCtx*);
/* AArch64 __chkstk for PE/COFF: probes `x15 * 16` bytes of stack one page at a
* time, then returns. Mirrors the LLVM compiler-rt implementation (chkstk.S in
@@ -206,6 +207,7 @@ const LinkArchDesc link_arch_aa64 = {
.reloc_desc = aa64_reloc_desc,
.reloc_apply_insn = aa64_reloc_apply_insn,
.jit_tls_le_relax = aa64_jit_tls_le_relax,
+ .jit_reloc_relax = aa64_jit_reloc_relax,
.coff_chkstk_bytes = aa64_coff_chkstk,
.coff_chkstk_len = sizeof aa64_coff_chkstk,
diff --git a/src/arch/aa64/reloc.c b/src/arch/aa64/reloc.c
@@ -15,6 +15,7 @@
#include "obj/reloc.h"
#include "core/bytes.h"
+#include "link/link_arch.h"
static const RelocDescRow aa64_rows[] = {
{R_AARCH64_JUMP26, {4, RELOC_IS_BRANCH}},
@@ -40,9 +41,10 @@ static const RelocDescRow aa64_rows[] = {
{R_AARCH64_TLVP_LOAD_PAGE21, {4, RELOC_IS_TLVP}},
{R_AARCH64_TLVP_LOAD_PAGEOFF12, {4, RELOC_IS_TLVP}},
/* COFF AArch64 TLS SECREL imm12 pair: ADD-imm12 instruction relocs,
- * AArch64-only, applied only into PE/COFF output. */
- {R_COFF_AARCH64_SECREL_LOW12A, {4, 0}},
- {R_COFF_AARCH64_SECREL_HIGH12A, {4, 0}},
+ * AArch64-only, applied only into PE/COFF output. TLS-only, so the JIT
+ * classifies them as Local-Exec accesses via RELOC_IS_TLS_LE. */
+ {R_COFF_AARCH64_SECREL_LOW12A, {4, RELOC_IS_TLS_LE}},
+ {R_COFF_AARCH64_SECREL_HIGH12A, {4, RELOC_IS_TLS_LE}},
};
const RelocDesc* aa64_reloc_desc(RelocKind k) {
@@ -281,3 +283,75 @@ void aa64_jit_tls_le_relax(Compiler* c, RelocKind k, u8* site, u64 storage,
aa64_reloc_apply_insn(c, R_AARCH64_ADD_ABS_LO12_NC, site, storage, 0, site_pc);
wr_u32_le(add_lo, 0xd503201fu); /* nop */
}
+
+/* In-process JIT relaxation of AArch64 indirection idioms (LinkArchDesc
+ * .jit_reloc_relax): the single-threaded JIT has no dynamic loader, GOT, or
+ * TLV resolver, so each access idiom is rewritten to address the in-image
+ * instance directly. Returns 1 if it owned `k`, 0 to fall through to the
+ * ordinary reloc apply. */
+int aa64_jit_reloc_relax(Compiler* c, RelocKind k, const JitRelaxCtx* ctx) {
+ u8* P_bytes = ctx->site;
+
+ /* Weak undefined target: address-of must evaluate to NULL. An ADRP + ADD
+ * pair would form a PC-relative address to vaddr 0 that exceeds ±4 GiB once
+ * the JIT places segments far from 0 (tripping link_reloc's range check).
+ * Rewrite the ADRP to MOVZ rd,#0 so rd becomes 0 directly; the paired ADD's
+ * assembled imm12 of 0 already gives rd += 0, so leave it as add rd,rd,#0. */
+ if (ctx->weak_undef_zero) {
+ if (k == R_AARCH64_ADR_PREL_PG_HI21 ||
+ k == R_AARCH64_ADR_PREL_PG_HI21_NC) {
+ u32 rd = rd_u32_le(P_bytes) & 0x1fu;
+ wr_u32_le(P_bytes, 0xd2800000u | rd); /* movz rd, #0 */
+ return 1;
+ }
+ if (k == R_AARCH64_ADD_ABS_LO12_NC) return 1; /* leave add rd,rd,#0 */
+ }
+
+ /* Mach-O TLV access -> ordinary in-image load. Codegen emits the 4-insn
+ * Apple TLV sequence:
+ * adrp x0, desc@TLVPPAGE (PAGE21)
+ * ldr x0, [x0, desc@TLVPPAGEOFF] (PAGEOFF12) <- this reloc
+ * ldr xN, [x0] -- load the resolver thunk from desc[0]
+ * blr xN -- call thunk(desc) -> &var in x0
+ * With one thread the in-image .tdata/.tbss IS the single instance, and
+ * desc[+16] already holds the variable's in-image storage address (filled by
+ * the normal R_ABS64 against the storage symbol). Collapse to a direct load,
+ * dropping the thunk and the per-thread block:
+ * PAGEOFF12 : ldr x0,[x0,#imm] -> add x0,x0,#(desc & 0xfff) (x0 = &desc)
+ * +4 : ldr xN,[x0] -> ldr x0,[x0,#16] (x0 = &var)
+ * +8 : blr xN -> nop
+ * The thunk register N is scratch (the Apple TLV ABI fixes only x0:
+ * descriptor in, &var out); kit's codegen uses x16, clang picks any free
+ * register (e.g. x8). Accept any N so long as the pair is `ldr xN,[x0]`
+ * (Rn=x0, imm12=0) followed by `blr xN`. */
+ if (k == R_AARCH64_TLVP_LOAD_PAGEOFF12) {
+ u64 v = ((u64)ctx->S + (u64)ctx->addend) & 0xfffu;
+ u32 instr = rd_u32_le(P_bytes);
+ u8* i_thunk = P_bytes + 4u;
+ u8* i_call = P_bytes + 8u;
+ u32 thunk = rd_u32_le(i_thunk);
+ u32 call = rd_u32_le(i_call);
+ u32 n = thunk & 0x1fu;
+ wr_u32_le(P_bytes, 0x91000000u | (instr & 0x3ffu) | ((u32)v << 10));
+ if ((thunk & ~0x1fu) != 0xf9400000u || call != (0xd63f0000u | (n << 5u)))
+ compiler_panic(c, SRCLOC_NONE,
+ "kit_jit: unexpected Mach-O TLV access sequence");
+ wr_u32_le(i_thunk, 0xf9400800u); /* ldr x0, [x0, #16] -> &var */
+ wr_u32_le(i_call, 0xd503201fu); /* nop */
+ return 1;
+ }
+
+ /* No real GOT in the append image: the GOT load becomes a direct add, so the
+ * register holds the symbol address itself instead of loading it from a slot.
+ * LD64_GOT_LO12_NC shares the LDR uimm12 encoding; rewrite to ADD imm12. */
+ if (ctx->got_relaxed && k == R_AARCH64_LD64_GOT_LO12_NC) {
+ u64 v = ((u64)ctx->S + (u64)ctx->addend) & 0xfffu;
+ u32 instr = rd_u32_le(P_bytes);
+ u32 rd = instr & 0x1fu;
+ u32 rn = (instr >> 5) & 0x1fu;
+ wr_u32_le(P_bytes, 0x91000000u | rd | (rn << 5) | ((u32)v << 10));
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/src/arch/riscv/link.c b/src/arch/riscv/link.c
@@ -115,6 +115,7 @@ static u32 rv32_emit_iplt_stub(u8* dst, u64 stub_vaddr, u64 slot_vaddr,
const RelocDesc* rv_reloc_desc(RelocKind);
int rv_reloc_apply_insn(Compiler*, RelocKind, u8*, u64, i64, u64);
void rv_jit_tls_le_relax(Compiler*, RelocKind, u8*, u64, u64);
+int rv_jit_reloc_relax(Compiler*, RelocKind, const JitRelaxCtx*);
const LinkArchDesc link_arch_rv64 = {
.plt0_size = RV64_PLT0_SIZE,
@@ -128,6 +129,7 @@ const LinkArchDesc link_arch_rv64 = {
.reloc_desc = rv_reloc_desc,
.reloc_apply_insn = rv_reloc_apply_insn,
.jit_tls_le_relax = rv_jit_tls_le_relax,
+ .jit_reloc_relax = rv_jit_reloc_relax,
};
/* RV32 link descriptor: identical to rv64 (PLT0/entry/stub byte sizes,
@@ -145,4 +147,5 @@ const LinkArchDesc link_arch_rv32 = {
.emit_iplt_stub = rv32_emit_iplt_stub,
.reloc_desc = rv_reloc_desc,
.reloc_apply_insn = rv_reloc_apply_insn,
+ .jit_reloc_relax = rv_jit_reloc_relax,
};
diff --git a/src/arch/riscv/reloc.c b/src/arch/riscv/reloc.c
@@ -20,6 +20,7 @@
#include "obj/reloc.h"
#include "core/bytes.h"
+#include "link/link_arch.h"
static const RelocDescRow rv_rows[] = {
{R_RV_HI20, {4, 0}},
@@ -27,10 +28,10 @@ static const RelocDescRow rv_rows[] = {
{R_RV_LO12_S, {4, 0}},
{R_RV_BRANCH, {4, 0}},
{R_RV_JAL, {4, 0}},
- {R_RV_PCREL_HI20, {4, 0}},
+ {R_RV_PCREL_HI20, {4, RELOC_IS_PCREL_ANCHOR}},
{R_RV_PCREL_LO12_I, {4, 0}},
{R_RV_PCREL_LO12_S, {4, 0}},
- {R_RV_GOT_HI20, {4, RELOC_USES_GOT}},
+ {R_RV_GOT_HI20, {4, RELOC_USES_GOT | RELOC_IS_PCREL_ANCHOR}},
{R_RV_TLS_GOT_HI20, {4, RELOC_IS_TLS_GOT}},
{R_RV_TPREL_HI20, {4, RELOC_IS_TLS_LE}},
{R_RV_TPREL_LO12_I, {4, RELOC_IS_TLS_LE}},
@@ -302,3 +303,19 @@ void rv_jit_tls_le_relax(Compiler* c, RelocKind k, u8* site, u64 storage,
addi_w = rd_u32_le(addi);
wr_u32_le(addi, (addi_w & 0x000fffffu) | (((u32)lo12 & 0xfffu) << 20));
}
+
+/* In-process JIT relaxation of RISC-V indirection idioms (LinkArchDesc
+ * .jit_reloc_relax). Today only PCREL_LO12: it has no S of its own — its
+ * low-12 bits must match the paired AUIPC's PC-relative displacement, which
+ * the JIT recomputes against its layout via ctx->resolve_pair. Feed that
+ * displacement to the LO12_I/S encoder (same instruction encoding; the addend
+ * is unused per the psABI). Returns 1 if it owned `k`, 0 otherwise. */
+int rv_jit_reloc_relax(Compiler* c, RelocKind k, const JitRelaxCtx* ctx) {
+ if (k == R_RV_PCREL_LO12_I || k == R_RV_PCREL_LO12_S) {
+ i64 disp = ctx->resolve_pair(ctx->resolve_user, ctx->anchor_vaddr);
+ RelocKind alias = (k == R_RV_PCREL_LO12_I) ? R_RV_LO12_I : R_RV_LO12_S;
+ rv_reloc_apply_insn(c, alias, ctx->site, (u64)disp, 0, ctx->site_pc);
+ return 1;
+ }
+ return 0;
+}
diff --git a/src/link/link_arch.h b/src/link/link_arch.h
@@ -32,6 +32,38 @@ typedef struct LinkArchIPltReloc {
RelocKind kind;
} LinkArchIPltReloc;
+/* Context for LinkArchDesc.jit_reloc_relax — the in-process JIT's per-reloc
+ * "collapse runtime indirection to direct in-image addressing" pass. The
+ * single-threaded JIT has no dynamic loader, GOT, or TLV resolver, so the
+ * arch rewrites the access idiom in place. The hook reads only the fields
+ * relevant to the kinds it owns; the rest cost nothing. */
+typedef struct JitRelaxCtx {
+ u8* site; /* write-alias bytes at the reloc site */
+ u64 S; /* resolved runtime symbol address (0 when weak_undef_zero) */
+ i64 addend; /* the reloc's addend */
+ u64 site_pc; /* runtime address of the reloc site */
+
+ /* Address-of a weak *undefined* symbol must read as 0. Set when the target
+ * is weak + SK_ABS + vaddr==0; the arch zeroes the materialized register
+ * rather than forming a (huge, out-of-range) PC-relative address. */
+ int weak_undef_zero;
+
+ /* The image has no real GOT — the incremental-append path does not re-run
+ * GOT layout, so GOT-load relocs must be relaxed to direct in-image
+ * addressing. 0 for the full from-image link, where .got slots exist and
+ * GOT loads apply normally (S already points at the slot). */
+ int got_relaxed;
+
+ /* RISC-V PCREL_LO12 pairs with an AUIPC anchor whose runtime PC-relative
+ * displacement must be recomputed against the JIT layout. resolve_pair
+ * returns that displacement for the anchor at image vaddr `anchor_vaddr`;
+ * the arch calls it opaquely (the implementation, in link_jit.c, needs
+ * LinkImage internals the arch must not reach). */
+ i64 (*resolve_pair)(void* user, u64 anchor_vaddr);
+ void* resolve_user;
+ u64 anchor_vaddr;
+} JitRelaxCtx;
+
typedef struct LinkArchDesc {
/* ---- PLT geometry ----
* All three arches today use a 32-byte PLT0 + 16-byte per-import
@@ -112,6 +144,16 @@ typedef struct LinkArchDesc {
void (*jit_tls_le_relax)(Compiler* c, RelocKind k, u8* site, u64 storage,
u64 site_pc);
+ /* In-process JIT only: collapse one relocation's runtime indirection to
+ * direct in-image addressing — the Mach-O TLV descriptor call, a GOT load
+ * (append path), the weak-undef-zero materialization, and the RISC-V
+ * PCREL_LO12 anchor recompute. The arch owns the instruction byte rewrites
+ * for exactly the kinds it recognizes (consulting JitRelaxCtx for the
+ * weak/GOT/pair context). Returns 1 if it handled `k`, 0 to fall through to
+ * link_reloc_apply. NULL on arches with no JIT relaxation beyond TLS LE
+ * (x86-64). */
+ int (*jit_reloc_relax)(Compiler* c, RelocKind k, const JitRelaxCtx* ctx);
+
/* TLS variant: 1 = variant II (x86-64, tpoff = X - tls_memsz_rounded);
* 0 = variant I (AArch64/RISC-V, tpoff = (X - tls_vaddr) + tcb_bias).
* Consulted by the ELF linker when applying R_TPOFF64. */
diff --git a/src/link/link_jit.c b/src/link/link_jit.c
@@ -121,38 +121,24 @@ static int perms_for(u32 secflags) {
return p;
}
-/* Windows (COFF) Local-Exec TLS in the single-threaded in-process JIT.
- *
- * The codegen emits the PE TLS-access idiom: read TEB.ThreadLocalStoragePointer,
- * index it by the module's `_tls_index`, then `+ SECREL(var)`. That cannot run
- * under the JIT — there is no loaded PE the OS could assign a `_tls_index` to,
- * nor a per-module TLS block in this thread's TEB array. As with ELF Local-Exec
- * (RELOC_IS_TLS_LE), the JIT relaxes the idiom to address the single in-image
- * .tls instance directly: the terminal SECREL reloc(s) drive the relaxation
- * (jit_tls_le_relax), and the idiom's `_tls_index` ADRP/PC32 relocs are dropped
- * because those instructions get rewritten away. */
+/* A reference the single-threaded JIT drops because the format's TLS access
+ * idiom that materializes its target is relaxed in-image. On Windows/COFF the
+ * idiom reads TEB.ThreadLocalStoragePointer, indexes it by the module's
+ * `_tls_index`, then `+ SECREL(var)` — there is no loaded PE for the OS to
+ * assign a `_tls_index`, so jit_tls_le_relax rewrites that load away and the
+ * `_tls_index` ADRP/PC32 relocs must not be applied. The format authority
+ * (obj_format_jit_drops_symbol_ref) owns which symbol that is. */
static int jit_tls_is_index_ref(Compiler* c, const LinkSymbol* tgt) {
- Slice nm;
- if (obj_format_tls_model(c) != OBJ_TLS_WINDOWS_TEB) return 0;
- if (!tgt || tgt->name == 0) return 0;
- nm = pool_slice(c->global, tgt->name);
- return nm.len == 10u && memcmp(nm.s, "_tls_index", 10u) == 0;
-}
-
-/* True for the terminal SECREL reloc(s) of the Windows TLS idiom: aarch64's
- * SECREL12A kinds are TLS-only, while x86-64's generic SECREL is a TLS access
- * exactly when its target is a TLS symbol. */
-static int jit_reloc_is_win_tls_le(Compiler* c, RelocKind k, u8 tgt_kind) {
- if (obj_format_tls_model(c) != OBJ_TLS_WINDOWS_TEB) return 0;
- return k == R_COFF_AARCH64_SECREL_HIGH12A ||
- k == R_COFF_AARCH64_SECREL_LOW12A ||
- (k == R_COFF_SECREL && tgt_kind == SK_TLS);
+ return tgt && obj_format_jit_drops_symbol_ref(c, tgt->name);
}
-/* Unified TLS Local-Exec classifier for the JIT reloc passes: the arch-neutral
- * RELOC_IS_TLS_LE flag (ELF) plus the COFF Windows idiom above. */
+/* TLS Local-Exec classifier for the JIT reloc passes, entirely flag-driven:
+ * the per-arch tp-relative idiom and the aa64 COFF SECREL12A pair carry
+ * RELOC_IS_TLS_LE; x86-64's generic COFF SECREL (RELOC_IS_SECREL) is a TLS
+ * access exactly when its target is a TLS symbol. */
static int jit_reloc_is_tls_le(Compiler* c, RelocKind k, u8 tgt_kind) {
- return reloc_kind_is_tls_le(c, k) || jit_reloc_is_win_tls_le(c, k, tgt_kind);
+ return reloc_kind_is_tls_le(c, k) ||
+ (reloc_kind_is_secrel(c, k) && tgt_kind == SK_TLS);
}
/* Find the segment that contains image-relative `vaddr` and return its
@@ -217,7 +203,7 @@ static i64 jit_rv_pcrel_lo12_disp(LinkImage* img, KitExecMemRegion* segs,
const LinkRelocApply* hi = LinkRelocs_at(&img->relocs, i);
const LinkSymbol* hi_tgt;
u64 hi_S, hi_P;
- if (hi->kind != R_RV_PCREL_HI20 && hi->kind != R_RV_GOT_HI20) continue;
+ if (!reloc_kind_is_pcrel_anchor(img->c, hi->kind)) continue;
if (hi->write_vaddr != auipc_image_vaddr) continue;
hi_tgt = LinkSyms_at(&img->syms, hi->target - 1);
if (!hi_tgt) continue;
@@ -267,6 +253,92 @@ static void jit_copy_input_section_bytes(LinkImage* img,
}
}
+/* Opaque user data for the JitRelaxCtx.resolve_pair callback: the arch hook
+ * (RISC-V PCREL_LO12) gets the paired AUIPC's runtime displacement without
+ * reaching into LinkImage itself. */
+typedef struct {
+ LinkImage* img;
+ KitExecMemRegion* segs;
+} JitPairResolve;
+
+static i64 jit_resolve_pcrel_pair(void* user, u64 anchor_vaddr) {
+ JitPairResolve* p = (JitPairResolve*)user;
+ return jit_rv_pcrel_lo12_disp(p->img, p->segs, anchor_vaddr);
+}
+
+/* Apply one relocation against the JIT runtime base. Shared by the bulk
+ * from-image pass and the incremental-append pass so their reloc handling can
+ * never drift. Patch-site bytes go through the write alias; PC-relative
+ * arithmetic uses the runtime alias.
+ *
+ * Order mirrors the resolution chain: drop Windows `_tls_index` refs, relax
+ * Local-Exec TLS via the arch idiom hook, then resolve S and offer the reloc
+ * to the arch's JIT-relaxation hook (TLV / GOT / weak-undef / RISC-V pcrel
+ * pairing) before the ordinary apply. `got_relaxed` is 1 on the append path
+ * (no real GOT was built) and 0 for the full link (where .got slots exist). */
+static void jit_apply_reloc(Compiler* c, LinkImage* img, KitExecMemRegion* segs,
+ const LinkRelocApply* r, int got_relaxed) {
+ const LinkSymbol* tgt = LinkSyms_at(&img->syms, r->target - 1);
+ const LinkArchDesc* d;
+ JitPairResolve pair;
+ JitRelaxCtx ctx;
+ u64 S, P;
+ u8* P_bytes;
+
+ /* Windows TLS idiom `_tls_index` references are dropped: the access is
+ * relaxed to in-image addressing, so the index load is rewritten away. */
+ if (jit_tls_is_index_ref(c, tgt)) return;
+
+ /* Local-Exec TLS -> in-image addressing (single-threaded JIT). The per-arch
+ * idiom rewrite lives behind LinkArchDesc.jit_tls_le_relax; this stays
+ * arch-neutral, classifying ELF via RELOC_IS_TLS_LE and the COFF Windows
+ * idiom via its terminal SECREL reloc(s). */
+ if (jit_reloc_is_tls_le(c, r->kind, tgt->kind)) {
+ u64 storage = (tgt->kind == SK_ABS)
+ ? tgt->vaddr + (u64)r->addend
+ : (u64)vaddr_to_runtime(img, segs, tgt->vaddr) +
+ (u64)r->addend;
+ u8* bytes = (u8*)vaddr_to_write(img, segs, r->write_vaddr);
+ u64 site_pc = (u64)vaddr_to_runtime(img, segs, r->write_vaddr);
+ d = link_arch_desc_for(c);
+ if (!d || !d->jit_tls_le_relax || !bytes)
+ compiler_panic(c, SRCLOC_NONE,
+ "kit_jit: target has no TLS Local-Exec relaxation");
+ d->jit_tls_le_relax(c, r->kind, bytes, storage, site_pc);
+ return;
+ }
+
+ /* extern resolver result OR true absolute symbol — vaddr already holds the
+ * runtime address; otherwise map the image vaddr to its runtime alias. */
+ S = (tgt->kind == SK_ABS) ? tgt->vaddr
+ : (u64)vaddr_to_runtime(img, segs, tgt->vaddr);
+ P = (u64)vaddr_to_runtime(img, segs, r->write_vaddr);
+ P_bytes = (u8*)vaddr_to_write(img, segs, r->write_vaddr);
+ if (!P_bytes)
+ compiler_panic(c, SRCLOC_NONE, "kit_jit: relocation site is unmapped");
+
+ /* Arch JIT relaxation: collapse the runtime indirection (Mach-O TLV
+ * descriptor call, GOT load on the append path, weak-undef zeroing, RISC-V
+ * PCREL_LO12 anchor recompute) to direct in-image addressing. Returns 1 if
+ * it owned the kind; otherwise fall through to the ordinary apply. */
+ pair.img = img;
+ pair.segs = segs;
+ ctx.site = P_bytes;
+ ctx.S = S;
+ ctx.addend = r->addend;
+ ctx.site_pc = P;
+ ctx.weak_undef_zero =
+ (tgt->bind == SB_WEAK && tgt->kind == SK_ABS && tgt->vaddr == 0);
+ ctx.got_relaxed = got_relaxed;
+ ctx.resolve_pair = jit_resolve_pcrel_pair;
+ ctx.resolve_user = &pair;
+ ctx.anchor_vaddr = tgt->vaddr;
+ d = link_arch_desc_for(c);
+ if (d && d->jit_reloc_relax && d->jit_reloc_relax(c, r->kind, &ctx)) return;
+
+ link_reloc_apply(c, r->kind, P_bytes, S, r->addend, P);
+}
+
KitJit* kit_jit_from_image(LinkImage* img) {
Compiler* c;
Heap* heap;
@@ -389,111 +461,9 @@ KitJit* kit_jit_from_image(LinkImage* img) {
* alias; PC-relative arithmetic uses the runtime alias address. */
metrics_scope_begin(c, "jit.apply_relocs");
for (i = 0; i < LinkRelocs_count(&img->relocs); ++i) {
- const LinkRelocApply* r = LinkRelocs_at(&img->relocs, i);
- const LinkSymbol* tgt = LinkSyms_at(&img->syms, r->target - 1);
- u64 S, P;
- u8* P_bytes;
- /* Windows TLS idiom `_tls_index` references are dropped: the access is
- * relaxed to in-image addressing, so the index load is rewritten away. */
- if (jit_tls_is_index_ref(c, tgt)) continue;
- /* Local-Exec TLS -> in-image addressing (single-threaded JIT). The per-arch
- * idiom rewrite lives behind LinkArchDesc.jit_tls_le_relax; this loop stays
- * arch-neutral, classifying ELF via RELOC_IS_TLS_LE and the COFF Windows
- * idiom via its terminal SECREL reloc(s). */
- if (jit_reloc_is_tls_le(c, r->kind, tgt->kind)) {
- const LinkArchDesc* d = link_arch_desc_for(c);
- u64 storage = (tgt->kind == SK_ABS)
- ? tgt->vaddr + (u64)r->addend
- : (u64)vaddr_to_runtime(img, segs, tgt->vaddr) +
- (u64)r->addend;
- u8* bytes = (u8*)vaddr_to_write(img, segs, r->write_vaddr);
- u64 site_pc = (u64)vaddr_to_runtime(img, segs, r->write_vaddr);
- if (!d || !d->jit_tls_le_relax || !bytes)
- compiler_panic(c, SRCLOC_NONE,
- "kit_jit: target has no TLS Local-Exec relaxation");
- d->jit_tls_le_relax(c, r->kind, bytes, storage, site_pc);
- continue;
- }
- if (r->kind == R_RV_PCREL_LO12_I || r->kind == R_RV_PCREL_LO12_S) {
- /* RISC-V PCREL_LO12: target.vaddr is the paired AUIPC site
- * (a local anchor symbol). Recompute the AUIPC's runtime
- * displacement and feed it as S to the LO12_I/S apply path so
- * the encoded low-12 bits match the AUIPC's HI20. The reloc's
- * own addend is unused per the psABI. */
- i64 disp = jit_rv_pcrel_lo12_disp(img, segs, tgt->vaddr);
- RelocKind alias =
- (r->kind == R_RV_PCREL_LO12_I) ? R_RV_LO12_I : R_RV_LO12_S;
- P_bytes = (u8*)vaddr_to_write(img, segs, r->write_vaddr);
- link_reloc_apply(c, alias, P_bytes, (u64)disp, 0,
- (u64)vaddr_to_runtime(img, segs, r->write_vaddr));
- continue;
- } else if (tgt->kind == SK_ABS) {
- /* extern resolver result OR true absolute symbol — vaddr
- * already holds the runtime address. */
- S = tgt->vaddr;
- } else {
- S = (u64)vaddr_to_runtime(img, segs, tgt->vaddr);
- }
- P = (u64)vaddr_to_runtime(img, segs, r->write_vaddr);
- P_bytes = (u8*)vaddr_to_write(img, segs, r->write_vaddr);
- /* Weak-undef target: vaddr is 0, address-of must evaluate to NULL
- * (§"weak attribute resolves to 0 at link time"). For an AArch64
- * ADRP + ADD pair against such a target, the PCREL displacement
- * exceeds ±4 GiB once the JIT places segments far from address 0,
- * which would trip link_reloc's range check. Rewrite the ADRP to
- * MOVZ Xd, #0 so Xd becomes 0 directly; the paired ADD's imm12
- * default of 0 already gives Xd += 0, so the LO12_NC reloc is a
- * no-op and we skip it. Dereferencing the resulting NULL is UB,
- * same as GCC/Clang's behavior for weak loads. */
- if (tgt->bind == SB_WEAK && tgt->kind == SK_ABS && tgt->vaddr == 0) {
- if (r->kind == R_AARCH64_ADR_PREL_PG_HI21 ||
- r->kind == R_AARCH64_ADR_PREL_PG_HI21_NC) {
- u32 instr = rd_u32_le(P_bytes);
- u32 rd = instr & 0x1fu;
- wr_u32_le(P_bytes, 0xd2800000u | rd); /* movz Xrd, #0 */
- continue;
- }
- if (r->kind == R_AARCH64_ADD_ABS_LO12_NC) {
- /* The default imm12 in the assembled ADD is 0 (the assembler
- * placeholder), so leaving the site unpatched encodes ADD Xd,
- * Xd, #0 — exactly what we want after the ADRP→MOVZ rewrite. */
- continue;
- }
- }
- /* Mach-O TLV access -> ordinary in-image load (single-threaded JIT).
- *
- * Codegen emits the Apple TLV sequence (4 insns):
- * adrp x0, desc@TLVPPAGE (PAGE21)
- * ldr x0, [x0, desc@TLVPPAGEOFF] (PAGEOFF12) -- via a __thread_ptrs slot
- * ldr x16, [x0] -- load the resolver thunk from desc[+0]
- * blr x16 -- call thunk(desc) -> &var in x0
- * The AOT writer/dyld back that with a thunk + per-thread block. With one
- * thread the in-image .tdata/.tbss IS the single instance, and desc[+16]
- * already holds the variable's in-image storage address (the normal
- * R_ABS64 against the storage symbol the loop applies). So we collapse the
- * access to a direct load, dropping the thunk, the per-thread block, and
- * descriptor patching entirely:
- * PAGE21 : adrp x0, desc (unchanged, applied below)
- * PAGEOFF12 : ldr x0,[x0,#imm] -> add x0,x0,#(desc & 0xfff) (x0 = &desc)
- * +4 : ldr x16,[x0] -> ldr x0,[x0,#16] (x0 = &var)
- * +8 : blr x16 -> nop
- * (LDR uimm12 64-bit bits[31:22]=0x3E5; ADD imm sh=0 bits[31:22]=0x244;
- * Rn[9:5]/Rt[4:0]/imm12[21:10] keep their positions.) */
- if (r->kind == R_AARCH64_TLVP_LOAD_PAGEOFF12) {
- u64 v = ((u64)S + (u64)r->addend) & 0xfffu;
- u32 instr = rd_u32_le(P_bytes);
- u8* i_thunk = P_bytes + 4u; /* ldr x16, [x0] */
- u8* i_call = P_bytes + 8u; /* blr x16 */
- instr = 0x91000000u | (instr & 0x3ffu) | ((u32)v << 10);
- wr_u32_le(P_bytes, instr);
- if (rd_u32_le(i_thunk) != 0xf9400010u || rd_u32_le(i_call) != 0xd63f0200u)
- compiler_panic(c, SRCLOC_NONE,
- "kit_jit: unexpected Mach-O TLV access sequence");
- wr_u32_le(i_thunk, 0xf9400800u); /* ldr x0, [x0, #16] -> &var */
- wr_u32_le(i_call, 0xd503201fu); /* nop */
- continue;
- }
- link_reloc_apply(c, r->kind, P_bytes, S, r->addend, P);
+ /* got_relaxed=0: the full from-image link ran GOT layout, so .got slots
+ * exist and GOT loads apply normally (S points at the slot). */
+ jit_apply_reloc(c, img, segs, LinkRelocs_at(&img->relocs, i), 0);
}
metrics_scope_end(c, "jit.apply_relocs");
@@ -734,79 +704,11 @@ static void jit_invalidate_view(KitJit* jit) {
jit->view_built = 0u;
}
+/* Incremental-append reloc apply. Delegates to the shared jit_apply_reloc
+ * with got_relaxed=1: the append path does not re-run GOT layout, so GOT loads
+ * in the appended object are relaxed to direct in-image addressing. */
static void jit_apply_one_reloc(KitJit* jit, const LinkRelocApply* r) {
- LinkImage* img = jit->image;
- const LinkSymbol* tgt = LinkSyms_at(&img->syms, r->target - 1);
- u64 S;
- u64 P;
- u8* P_bytes;
- /* Windows TLS idiom `_tls_index` references are dropped (see kit_jit_from_image). */
- if (jit_tls_is_index_ref(jit->c, tgt)) return;
- /* Local-Exec TLS -> in-image addressing (single-threaded JIT); arch idiom
- * rewrite behind LinkArchDesc.jit_tls_le_relax (see kit_jit_from_image for
- * the mirror of this in the initial reloc pass). */
- if (jit_reloc_is_tls_le(jit->c, r->kind, tgt->kind)) {
- const LinkArchDesc* d = link_arch_desc_for(jit->c);
- u64 storage = (tgt->kind == SK_ABS)
- ? tgt->vaddr + (u64)r->addend
- : (u64)vaddr_to_runtime(img, jit->segs, tgt->vaddr) +
- (u64)r->addend;
- u8* bytes = (u8*)vaddr_to_write(img, jit->segs, r->write_vaddr);
- u64 site_pc = (u64)vaddr_to_runtime(img, jit->segs, r->write_vaddr);
- if (!d || !d->jit_tls_le_relax || !bytes)
- compiler_panic(jit->c, SRCLOC_NONE,
- "kit_jit_append_obj: no TLS Local-Exec relaxation");
- d->jit_tls_le_relax(jit->c, r->kind, bytes, storage, site_pc);
- return;
- }
- if (r->kind == R_RV_PCREL_LO12_I || r->kind == R_RV_PCREL_LO12_S) {
- i64 disp = jit_rv_pcrel_lo12_disp(img, jit->segs, tgt->vaddr);
- RelocKind alias =
- (r->kind == R_RV_PCREL_LO12_I) ? R_RV_LO12_I : R_RV_LO12_S;
- P_bytes = (u8*)vaddr_to_write(img, jit->segs, r->write_vaddr);
- if (!P_bytes)
- compiler_panic(jit->c, SRCLOC_NONE,
- "kit_jit_append_obj: relocation site is unmapped");
- link_reloc_apply(jit->c, alias, P_bytes, (u64)disp, 0,
- (u64)vaddr_to_runtime(img, jit->segs, r->write_vaddr));
- return;
- }
- if (tgt->kind == SK_ABS) {
- S = tgt->vaddr;
- } else {
- S = (u64)vaddr_to_runtime(img, jit->segs, tgt->vaddr);
- }
- P = (u64)vaddr_to_runtime(img, jit->segs, r->write_vaddr);
- P_bytes = (u8*)vaddr_to_write(img, jit->segs, r->write_vaddr);
- if (!P_bytes)
- compiler_panic(jit->c, SRCLOC_NONE,
- "kit_jit_append_obj: relocation site is unmapped");
- if (tgt->bind == SB_WEAK && tgt->kind == SK_ABS && tgt->vaddr == 0) {
- if (r->kind == R_AARCH64_ADR_PREL_PG_HI21 ||
- r->kind == R_AARCH64_ADR_PREL_PG_HI21_NC) {
- u32 instr = rd_u32_le(P_bytes);
- u32 rd = instr & 0x1fu;
- wr_u32_le(P_bytes, 0xd2800000u | rd);
- return;
- }
- if (r->kind == R_AARCH64_ADD_ABS_LO12_NC) return;
- }
- if (r->kind == R_AARCH64_TLVP_LOAD_PAGEOFF12) {
- u64 v = ((u64)S + (u64)r->addend) & 0xfffu;
- u32 instr = rd_u32_le(P_bytes);
- instr = 0x91000000u | (instr & 0x3ffu) | ((u32)v << 10);
- wr_u32_le(P_bytes, instr);
- return;
- }
- if (r->kind == R_AARCH64_LD64_GOT_LO12_NC) {
- u64 v = ((u64)S + (u64)r->addend) & 0xfffu;
- u32 instr = rd_u32_le(P_bytes);
- u32 rd = instr & 0x1fu;
- u32 rn = (instr >> 5) & 0x1fu;
- wr_u32_le(P_bytes, 0x91000000u | rd | (rn << 5) | ((u32)v << 10));
- return;
- }
- link_reloc_apply(jit->c, r->kind, P_bytes, S, r->addend, P);
+ jit_apply_reloc(jit->c, jit->image, jit->segs, r, 1);
}
static void jit_append_obj_inner(KitJit* jit, ObjBuilder* ob) {
diff --git a/src/link/link_reloc_desc.h b/src/link/link_reloc_desc.h
@@ -72,4 +72,19 @@ static inline int reloc_kind_is_direct_page(const Compiler* c, RelocKind k) {
return d && (d->flags & RELOC_DIRECT_PAGE) ? 1 : 0;
}
+/* PC-relative HI20 anchor (RISC-V AUIPC for PCREL_HI20 / GOT_HI20) that a
+ * paired PCREL_LO12 resolves against. The in-process JIT scans for the anchor
+ * to recompute the low-12 displacement under its layout. */
+static inline int reloc_kind_is_pcrel_anchor(const Compiler* c, RelocKind k) {
+ const RelocDesc* d = reloc_desc(c, k);
+ return d && (d->flags & RELOC_IS_PCREL_ANCHOR) ? 1 : 0;
+}
+
+/* COFF section-relative value (R_COFF_SECREL). A TLS Local-Exec access exactly
+ * when its target is a TLS symbol; the in-process JIT relaxes those. */
+static inline int reloc_kind_is_secrel(const Compiler* c, RelocKind k) {
+ const RelocDesc* d = reloc_desc(c, k);
+ return d && (d->flags & RELOC_IS_SECREL) ? 1 : 0;
+}
+
#endif
diff --git a/src/obj/obj.h b/src/obj/obj.h
@@ -815,6 +815,12 @@ typedef enum ObjTlsModel {
* wrapper over (model == OBJ_TLS_MACHO_DESCRIPTOR). */
ObjTlsModel obj_format_tls_model(const Compiler*);
+/* In-process JIT: true when a reference to symbol `name` is dropped because the
+ * format's TLS access idiom that materializes it is relaxed to in-image
+ * addressing (COFF Windows `_tls_index`; none elsewhere). Beside
+ * obj_format_tls_model as the TLS-mechanism authority. */
+int obj_format_jit_drops_symbol_ref(const Compiler*, Sym name);
+
/* True when the active object format carries DWARF debug sections
* file-only (not mapped into a loadable segment): ELF / Mach-O yes,
* COFF no. */
diff --git a/src/obj/obj_tls.c b/src/obj/obj_tls.c
@@ -61,6 +61,20 @@ int obj_format_tls_via_descriptor(const Compiler* c) {
return obj_format_tls_model(c) == OBJ_TLS_MACHO_DESCRIPTOR;
}
+/* In-process JIT: 1 when a reference to symbol `name` is dropped because the
+ * access idiom that materializes it is relaxed to in-image addressing. The
+ * COFF Windows TEB model loads a per-module `_tls_index`; the JIT rewrites that
+ * load away, so its relocs must not be applied. None elsewhere. Sits beside
+ * obj_format_tls_model as the TLS-mechanism authority, so src/link stays free
+ * of the format-specific pseudo-symbol name. */
+int obj_format_jit_drops_symbol_ref(const Compiler* c, Sym name) {
+ Slice nm;
+ if (!c || name == 0) return 0;
+ if (obj_format_tls_model(c) != OBJ_TLS_WINDOWS_TEB) return 0;
+ nm = pool_slice(c->global, name);
+ return nm.len == 10u && memcmp(nm.s, "_tls_index", 10u) == 0;
+}
+
static void define_tls_elf(ObjBuilder* ob, Compiler* c, ObjSymId sym,
const u8* data, u32 size, int has_nonzero_init,
u32 align, const ObjTlsReloc* relocs, u32 nrelocs) {
diff --git a/src/obj/reloc.c b/src/obj/reloc.c
@@ -35,7 +35,8 @@ static const RelocDescRow neutral_rows[] = {
{R_REL32, {4, 0}}, {R_REL64, {8, 0}},
{R_PC32, {4, 0}}, {R_PC64, {8, 0}},
{R_GOT32, {4, 0}}, {R_PLT32, {4, 0}},
- {R_COFF_SECREL, {4, 0}}, {R_COFF_SECTION, {2, 0}},
+ {R_COFF_SECREL, {4, RELOC_IS_SECREL}},
+ {R_COFF_SECTION, {2, 0}},
{R_COFF_ADDR32NB, {4, 0}},
};
diff --git a/src/obj/reloc.h b/src/obj/reloc.h
@@ -23,14 +23,21 @@ typedef enum RelocDescFlag {
RELOC_DIRECT_PAGE = 1u << 4, /* Mach-O ADRP-direct (non-GOT) page / pageoff */
RELOC_MARKER = 1u << 5, /* no bytes patched (RELAX / TPREL_ADD) */
RELOC_WIDTH_DYN = 1u << 6, /* width read from the bytes at apply (ULEB128) */
- RELOC_IS_TLS_LE = 1u << 7, /* ELF Local-Exec TLS access (tp-relative idiom);
- * the in-process JIT relaxes it to in-image
- * addressing via LinkArchDesc.jit_tls_le_relax */
+ RELOC_IS_TLS_LE = 1u << 7, /* ELF Local-Exec TLS access (tp-relative idiom),
+ * plus the COFF Windows SECREL12A TLS pair; the
+ * in-process JIT relaxes it to in-image
+ * addressing via LinkArchDesc.jit_tls_le_relax */
+ RELOC_IS_PCREL_ANCHOR = 1u << 8, /* PC-relative HI20 anchor (RISC-V AUIPC) a
+ * paired PCREL_LO12 reloc resolves against;
+ * the JIT recomputes its displacement */
+ RELOC_IS_SECREL = 1u << 9, /* COFF section-relative value (R_COFF_SECREL); a
+ * TLS Local-Exec access when its target is a TLS
+ * symbol, which the in-process JIT relaxes */
} RelocDescFlag;
typedef struct RelocDesc {
- u8 width; /* patched-field width in bytes; nominal when RELOC_WIDTH_DYN */
- u8 flags; /* RelocDescFlag bitset */
+ u8 width; /* patched-field width in bytes; nominal when RELOC_WIDTH_DYN */
+ u16 flags; /* RelocDescFlag bitset */
} RelocDesc;
/* One row of a per-arch / neutral descriptor table. `kind` holds a
