kit

reloc.c (17470B)

---

 /* RISC-V relocation descriptors (width + classification), shared by the
  * rv64 and rv32 backends (their reloc kinds are identical).
  *
  * One row per relocation kind this backend applies.  Reached through
  * LinkArchDesc.reloc_desc (wired in link.c for both rv64 and rv32) and the
  * arch-aware reloc_desc() dispatcher.  Wire encoding + name live in
  * src/obj/<fmt>/reloc_riscv{32,64}.c.
  *
  * R_RV_CALL and the arch-neutral R_PLT32 (the kit-canonical kind
  * R_RISCV_CALL_PLT maps onto) are both branches: too-far targets route
  * through the JIT/range call-stub pass.  R_RV_CALL patches an 8-byte
  * AUIPC+JALR pair; R_PLT32 keeps its neutral 4-byte width (a gate value —
  * the apply path re-derives the real span from the kind).
  *
  * RELAX / TPREL_ADD / ALIGN are relaxation markers (RELOC_MARKER): no reloc
  * record is emitted for them — the link layout pass drops them via
  * reloc_kind_is_marker, so the width here is an unused gate value.
  * SET/SUB_ULEB128 are variable-width (RELOC_WIDTH_DYN): the apply path reads
  * the true field length from the bytes, so the width here is only nominal. */
 
 #include "obj/reloc.h"
 
 #include "core/bytes.h"
 #include "link/link_arch.h"
 
 static const RelocDescRow rv_rows[] = {
     {R_RV_HI20, {4, 0}},
     {R_RV_LO12_I, {4, 0}},
     {R_RV_LO12_S, {4, 0}},
     {R_RV_BRANCH, {4, 0}},
     {R_RV_JAL, {4, 0}},
     {R_RV_PCREL_HI20, {4, RELOC_IS_PCREL_ANCHOR}},
     {R_RV_PCREL_LO12_I, {4, 0}},
     {R_RV_PCREL_LO12_S, {4, 0}},
     {R_RV_GOT_HI20, {4, RELOC_USES_GOT | RELOC_IS_PCREL_ANCHOR}},
     {R_RV_TLS_GOT_HI20, {4, RELOC_IS_TLS_GOT}},
     {R_RV_TLS_GD_HI20, {4, RELOC_IS_TLS_LE}},
     {R_RV_TPREL_HI20, {4, RELOC_IS_TLS_LE}},
     {R_RV_TPREL_LO12_I, {4, RELOC_IS_TLS_LE}},
     {R_RV_TPREL_LO12_S, {4, RELOC_IS_TLS_LE}},
     {R_RV_CALL, {8, RELOC_IS_BRANCH}},
     {R_PLT32, {4, RELOC_IS_BRANCH}},
     {R_RV_RVC_BRANCH, {2, 0}},
     {R_RV_RVC_JUMP, {2, 0}},
     {R_RV_RELAX, {4, RELOC_MARKER}},
     {R_RV_TPREL_ADD, {4, RELOC_MARKER}},
     {R_RV_ALIGN, {4, RELOC_MARKER}},
 };
 
 const RelocDesc* rv_reloc_desc(RelocKind k) {
   return reloc_desc_row_find(rv_rows, (u32)(sizeof rv_rows / sizeof rv_rows[0]),
                              k);
 }
 
 #define RV_X_RA 1u
 #define RV_X_TP 4u
 #define RV_X_A0 10u
 
 typedef struct RvTlsgdSeq {
   u32 tmp_reg;
   u32 call_off;
 } RvTlsgdSeq;
 
 static u32 rv_rd(u32 instr) { return (instr >> 7) & 0x1fu; }
 
 static u32 rv_rs1(u32 instr) { return (instr >> 15) & 0x1fu; }
 
 static u32 rv_c_rd(u16 instr) { return ((u32)instr >> 7) & 0x1fu; }
 
 static u32 rv_c_rs2(u16 instr) { return ((u32)instr >> 2) & 0x1fu; }
 
 static int rv_is_32b_at(const u8* p) { return (rd_u16_le(p) & 0x3u) == 0x3u; }
 
 static int rv_is_addi_same_rd_rs1(u32 instr, u32 reg) {
   return (instr & 0x707fu) == 0x13u && rv_rd(instr) == reg &&
          rv_rs1(instr) == reg;
 }
 
 static int rv_is_addi_mv(u32 instr, u32* rd, u32* rs) {
   if ((instr & 0xfff0707fu) != 0x13u) return 0;
   *rd = rv_rd(instr);
   *rs = rv_rs1(instr);
   return *rd != 0 && *rs != 0;
 }
 
 static int rv_is_c_mv(u16 instr, u32* rd, u32* rs) {
   if ((instr & 0xf003u) != 0x8002u) return 0;
   *rd = rv_c_rd(instr);
   *rs = rv_c_rs2(instr);
   return *rd != 0 && *rs != 0;
 }
 
 static int rv_tlsgd_call_pair_at(const u8* p) {
   return rd_u32_le(p) == (0x00000017u | (RV_X_RA << 7)) &&
          rd_u32_le(p + 4) == 0x000080e7u;
 }
 
 static int rv_find_tlsgd_seq(const u8* p, RvTlsgdSeq* seq) {
   enum { RV_TLSGD_SCAN_LIMIT = 24u };
   u32 auipc = rd_u32_le(p);
   u32 addi = rd_u32_le(p + 4);
   u32 tmp;
   u32 off;
   int a0_ready;
 
   if ((auipc & 0x7fu) != 0x17u) return 0;
   tmp = rv_rd(auipc);
   if (tmp == 0) return 0;
   if (!rv_is_addi_same_rd_rs1(addi, tmp)) return 0;
 
   a0_ready = tmp == RV_X_A0;
   for (off = 8u; off <= RV_TLSGD_SCAN_LIMIT;) {
     if (rv_is_32b_at(p + off) && a0_ready && rv_tlsgd_call_pair_at(p + off)) {
       seq->tmp_reg = tmp;
       seq->call_off = off;
       return 1;
     }
 
     if (rv_is_32b_at(p + off)) {
       u32 rd = 0, rs = 0;
       if (!rv_is_addi_mv(rd_u32_le(p + off), &rd, &rs)) return 0;
       if (rd == RV_X_A0 && rs == tmp) {
         a0_ready = 1;
       } else if (rd == tmp || rd == RV_X_A0 || a0_ready) {
         return 0;
       }
       off += 4u;
       continue;
     } else {
       u32 rd = 0, rs = 0;
       if (!rv_is_c_mv(rd_u16_le(p + off), &rd, &rs)) return 0;
       if (rd == RV_X_A0 && rs == tmp) {
         a0_ready = 1;
       } else if (rd == tmp || rd == RV_X_A0 || a0_ready) {
         return 0;
       }
       off += 2u;
       continue;
     }
   }
   return 0;
 }
 
 static u32 rv_encode_lui(u32 rd, u32 hi20) {
   return 0x00000037u | (rd << 7) | (hi20 << 12);
 }
 
 static u32 rv_encode_addi(u32 rd, u32 rs1, u32 lo12) {
   return 0x00000013u | (rd << 7) | (rs1 << 15) | (lo12 << 20);
 }
 
 /* RISC-V instruction-immediate byte encoders (WS-C), shared by rv64 and rv32.
  * Moved verbatim from the format-neutral byte-patcher; reached via
  * LinkArchDesc.reloc_apply_insn.  Encoding references: "RISC-V ELF psABI" §3
  * and "The RISC-V Instruction Set Manual, Volume I" Ch.19.  The +0x800 bias on
  * HI20 / CALL compensates the sign-extension of the paired 12-bit immediate.
  * The data-word ADD/SUB/SET arms and the ULEB128 codec are arch-neutral byte
  * writes and stay in the obj-core neutral path.  Returns 1 if it owns `k`. */
 int rv_reloc_apply_insn(Compiler* c, RelocKind k, u8* P_bytes, u64 S, i64 A,
                         u64 P) {
   switch (k) {
     case R_RV_HI20:
     case R_RV_TPREL_HI20: {
       /* U-type (LUI/AUIPC) imm[31:12] = high 20 bits of (S + A + 0x800).
        * The 0x800 bias compensates the sign-extension of the paired
        * 12-bit ADDI/load/store immediate, so HI20 + signext12(LO12)
        * reconstructs the full value. */
       i64 v = (i64)S + A;
       u32 hi20 = (u32)(((u64)(v + 0x800)) >> 12) & 0xfffffu;
       u32 instr = rd_u32_le(P_bytes);
       instr = (instr & 0x00000fffu) | (hi20 << 12);
       wr_u32_le(P_bytes, instr);
       return 1;
     }
     case R_RV_PCREL_HI20:
     case R_RV_GOT_HI20:
     case R_RV_TLS_GOT_HI20: {
       /* AUIPC pc-relative HI20: same encoding as HI20 but the
        * displacement is (S + A) - P. The paired PCREL_LO12 reloc at
        * the ADDI/load below recovers the low 12 bits of the same
        * displacement via a lookup keyed on this AUIPC's site vaddr.
        * GOT_HI20 collapses to PCREL_HI20 in static-link with no
        * indirection: the symbol resolves to its own address. */
       i64 disp = (i64)S + A - (i64)P;
       u32 hi20 = (u32)(((u64)(disp + 0x800)) >> 12) & 0xfffffu;
       u32 instr = rd_u32_le(P_bytes);
       instr = (instr & 0x00000fffu) | (hi20 << 12);
       wr_u32_le(P_bytes, instr);
       return 1;
     }
     case R_RV_TLS_GD_HI20: {
       /* Relax a general-dynamic block:
        *   auipc t, %tls_gd_pcrel_hi(sym)
        *   addi  t, t, %pcrel_lo(.Lpcrel_hi)
        *   ...
        *   mv    a0, t        (if t is not already a0)
        *   call  __tls_get_addr
        *
        * to local-exec address materialization:
        *   lui   t, %tprel_hi(sym)
        *   addi  t, t, %tprel_lo(sym)
        *   ...
        *   mv    a0, t
        *   add   a0, a0, tp
        *   nop
        *
        * Rust/LLVM sometimes uses a non-a0 temporary and keeps live values with
        * compressed moves between the descriptor setup and the call.  Accept
        * only that narrow move-only shape; other dataflow stays a hard error.
        */
       RvTlsgdSeq seq;
       i64 v = (i64)S + A;
       u32 hi20 = (u32)(((u64)(v + 0x800)) >> 12) & 0xfffffu;
       u32 lo12 = (u32)((u64)v & 0xfffu);
       if (!rv_find_tlsgd_seq(P_bytes, &seq)) {
         compiler_panic(c, SRCLOC_NONE,
                        "link: unexpected RISC-V TLS_GD access sequence");
       }
       wr_u32_le(P_bytes, rv_encode_lui(seq.tmp_reg, hi20));
       wr_u32_le(P_bytes + 4, rv_encode_addi(seq.tmp_reg, seq.tmp_reg, lo12));
       wr_u32_le(P_bytes + seq.call_off, 0x00450533u);  /* add a0,a0,tp */
       wr_u32_le(P_bytes + seq.call_off + 4u, 0x00000013u); /* nop */
       return 1;
     }
     case R_RV_LO12_I:
     case R_RV_TPREL_LO12_I: {
       /* I-type imm[11:0] in instruction bits [31:20]. Low 12 bits of
        * (S + A); the sign-extension at execute time pairs with HI20's
        * 0x800 bias to reconstruct the full address. */
       u64 v = (u64)((i64)S + A);
       u32 lo12 = (u32)(v & 0xfffu);
       u32 instr = rd_u32_le(P_bytes);
       instr = (instr & 0x000fffffu) | (lo12 << 20);
       wr_u32_le(P_bytes, instr);
       return 1;
     }
     case R_RV_LO12_S:
     case R_RV_TPREL_LO12_S: {
       /* S-type imm[11:5] in bits [31:25], imm[4:0] in bits [11:7]. */
       u64 v = (u64)((i64)S + A);
       u32 lo12 = (u32)(v & 0xfffu);
       u32 instr = rd_u32_le(P_bytes);
       instr = (instr & 0x01fff07fu) | ((lo12 & 0xfe0u) << 20) |
               ((lo12 & 0x1fu) << 7);
       wr_u32_le(P_bytes, instr);
       return 1;
     }
     case R_RV_BRANCH: {
       /* B-type 12-bit signed displacement in 2-byte units (13-bit
        * range). imm[12] in bit 31, imm[10:5] in 30:25, imm[4:1] in
        * 11:8, imm[11] in bit 7. */
       i64 disp = (i64)S + A - (i64)P;
       u32 instr;
       u32 b;
       if (disp & 1)
         compiler_panic(c, SRCLOC_NONE,
                        "link: RV BRANCH misaligned displacement");
       if (disp < -(i64)(1 << 12) || disp >= (i64)(1 << 12))
         compiler_panic(c, SRCLOC_NONE,
                        "link: RV BRANCH out of range (need ±4KiB)");
       b = (u32)((u64)disp & 0x1ffeu) | ((u32)(((u64)disp >> 11) & 1u) << 11) |
           ((u32)(((u64)disp >> 12) & 1u) << 12);
       instr = rd_u32_le(P_bytes);
       instr &= 0x01fff07fu;
       instr |= ((b >> 12) & 1u) << 31;
       instr |= ((b >> 5) & 0x3fu) << 25;
       instr |= ((b >> 1) & 0xfu) << 8;
       instr |= ((b >> 11) & 1u) << 7;
       wr_u32_le(P_bytes, instr);
       return 1;
     }
     case R_RV_JAL: {
       /* J-type 20-bit signed displacement in 2-byte units (21-bit
        * range). imm[20] in bit 31, imm[10:1] in 30:21, imm[11] in bit
        * 20, imm[19:12] in bits 19:12. */
       i64 disp = (i64)S + A - (i64)P;
       u32 instr;
       u32 b;
       if (disp & 1)
         compiler_panic(c, SRCLOC_NONE, "link: RV JAL misaligned displacement");
       if (disp < -(i64)(1 << 20) || disp >= (i64)(1 << 20))
         compiler_panic(c, SRCLOC_NONE,
                        "link: RV JAL out of range (need ±1MiB)");
       b = (u32)((u64)disp & 0x1ffffeu) | ((u32)(((u64)disp >> 11) & 1u) << 11) |
           ((u32)(((u64)disp >> 20) & 1u) << 20);
       instr = rd_u32_le(P_bytes);
       instr &= 0x00000fffu;
       instr |= ((b >> 20) & 1u) << 31;
       instr |= ((b >> 1) & 0x3ffu) << 21;
       instr |= ((b >> 11) & 1u) << 20;
       instr |= ((b >> 12) & 0xffu) << 12;
       wr_u32_le(P_bytes, instr);
       return 1;
     }
     case R_RV_CALL:
     case R_PLT32: {
       /* AUIPC + JALR pair encoding the same 32-bit signed PC-relative
        * displacement. AUIPC at P, JALR at P+4. The 0x800 bias on the
        * AUIPC immediate compensates JALR's signed 12-bit imm so that
        * (auipc_imm << 12) + signext12(jalr_imm) == disp.
        *
        * R_PLT32 is the kit-canonical RelocKind that
        * elf_riscv64_reloc_from(R_RISCV_CALL_PLT) maps to; static-link
        * with no PLT collapses CALL_PLT to a direct CALL (no
        * indirection). */
       i64 disp = (i64)S + A - (i64)P;
       u32 hi20 = (u32)(((u64)(disp + 0x800)) >> 12) & 0xfffffu;
       u32 lo12 = (u32)((u64)disp & 0xfffu);
       u32 auipc = rd_u32_le(P_bytes);
       u32 jalr = rd_u32_le(P_bytes + 4);
       if (disp < -(i64)(1ll << 31) || disp >= (i64)(1ll << 31))
         compiler_panic(c, SRCLOC_NONE,
                        "link: RV CALL out of range (need ±2GiB)");
       auipc = (auipc & 0x00000fffu) | (hi20 << 12);
       jalr = (jalr & 0x000fffffu) | (lo12 << 20);
       wr_u32_le(P_bytes, auipc);
       wr_u32_le(P_bytes + 4, jalr);
       return 1;
     }
     case R_RV_RVC_BRANCH: {
       /* CB-type 8-bit signed displacement in 2-byte units (9-bit
        * range). c.beqz / c.bnez. Encoding (16-bit instruction):
        *   bit 12   = imm[8]
        *   bits 11:10 = imm[4:3]
        *   bits 9:7  = rs1' (untouched)
        *   bits 6:5  = imm[7:6]
        *   bits 4:3  = imm[2:1]
        *   bit 2    = imm[5] */
       i64 disp = (i64)S + A - (i64)P;
       u16 instr = (u16)(P_bytes[0] | ((u16)P_bytes[1] << 8));
       u32 b;
       if (disp & 1)
         compiler_panic(c, SRCLOC_NONE,
                        "link: RV RVC_BRANCH misaligned displacement");
       if (disp < -(i64)(1 << 8) || disp >= (i64)(1 << 8))
         compiler_panic(c, SRCLOC_NONE,
                        "link: RV RVC_BRANCH out of range (need ±256B)");
       b = (u32)((u64)disp & 0x1feu);
       instr = (u16)(instr & 0xe383u);
       instr = (u16)(instr | (((b >> 8) & 1u) << 12));
       instr = (u16)(instr | (((b >> 3) & 3u) << 10));
       instr = (u16)(instr | (((b >> 6) & 3u) << 5));
       instr = (u16)(instr | (((b >> 1) & 3u) << 3));
       instr = (u16)(instr | (((b >> 5) & 1u) << 2));
       P_bytes[0] = (u8)(instr & 0xffu);
       P_bytes[1] = (u8)((instr >> 8) & 0xffu);
       return 1;
     }
     case R_RV_RVC_JUMP: {
       /* CJ-type 11-bit signed displacement in 2-byte units (12-bit
        * range). c.j / c.jal. Encoding bits in the 16-bit instruction:
        *   12=imm[11], 11=imm[4], 10:9=imm[9:8], 8=imm[10],
        *   7=imm[6], 6=imm[7], 5:3=imm[3:1], 2=imm[5]. */
       i64 disp = (i64)S + A - (i64)P;
       u16 instr = (u16)(P_bytes[0] | ((u16)P_bytes[1] << 8));
       u32 b;
       if (disp & 1)
         compiler_panic(c, SRCLOC_NONE,
                        "link: RV RVC_JUMP misaligned displacement");
       if (disp < -(i64)(1 << 11) || disp >= (i64)(1 << 11))
         compiler_panic(c, SRCLOC_NONE,
                        "link: RV RVC_JUMP out of range (need ±2KiB)");
       b = (u32)((u64)disp & 0xffeu);
       instr = (u16)(instr & 0xe003u);
       instr = (u16)(instr | (((b >> 11) & 1u) << 12));
       instr = (u16)(instr | (((b >> 4) & 1u) << 11));
       instr = (u16)(instr | (((b >> 8) & 3u) << 9));
       instr = (u16)(instr | (((b >> 10) & 1u) << 8));
       instr = (u16)(instr | (((b >> 6) & 1u) << 7));
       instr = (u16)(instr | (((b >> 7) & 1u) << 6));
       instr = (u16)(instr | (((b >> 1) & 7u) << 3));
       instr = (u16)(instr | (((b >> 5) & 1u) << 2));
       P_bytes[0] = (u8)(instr & 0xffu);
       P_bytes[1] = (u8)((instr >> 8) & 0xffu);
       return 1;
     }
     case R_RV_RELAX:
     case R_RV_TPREL_ADD:
       /* Marker relocs only — RELAX permits the prior reloc to be
        * compressed, TPREL_ADD annotates a TLS thread-pointer ADD that
        * the linker may fold during relaxation. We don't relax, so
        * both are no-ops. */
       return 1;
     default:
       return 0;
   }
 }
 
 /* In-process JIT TLS Local-Exec relaxation (LinkArchDesc.jit_tls_le_relax).
  * Codegen emits, per access:
  *   lui  t, %tprel_hi(var)      R_RV_TPREL_HI20   <- `site`
  *   add  t, tp, t              (no reloc)
  *   addi rd, t, %tprel_lo(var)  R_RV_TPREL_LO12_I
  * Single-threaded JIT: address the in-image storage PC-relative, dropping the
  * tp add:
  *   auipc t, %pcrel_hi(&var) ; nop ; addi rd, t, %pcrel_lo(&var)
  * The HI20 reloc drives the whole rewrite; the LO12 half is then a no-op. The
  * riscv `tp` ABI register is x4. */
 void rv_jit_tls_le_relax(Compiler* c, RelocKind k, u8* site, u64 storage,
                          u64 site_pc) {
   u8* add;
   u8* addi;
   u32 rd_tmp, add_w, addi_w, hi20;
   i64 disp;
   i32 lo12;
   if (k == R_RV_TPREL_LO12_I || k == R_RV_TPREL_LO12_S)
     return; /* handled with HI20 */
   if (k != R_RV_TPREL_HI20)
     compiler_panic(c, SRCLOC_NONE, "riscv jit tls: unexpected reloc kind %u",
                    (unsigned)k);
   add = site + 4;  /* add t, tp, t -> nop */
   addi = site + 8; /* addi rd, t, %lo -> addi rd, t, %pcrel_lo */
   if ((rd_u32_le(site) & 0x7fu) != 0x37u) /* lui */
     compiler_panic(c, SRCLOC_NONE, "riscv jit tls: unexpected access sequence");
   add_w = rd_u32_le(add);
   if ((add_w & 0x7fu) != 0x33u ||
       ((add_w >> 15) & 0x1fu) != 4u) /* add ?,tp,? */
     compiler_panic(c, SRCLOC_NONE, "riscv jit tls: unexpected access sequence");
   rd_tmp = (rd_u32_le(site) >> 7) & 0x1fu;
   disp = (i64)storage - (i64)site_pc;
   lo12 = (i32)((u32)disp & 0xfffu);
   if (lo12 & 0x800) lo12 -= 0x1000; /* sign-extend 12-bit */
   hi20 = (u32)(((disp - (i64)lo12) >> 12) & 0xfffffu);
   wr_u32_le(site, 0x00000017u | (rd_tmp << 7) | (hi20 << 12)); /* auipc */
   wr_u32_le(add, 0x00000013u);                                 /* nop */
   addi_w = rd_u32_le(addi);
   wr_u32_le(addi, (addi_w & 0x000fffffu) | (((u32)lo12 & 0xfffu) << 20));
 }
 
 /* In-process JIT relaxation of RISC-V indirection idioms (LinkArchDesc
  * .jit_reloc_relax).  Today only PCREL_LO12: it has no S of its own — its
  * low-12 bits must match the paired AUIPC's PC-relative displacement, which
  * the JIT recomputes against its layout via ctx->resolve_pair.  Feed that
  * displacement to the LO12_I/S encoder (same instruction encoding; the addend
  * is unused per the psABI).  Returns 1 if it owned `k`, 0 otherwise. */
 int rv_jit_reloc_relax(Compiler* c, RelocKind k, const JitRelaxCtx* ctx) {
   if (k == R_RV_PCREL_LO12_I || k == R_RV_PCREL_LO12_S) {
     i64 disp = ctx->resolve_pair(ctx->resolve_user, ctx->anchor_vaddr);
     RelocKind alias = (k == R_RV_PCREL_LO12_I) ? R_RV_LO12_I : R_RV_LO12_S;
     rv_reloc_apply_insn(c, alias, ctx->site, (u64)disp, 0, ctx->site_pc);
     return 1;
   }
   return 0;
 }