kit

native_direct_target.c (114087B)

---

 #include "cg/native_direct_target.h"
 
 /* NativeDirectTarget is intentionally single-pass: semantic CG calls are
  * lowered immediately to NativeTarget operations, MCEmitter owns label fixups,
  * and function-end calls note_frame_state()/patch_apply() let the native
  * backend patch deferred frame/prologue details after max outgoing space is
  * known. Direct lowering currently forwards final frame state but does not
  * author generic NativePatch records itself.
  *
  * Remaining direct/backend cutover work: stack arguments, tail/musttail,
  * varargs, typed inline-asm register/memory bindings and outputs,
  * label-address data and computed gotos, records/sret/large aggregates, FP and
  * rounding conversions, fuller scalar intrinsics, and production-grade atomic
  * RMW/CAS lowering. */
 
 #include <string.h>
 
 #include "abi/abi.h"
 #include "cg/native_asm.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/pool.h"
 #include "core/slice.h"
 
 #define NATIVE_DIRECT_MAGIC 0x4e445447u
 
 static NativeDirectTarget* nd_of(CgTarget* t) { return (NativeDirectTarget*)t; }
 
 static _Noreturn void nd_panic(NativeDirectTarget* d, const char* what) {
   compiler_panic(d->base.c, d->loc, "native direct target: %s", what);
 }
 
 static void* nd_arena(NativeDirectTarget* d, size_t size, size_t align) {
   void* p = arena_zalloc(d->base.c->tu, size, align);
   if (!p) nd_panic(d, "out of memory");
   return p;
 }
 
 /* Pick a transient NativeLoc array for one op: the on-struct buffer when N
  * fits, else a one-shot arena allocation. N == 0 yields NULL. */
 static NativeLoc* nd_loc_buf(NativeDirectTarget* d, NativeLoc* buf, u32 cap,
                              u32 n) {
   if (!n) return NULL;
   if (n <= cap) return buf;
   return nd_arena(d, sizeof(NativeLoc) * n, _Alignof(NativeLoc));
 }
 
 void native_direct_project_tail_call_desc(NativeDirectTarget* d,
                                           const CGCallDesc* call,
                                           NativeCallDesc* nd) {
   NativeLoc* args = NULL;
   NativeLoc* results = NULL;
   u32 nresults = call->result != CG_LOCAL_NONE ? 1u : 0u;
   u32 i;
   memset(nd, 0, sizeof *nd);
   if (call->nargs) args = arena_zarray(d->base.c->tu, NativeLoc, call->nargs);
   if (nresults) results = arena_zarray(d->base.c->tu, NativeLoc, nresults);
   for (i = 0; i < call->nargs; ++i) {
     args[i].kind = NATIVE_LOC_FRAME;
     args[i].type = d->locals[call->args[i] - 1u].type;
     args[i].cls = d->locals[call->args[i] - 1u].cls;
     args[i].v.frame = d->locals[call->args[i] - 1u].home;
   }
   if (nresults) {
     results[0].kind = NATIVE_LOC_FRAME;
     results[0].type = d->locals[call->result - 1u].type;
     results[0].cls = d->locals[call->result - 1u].cls;
     results[0].v.frame = d->locals[call->result - 1u].home;
   }
   nd->fn_type = call->fn_type;
   nd->args = args;
   nd->results = results;
   nd->nargs = call->nargs;
   nd->nresults = nresults;
 }
 
 static void nd_grow_locals(NativeDirectTarget* d, u32 want) {
   NativeDirectLocal* next;
   u32 cap;
   if (d->locals_cap >= want) return;
   cap = d->locals_cap ? d->locals_cap : 32u;
   while (cap < want) cap *= 2u;
   next = nd_arena(d, sizeof(*next) * cap, _Alignof(NativeDirectLocal));
   if (d->locals) memcpy(next, d->locals, sizeof(*next) * d->nlocals);
   d->locals = next;
   d->locals_cap = cap;
 }
 
 static void nd_grow_transient(NativeDirectTarget* d, u32 want) {
   CGLocal* next;
   u32 cap;
   if (d->transient_cap >= want) return;
   cap = d->transient_cap ? d->transient_cap : 32u;
   while (cap < want) cap *= 2u;
   next = nd_arena(d, sizeof(*next) * cap, _Alignof(CGLocal));
   if (d->transient_locals)
     memcpy(next, d->transient_locals, sizeof(*next) * d->ntransient);
   d->transient_locals = next;
   d->transient_cap = cap;
 }
 
 static void nd_grow_labels(NativeDirectTarget* d, u32 want) {
   MCLabel* next;
   u32 cap;
   if (d->labels_cap >= want) return;
   cap = d->labels_cap ? d->labels_cap : 32u;
   while (cap < want) cap *= 2u;
   next = nd_arena(d, sizeof(*next) * cap, _Alignof(MCLabel));
   if (d->labels) memcpy(next, d->labels, sizeof(*next) * d->labels_cap);
   d->labels = next;
   d->labels_cap = cap;
 }
 
 static void nd_grow_scopes(NativeDirectTarget* d, u32 want) {
   NativeDirectScope* next;
   u32 cap;
   if (d->scopes_cap >= want) return;
   cap = d->scopes_cap ? d->scopes_cap : 16u;
   while (cap < want) cap *= 2u;
   next = nd_arena(d, sizeof(*next) * cap, _Alignof(NativeDirectScope));
   if (d->scopes) memcpy(next, d->scopes, sizeof(*next) * d->nscopes);
   d->scopes = next;
   d->scopes_cap = cap;
 }
 
 static NativeDirectLocal* nd_local(NativeDirectTarget* d, CGLocal local) {
   if (local == CG_LOCAL_NONE || local > d->nlocals)
     nd_panic(d, "bad semantic local");
   return &d->locals[local - 1u];
 }
 
 static NativeAllocClass nd_class_for_type(NativeDirectTarget* d,
                                           KitCgTypeId type) {
   if (d->native && d->native->class_for_type)
     return d->native->class_for_type(d->native, type);
   return NATIVE_REG_INT;
 }
 
 static const NativeAllocClassInfo* nd_class_info(NativeDirectTarget* d,
                                                  NativeAllocClass cls) {
   const NativeAllocClassInfo* ci =
       (u32)cls < NATIVE_REG_CLASS_COUNT ? d->class_info[cls] : NULL;
   if (!ci) nd_panic(d, "target has no requested register class");
   return ci;
 }
 
 /* Register-location constructor is shared as native_loc_reg in
  * native_target.h (arg order: type, cls, reg). nd_loc_reg wraps it to stamp
  * the cached scalar-width descriptor (szinfo) once, here at the single
  * register-loc choke point, so the per-arch binop/move/convert/cmp hot path
  * reads loc.szinfo instead of re-crossing the type bridge per operand. */
 static inline NativeLoc nd_loc_reg(NativeDirectTarget* d, KitCgTypeId type,
                                    NativeAllocClass cls, Reg reg) {
   NativeLoc loc = native_loc_reg(type, cls, reg);
   if (d->native) native_loc_stamp_size(d->native, &loc);
   return loc;
 }
 
 static void nd_flush_local(NativeDirectTarget* d, CGLocal local);
 static Reg nd_cache_reg_for(NativeDirectTarget* d, CGLocal local,
                             KitCgTypeId access_type);
 static Reg nd_pick_cache_victim(NativeDirectTarget* d, NativeAllocClass cls);
 
 static u32 nd_callee_saved_mask(NativeDirectTarget* d, NativeAllocClass cls) {
   return native_target_callee_saved_mask(d->native, cls);
 }
 
 static u32 nd_caller_saved_mask(NativeDirectTarget* d, NativeAllocClass cls) {
   return native_target_caller_saved_mask(d->native, cls);
 }
 
 static void nd_note_reg_used(NativeDirectTarget* d, NativeAllocClass cls,
                              Reg reg) {
   if ((u32)cls >= NATIVE_REG_CLASS_COUNT || reg >= NATIVE_MAX_HARD_REGS) return;
   if (nd_callee_saved_mask(d, cls) & (1u << reg))
     d->callee_saved_used[cls] |= 1u << reg;
 }
 
 /* Mask of registers the single-pass path may hand out as scratch in `cls` — its
  * dedicated scratch list plus the allocable pool. A multi-register (wide) value's
  * whole register run must lie within this set so each lane is a real, writable
  * GPR the NDT owns: this is what keeps a wide value off a base whose high lane is
  * a reserved register (the arm32 trap, where the lone scratch ip(r12) would pair
  * its high lane onto r13/sp). On 64-bit targets every value is one register, so
  * the run is the base alone and this never narrows the choice. */
 static u32 nd_acquirable_mask(const NativeAllocClassInfo* ci) {
   u32 m = 0u, i;
   for (i = 0; i < ci->nscratch; ++i)
     if (ci->scratch[i] < 32u) m |= 1u << ci->scratch[i];
   for (i = 0; i < ci->ndt_allocable_count; ++i)
     if (ci->ndt_allocable[i] < 32u) m |= 1u << ci->ndt_allocable[i];
   return m;
 }
 
 /* A `span`-register value placed at base `r` is legal iff the high lanes
  * r+1..r+span-1 are all acquirable registers (never sp/pc/fp/lr). The base lane
  * itself is whatever the caller's scratch/allocable scan already vetted (it may
  * legitimately be a reserved-from-allocation scratch like ip). */
 static int nd_span_high_acquirable(u32 acq, Reg r, u32 span) {
   u32 k;
   for (k = 1u; k < span; ++k)
     if ((u32)(r + k) >= 32u || !(acq & (1u << (r + k)))) return 0;
   return 1;
 }
 
 /* Acquire a run of `span` consecutive registers (span==1 is the common case)
  * whose base is returned; the whole run is pinned. span>1 only arises for a wide
  * (8-byte) integer value on a 32-bit target, which occupies a register pair. */
 static Reg nd_scratch_acquire_span(NativeDirectTarget* d, NativeAllocClass cls,
                                    u32 span) {
   const NativeAllocClassInfo* ci = nd_class_info(d, cls);
   const Reg* regs = ci->scratch;
   u32 nregs = ci->nscratch;
   u32 acq = span > 1u ? nd_acquirable_mask(ci) : 0u;
   if (span < 1u) span = 1u;
   /* Prefer a register (run) that is neither pinned (scratch_used) nor caching a
    * live local (reg_owner). */
   for (u32 pass = 0; pass < 2u; ++pass) {
     for (u32 i = 0; i < nregs; ++i) {
       Reg r = regs[i];
       u32 k;
       int run_ok = 1;
       if (r >= 32u) continue;
       /* On targets whose register model declares ndt_caller_saved_only, the
        * single-pass (-O0) path never takes a callee-saved register as scratch
        * (the `allocable` pass would otherwise offer them under pressure);
        * pressure is absorbed by evicting a cached local instead. This keeps
        * callee_saved_used empty so the backend can reserve only a tiny
        * deferred-prologue region (see nd_func_end). Targets without a rich
        * enough caller-saved pool (x64, riscv today) leave it clear and keep the
        * historical callee-saved-as-scratch fallback. */
       if (d->reg_info && d->reg_info->ndt_caller_saved_only &&
           (nd_callee_saved_mask(d, cls) & (1u << r)))
         continue;
       if (span > 1u && !nd_span_high_acquirable(acq, r, span)) continue;
       /* The whole run must be free: none pinned, none caching a live local. */
       for (k = 0; k < span; ++k) {
         Reg h = r + k;
         if ((d->scratch_used[cls] & (1u << h)) != 0 ||
             d->reg_owner[cls][h] != CG_LOCAL_NONE) {
           run_ok = 0;
           break;
         }
       }
       if (!run_ok) continue;
       for (k = 0; k < span; ++k) {
         d->scratch_used[cls] |= 1u << (r + k);
         nd_note_reg_used(d, cls, r + k);
       }
       return r;
     }
     regs = ci->ndt_allocable;
     nregs = ci->ndt_allocable_count;
   }
   /* Under pressure, evict the LRU non-pinned cached local (spilling it to its
    * home) and reuse its register as a scratch temporary. A wide run evicts every
    * cached local across the run; it never crosses a pinned register. */
   if (span <= 1u) {
     Reg r = nd_pick_cache_victim(d, cls);
     if (r != REG_NONE) {
       nd_flush_local(d, d->reg_owner[cls][r]);
       d->scratch_used[cls] |= 1u << r;
       nd_note_reg_used(d, cls, r);
       return r;
     }
   } else {
     const Reg* alloc = ci->ndt_allocable;
     for (u32 i = 0; i < ci->ndt_allocable_count; ++i) {
       Reg r = alloc[i];
       u32 k;
       int run_ok = 1;
       if (r >= 32u || !nd_span_high_acquirable(acq, r, span)) continue;
       for (k = 0; k < span; ++k)
         if ((d->scratch_used[cls] & (1u << (r + k))) != 0) { run_ok = 0; break; }
       if (!run_ok) continue;
       for (k = 0; k < span; ++k)
         if (d->reg_owner[cls][r + k] != CG_LOCAL_NONE)
           nd_flush_local(d, d->reg_owner[cls][r + k]);
       for (k = 0; k < span; ++k) {
         d->scratch_used[cls] |= 1u << (r + k);
         nd_note_reg_used(d, cls, r + k);
       }
       return r;
     }
   }
   nd_panic(d, "out of scratch registers");
 }
 
 static Reg nd_scratch_acquire(NativeDirectTarget* d, NativeAllocClass cls) {
   return nd_scratch_acquire_span(d, cls, 1u);
 }
 
 /* Number of consecutive registers a value of `type` occupies in class `cls`. On
  * a 32-bit target an 8-byte integer (i64 / soft-double) is a register pair; a
  * value that fits the register width — and any value in a non-integer class,
  * which holds wide data in a single vector register — is one register. */
 static u32 nd_reg_span(NativeDirectTarget* d, NativeAllocClass cls,
                        KitCgTypeId type) {
   u32 ptr;
   u64 size;
   if (cls != NATIVE_REG_INT || type == 0) return 1u;
   ptr = (u32)d->base.c->target.ptr_size;
   size = cg_type_size(d->base.c, type);
   return (ptr && size > (u64)ptr) ? (u32)((size + ptr - 1u) / ptr) : 1u;
 }
 
 static void nd_scratch_release(NativeDirectTarget* d, NativeAllocClass cls,
                                Reg reg) {
   if (reg < 32u) d->scratch_used[cls] &= ~(1u << reg);
 }
 
 static NativeFrameSlot nd_alloc_frame_slot(NativeDirectTarget* d,
                                            const NativeFrameSlotDesc* desc) {
   NativeFrameSlot slot = NATIVE_FRAME_SLOT_NONE;
   if (d->native && d->native->frame_slot)
     slot = d->native->frame_slot(d->native, desc);
   else
     nd_panic(d, "target does not allocate frame slots");
   if (slot == NATIVE_FRAME_SLOT_NONE)
     nd_panic(d, "frame slot allocation failed");
   return slot;
 }
 
 /* Ensure LOCAL has a frame home, allocating one lazily on first demand.
  * Transient temps are created without a home (nd_alloc_local) so a value that
  * stays register-resident never reserves frame space; the slot is minted here
  * the first time the value must live in memory (a spill, or a reader that needs
  * the home address). The TRANSIENT flag wires the lazy slot into the same
  * free-list reuse as eager transient homes (native_frame_slot_alloc pops a free
  * bin first; nd_reclaim_temps recycles it at the next statement boundary).
  * Non-transient locals and address-taken/memory-required temps are homed
  * eagerly, so for them this is a pure pass-through. */
 static NativeFrameSlot nd_home(NativeDirectTarget* d, CGLocal local) {
   NativeDirectLocal* l = nd_local(d, local);
   if (l->home == NATIVE_FRAME_SLOT_NONE) {
     NativeFrameSlotDesc fsd = {.type = l->type,
                                .size = l->size,
                                .align = l->align,
                                .kind = NATIVE_FRAME_SLOT_LOCAL,
                                .flags = NATIVE_FRAME_SLOT_TRANSIENT};
     l->home = nd_alloc_frame_slot(d, &fsd);
   }
   return l->home;
 }
 
 /* Ensure the home(s) of every local an operand references are allocated before
  * handing the operand to arch code that reads `home` directly (va_arg / asm
  * helpers). A no-op for IMM/GLOBAL operands. */
 static void nd_home_operand(NativeDirectTarget* d, Operand op) {
   switch ((OpKind)op.kind) {
     case OPK_LOCAL:
       nd_home(d, op.v.local);
       break;
     case OPK_INDIRECT:
       nd_home(d, op.v.ind.base);
       if (op.v.ind.index != CG_LOCAL_NONE) nd_home(d, op.v.ind.index);
       break;
     default:
       break;
   }
 }
 
 static NativeFrameSlotDesc nd_slot_desc_local(const CGLocalDesc* in) {
   NativeFrameSlotDesc out = {.type = in->type,
                              .name = in->name,
                              .loc = in->loc,
                              .size = in->size,
                              .align = in->align,
                              .kind = NATIVE_FRAME_SLOT_LOCAL};
   if (in->flags & CG_LOCAL_ADDR_TAKEN)
     out.flags |= NATIVE_FRAME_SLOT_ADDR_TAKEN;
   if (in->flags & CG_LOCAL_MEMORY_REQUIRED)
     out.flags |= NATIVE_FRAME_SLOT_MEMORY_REQUIRED;
   if (in->flags & CG_LOCAL_TRANSIENT)
     out.flags |= NATIVE_FRAME_SLOT_TRANSIENT;
   return out;
 }
 
 static NativeFrameSlotDesc nd_slot_desc_param(const CGParamDesc* in) {
   NativeFrameSlotDesc out = {.type = in->type,
                              .name = in->name,
                              .loc = in->loc,
                              .size = in->size,
                              .align = in->align,
                              .kind = NATIVE_FRAME_SLOT_PARAM};
   if (in->flags & CG_LOCAL_ADDR_TAKEN)
     out.flags |= NATIVE_FRAME_SLOT_ADDR_TAKEN;
   if (in->flags & CG_LOCAL_MEMORY_REQUIRED)
     out.flags |= NATIVE_FRAME_SLOT_MEMORY_REQUIRED;
   return out;
 }
 
 static CGLocal nd_alloc_local(NativeDirectTarget* d, const CGLocalDesc* desc) {
   NativeDirectLocal* l;
   NativeFrameSlotDesc fsd;
   CGLocal id;
   nd_grow_locals(d, d->nlocals + 1u);
   id = d->nlocals + 1u;
   l = &d->locals[d->nlocals++];
   memset(l, 0, sizeof *l);
   l->type = desc->type;
   l->size = desc->size;
   l->align = desc->align;
   l->flags = desc->flags;
   l->reg = REG_NONE;
   l->address_taken = (desc->flags & CG_LOCAL_ADDR_TAKEN) != 0;
   l->memory_required = (desc->flags & CG_LOCAL_MEMORY_REQUIRED) != 0;
   l->transient = (desc->flags & CG_LOCAL_TRANSIENT) != 0;
   l->cls = (u8)nd_class_for_type(d, desc->type);
   if (l->transient && !l->address_taken && !l->memory_required) {
     /* Lazy home: a plain transient temp reserves no frame slot until it must
      * actually live in memory (nd_home, on first spill / home read). Address-
      * taken and memory-required (wide8) temps need a stable home, so they stay
      * eager. */
     l->home = NATIVE_FRAME_SLOT_NONE;
   } else {
     fsd = nd_slot_desc_local(desc);
     l->home = nd_alloc_frame_slot(d, &fsd);
   }
   if (l->transient) {
     nd_grow_transient(d, d->ntransient + 1u);
     d->transient_locals[d->ntransient++] = id;
   }
   return id;
 }
 
 static MCLabel nd_mc_label(NativeDirectTarget* d, Label label) {
   if (label == LABEL_NONE || label > d->nlabels || !d->labels[label])
     nd_panic(d, "bad label");
   return d->labels[label];
 }
 
 static Label nd_label_new_raw(NativeDirectTarget* d) {
   Label id;
   if (!d->native || !d->native->label_new)
     nd_panic(d, "target does not allocate labels");
   id = d->nlabels + 1u;
   nd_grow_labels(d, id + 1u);
   d->labels[id] = d->native->label_new(d->native);
   d->nlabels = id;
   return id;
 }
 
 /* Designated compound literals: value-identical to the former memset + field
  * stores (unnamed fields zero-fill), but the compiler emits only the live
  * stores rather than an out-of-line memset of the ~64-byte descriptor on this
  * per-operand path. */
 static NativeLoc nd_loc_frame(NativeDirectTarget* d, CGLocal local,
                               KitCgTypeId type) {
   NativeDirectLocal* l = nd_local(d, local);
   return (NativeLoc){.kind = NATIVE_LOC_FRAME,
                      .cls = l->cls,
                      .type = type ? type : l->type,
                      .v.frame = nd_home(d, local)};
 }
 
 static NativeLoc nd_loc_imm(i64 imm, KitCgTypeId type) {
   return (NativeLoc){.kind = NATIVE_LOC_IMM, .type = type, .v.imm = imm};
 }
 
 static NativeLoc nd_loc_global(ObjSymId sym, i64 addend, KitCgTypeId type) {
   return (NativeLoc){.kind = NATIVE_LOC_GLOBAL,
                      .type = type,
                      .v.global = {.sym = sym, .addend = addend}};
 }
 
 static NativeLoc nd_loc_operand(NativeDirectTarget* d, Operand op) {
   switch ((OpKind)op.kind) {
     case OPK_IMM:
       return nd_loc_imm(op.v.imm, op.type);
     case OPK_LOCAL:
       return nd_loc_frame(d, op.v.local, op.type);
     case OPK_GLOBAL:
       return nd_loc_global(op.v.global.sym, op.v.global.addend, op.type);
     case OPK_INDIRECT: {
       NativeDirectLocal* bl = nd_local(d, op.v.ind.base);
       NativeLoc out = (NativeLoc){
           .kind = NATIVE_LOC_ADDR,
           .type = op.type,
           .v.addr = {.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE,
                      .base.frame = nd_home(d, op.v.ind.base),
                      .cls = bl->cls,
                      .base_type = bl->type,
                      .log2_scale = op.v.ind.log2_scale,
                      .offset = op.v.ind.ofs}};
       if (op.v.ind.index != CG_LOCAL_NONE) {
         NativeDirectLocal* il = nd_local(d, op.v.ind.index);
         out.v.addr.index_kind = NATIVE_ADDR_INDEX_FRAME_VALUE;
         out.v.addr.index.frame = nd_home(d, op.v.ind.index);
         out.v.addr.index_cls = il->cls;
         out.v.addr.index_type = il->type;
       }
       return out;
     }
     default:
       nd_panic(d, "bad operand kind");
   }
 }
 
 static NativeAddr nd_addr_storage(NativeDirectTarget* d, Operand op) {
   switch ((OpKind)op.kind) {
     case OPK_LOCAL: {
       /* The local's home is addressed directly (a memory access reads/writes
        * the frame slot itself, e.g. by-value aggregate field extraction). This
        * is not pointer aliasing, but it does read the home, so a cached value
        * must be made current: spill if dirty and drop the entry. */
       NativeDirectLocal* l;
       nd_flush_local(d, op.v.local);
       l = nd_local(d, op.v.local);
       return (NativeAddr){.base_kind = NATIVE_ADDR_BASE_FRAME,
                           .base.frame = nd_home(d, op.v.local),
                           .cls = l->cls,
                           .base_type = l->type};
     }
     case OPK_GLOBAL:
       return (NativeAddr){
           .base_kind = NATIVE_ADDR_BASE_GLOBAL,
           .base.global = {.sym = op.v.global.sym, .addend = op.v.global.addend},
           .base_type = op.type};
     case OPK_INDIRECT: {
       NativeDirectLocal* bl = nd_local(d, op.v.ind.base);
       Reg br = nd_cache_reg_for(d, op.v.ind.base, bl->type);
       NativeAddr out = (NativeAddr){.cls = bl->cls,
                                     .base_type = bl->type,
                                     .log2_scale = op.v.ind.log2_scale,
                                     .offset = op.v.ind.ofs};
       if (br != REG_NONE) {
         out.base_kind = NATIVE_ADDR_BASE_REG;
         out.base.reg = br;
         d->scratch_used[bl->cls] |= 1u
                                     << br; /* pin; unpinned at temps release */
       } else {
         out.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
         out.base.frame = nd_home(d, op.v.ind.base);
       }
       if (op.v.ind.index != CG_LOCAL_NONE) {
         NativeDirectLocal* il = nd_local(d, op.v.ind.index);
         Reg ir = nd_cache_reg_for(d, op.v.ind.index, il->type);
         out.index_cls = il->cls;
         out.index_type = il->type;
         if (ir != REG_NONE) {
           out.index_kind = NATIVE_ADDR_INDEX_REG;
           out.index.reg = ir;
           d->scratch_used[il->cls] |= 1u << ir;
         } else {
           out.index_kind = NATIVE_ADDR_INDEX_FRAME_VALUE;
           out.index.frame = nd_home(d, op.v.ind.index);
         }
       }
       return out;
     }
     default:
       nd_panic(d, "operand is not addressable storage");
   }
 }
 
 static NativeAddr nd_addr_pointer(NativeDirectTarget* d, Operand op) {
   switch ((OpKind)op.kind) {
     case OPK_LOCAL: {
       NativeDirectLocal* l = nd_local(d, op.v.local);
       NativeAddr out = (NativeAddr){.cls = l->cls, .base_type = l->type};
       if (cg_type_is_ptr(d->base.c, op.type)) {
         /* Pointer value lives in the local: use its live register if cached
          * (a dirty cached pointer is a valid base), else load from the home. */
         Reg r = nd_cache_reg_for(d, op.v.local, l->type);
         if (r != REG_NONE) {
           out.base_kind = NATIVE_ADDR_BASE_REG;
           out.base.reg = r;
           d->scratch_used[l->cls] |= 1u << r;
         } else {
           out.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
           out.base.frame = nd_home(d, op.v.local);
         }
       } else {
         /* The local's home is addressed directly; make it current first (see
          * nd_addr_storage OPK_LOCAL). */
         nd_flush_local(d, op.v.local);
         out.base_kind = NATIVE_ADDR_BASE_FRAME;
         out.base.frame = nd_home(d, op.v.local);
       }
       return out;
     }
     case OPK_GLOBAL:
       return (NativeAddr){
           .base_kind = NATIVE_ADDR_BASE_GLOBAL,
           .base.global = {.sym = op.v.global.sym, .addend = op.v.global.addend},
           .base_type = op.type};
     case OPK_INDIRECT:
       return nd_addr_storage(d, op);
     default:
       nd_panic(d, "operand is not a pointer address");
   }
 }
 
 #define ND_REQUIRE_NATIVE(d, member, name)                           \
   do {                                                               \
     if (!(d)->native || !(d)->native->member) nd_panic((d), (name)); \
   } while (0)
 
 typedef struct NdAddrTemps {
   Reg base;
   Reg index;
   NativeAllocClass base_cls;
   NativeAllocClass index_cls;
 } NdAddrTemps;
 
 static void nd_addr_temps_release(NativeDirectTarget* d,
                                   const NdAddrTemps* temps);
 
 static MemAccess nd_scalar_mem(KitCgTypeId type, u32 size, u32 align) {
   return (MemAccess){.type = type, .size = size, .align = align};
 }
 
 static MemAccess nd_type_mem(NativeDirectTarget* d, KitCgTypeId type) {
   u64 size;
   if (!type) type = builtin_id(KIT_CG_BUILTIN_I64);
   size = cg_type_size(d->base.c, type);
   if (size > 0xffffffffu) nd_panic(d, "scalar type is too large");
   return nd_scalar_mem(type, (u32)size, cg_type_align(d->base.c, type));
 }
 
 static void nd_barrier(NativeDirectTarget* d, u32 flags) {
   if (d->ops && d->ops->barrier) d->ops->barrier(d, flags);
 }
 
 static void nd_load_frame_to_reg(NativeDirectTarget* d, NativeLoc dst,
                                  NativeFrameSlot frame, KitCgTypeId type) {
   NativeAddr addr = {.base_kind = NATIVE_ADDR_BASE_FRAME,
                      .base.frame = frame,
                      .base_type = type};
   MemAccess mem = nd_type_mem(d, type);
   ND_REQUIRE_NATIVE(d, load, "target does not emit loads");
   d->native->load(d->native, dst, addr, mem);
 }
 
 static void nd_store_reg_to_frame(NativeDirectTarget* d, NativeFrameSlot frame,
                                   KitCgTypeId type, NativeLoc src) {
   NativeAddr addr = {.base_kind = NATIVE_ADDR_BASE_FRAME,
                      .base.frame = frame,
                      .base_type = type};
   MemAccess mem = nd_type_mem(d, type);
   ND_REQUIRE_NATIVE(d, store, "target does not emit stores");
   d->native->store(d->native, addr, src, mem);
 }
 
 static void nd_copy_to_reg(NativeDirectTarget* d, NativeLoc dst, NativeLoc src);
 static void nd_release_materialized(NativeDirectTarget* d, NativeLoc loc);
 static void nd_store_operand_from_reg(NativeDirectTarget* d, Operand dst,
                                       NativeLoc src);
 
 /* --- Local register cache (write-back, basic-block-scoped) ---------------- *
  * Only scalar, non-address-taken locals are cached, and only in caller-saved
  * allocable registers. Entries are created solely by pure-compute destinations
  * (nd_dst_reg/nd_dst_writeback) and are always dirty; reads hit a live entry or
  * fall back to a frame load without creating one. nd_flush_all spills and
  * empties the cache at the top of every non-pure-compute op, so the cache only
  * survives across straight-line runs of compute ops. Caching prefers the
  * register-file caller-saved mask; if the live OS ABI treats one of those
  * registers as callee-saved, nd_note_reg_used reports it to the backend before
  * the deferred prologue is patched. */
 
 static int nd_local_cacheable(NativeDirectTarget* d,
                               const NativeDirectLocal* l) {
   return !l->address_taken && !l->memory_required && l->size != 0 &&
          !cg_type_is_aggregate(d->base.c, l->type) &&
          l->size <= (u32)d->base.c->target.ptr_size;
 }
 
 /* If LOCAL is currently cached and the access reads it at its cached (storage)
  * width, return its live register; else REG_NONE. Used by the address builders
  * to point an address at a base/index local's live register instead of reading
  * a possibly-stale frame home. Base/index reads are always of the local's own
  * type, so the width check is trivially met for that use; the value-read width
  * hazard is handled separately in nd_materialize_operand. */
 /* Stamp a cache touch (def/read/addressing use) for LRU victim selection. Every
  * caller holds l->reg set (l is currently cached), so the flat reg_last_use
  * mirror the victim scan reads stays in lockstep with l->last_use. */
 static void nd_touch_local(NativeDirectTarget* d, NativeDirectLocal* l) {
   l->last_use = ++d->use_tick;
   d->reg_last_use[l->cls][l->reg] = l->last_use;
 }
 
 static Reg nd_cache_reg_for(NativeDirectTarget* d, CGLocal local,
                             KitCgTypeId access_type) {
   NativeDirectLocal* l = nd_local(d, local);
   if (l->reg == REG_NONE) return REG_NONE;
   if (!nd_local_cacheable(d, l)) return REG_NONE;
   if (access_type && access_type != l->type) return REG_NONE;
   nd_touch_local(d, l);
   return l->reg;
 }
 
 /* Pick the least-recently-used non-pinned cached local in CLS as a spill victim
  * (its register can then be reused). REG_NONE if every owned reg is pinned.
  * Pressure is real in Design B (the cache survives across memory ops), so an
  * arbitrary victim would thrash a hot local; LRU keeps the live set resident.
  */
 static Reg nd_pick_cache_victim(NativeDirectTarget* d, NativeAllocClass cls) {
   const NativeAllocClassInfo* ci = nd_class_info(d, cls);
   Reg best = REG_NONE;
   u32 best_use = 0;
   for (u32 i = 0; i < ci->ndt_allocable_count; ++i) {
     Reg r = ci->ndt_allocable[i];
     u32 use;
     if (r >= 32u) continue;
     if (d->reg_owner[cls][r] == CG_LOCAL_NONE) continue;
     if (d->scratch_used[cls] & (1u << r)) continue; /* pinned: never a victim */
     /* Flat mirror: equals nd_local(owner)->last_use for the owning local. */
     use = d->reg_last_use[cls][r];
     if (best == REG_NONE || use < best_use) {
       best = r;
       best_use = use;
     }
   }
   return best;
 }
 
 /* Pick an allocable register to cache a local in: a free one from the cache
  * pool, else evict the LRU non-pinned cached local. REG_NONE means use the
  * frame-only path. */
 static Reg nd_cache_alloc(NativeDirectTarget* d, NativeAllocClass cls) {
   const Reg* pool = d->cache_pool[cls];
   u32 npool = d->ncache_pool[cls];
   Reg victim;
   /* Scan the precomputed cache pool (allocable order preserved; on
    * ndt_caller_saved_only targets it is the caller-saved subset). */
   for (u32 i = 0; i < npool; ++i) {
     Reg r = pool[i];
     if (d->reg_owner[cls][r] == CG_LOCAL_NONE &&
         (d->scratch_used[cls] & (1u << r)) == 0) {
       nd_note_reg_used(d, cls, r);
       return r;
     }
   }
   /* Under pressure, evict the LRU non-pinned cached local and reuse its
    * register. Every cached local was allocated from cache_pool, so the victim
    * is always a valid pool register — no caller/callee guard needed. */
   victim = nd_pick_cache_victim(d, cls);
   if (victim != REG_NONE) {
     nd_flush_local(d, d->reg_owner[cls][victim]);
     nd_note_reg_used(d, cls, victim);
     return victim;
   }
   return REG_NONE;
 }
 
 /* Append LOCAL to the tail of the cached-locals list (O(1)). Called only on the
  * REG_NONE -> reg transition in nd_dst_reg. */
 static void nd_cache_link(NativeDirectTarget* d, CGLocal local) {
   i32 idx = (i32)(local - 1u);
   i32 prev = d->cache_tail;
   d->locals[idx].cache_next = -1;
   d->locals[idx].cache_prev = prev;
   if (prev >= 0)
     d->locals[prev].cache_next = idx;
   else
     d->cache_head = idx;
   d->cache_tail = idx;
   d->ncached++;
 }
 
 /* Remove LOCAL (which must currently be cached) from the cached-locals list. */
 static void nd_cache_unlink(NativeDirectTarget* d, CGLocal local) {
   i32 idx = (i32)(local - 1u);
   i32 next = d->locals[idx].cache_next;
   i32 prev = d->locals[idx].cache_prev;
   if (next >= 0)
     d->locals[next].cache_prev = prev;
   else
     d->cache_tail = prev;
   if (prev >= 0)
     d->locals[prev].cache_next = next;
   else
     d->cache_head = next;
   d->ncached--;
 }
 
 /* Write a cached local back to its home (if dirty) and drop the entry. Safe to
  * call on an uncached local. */
 static void nd_flush_local(NativeDirectTarget* d, CGLocal local) {
   NativeDirectLocal* l = nd_local(d, local);
   if (l->reg == REG_NONE) return;
   if (l->dirty)
     nd_store_reg_to_frame(
         d, nd_home(d, local), l->type,
         nd_loc_reg(d, l->type, (NativeAllocClass)l->cls, l->reg));
   nd_cache_unlink(d, local);
   d->reg_owner[l->cls][l->reg] = CG_LOCAL_NONE;
   l->reg = REG_NONE;
   l->dirty = 0;
   l->load_zext = 0;
   l->load_sext = 0;
 }
 
 /* Drop a cache entry without writing it back, for when a store supersedes the
  * cached value. */
 static void nd_invalidate_local(NativeDirectTarget* d, CGLocal local) {
   NativeDirectLocal* l = nd_local(d, local);
   if (l->reg == REG_NONE) return;
   nd_cache_unlink(d, local);
   d->reg_owner[l->cls][l->reg] = CG_LOCAL_NONE;
   l->reg = REG_NONE;
   l->dirty = 0;
   l->load_zext = 0;
   l->load_sext = 0;
 }
 
 /* Spill the whole cache to memory and empty it. The list is sorted ascending,
  * so this spills in the same order as the former O(nlocals) index scan. */
 static void nd_flush_all(NativeDirectTarget* d) {
   while (d->cache_head >= 0) nd_flush_local(d, (CGLocal)(d->cache_head + 1));
 }
 
 /* Discard the whole cache WITHOUT writing entries back. Only sound where every
  * cached value is provably dead — at a return the frame is about to be
  * destroyed, so no home is ever read again. Lets nd_ret source the return value
  * from its live register and skip the round-trip + the dead writebacks a
  * flush-all would emit. */
 static void nd_drop_all(NativeDirectTarget* d) {
   while (d->cache_head >= 0)
     nd_invalidate_local(d, (CGLocal)(d->cache_head + 1));
 }
 
 /* A "kept" argument is one nd_call leaves register-resident across the call: it
  * is named in desc->args and flagged dead in arg_dead_mask (so dropping it after
  * the call loses nothing). A live or shared arg is not kept and is spilled. */
 static int nd_call_arg_kept(const CGCallDesc* desc, CGLocal local) {
   for (u32 i = 0; i < desc->nargs; ++i)
     if (desc->args[i] == local)
       return i < 64u && ((desc->arg_dead_mask >> i) & 1u);
   return 0;
 }
 
 /* Spill every cached local EXCEPT the kept (dead) argument locals to its home
  * and empty those entries. Kept args stay cached so the marshalling can source
  * them from their live registers; everything else is potentially live across
  * the call and, on the -O0 caller-saved-only cache, sits in a call-clobbered
  * register, so it must reach memory now. */
 static void nd_flush_all_except_kept_args(NativeDirectTarget* d,
                                           const CGCallDesc* desc) {
   i32 idx = d->cache_head;
   while (idx >= 0) {
     i32 next = d->locals[idx].cache_next; /* capture before flush unlinks idx */
     CGLocal local = (CGLocal)(idx + 1);
     if (!nd_call_arg_kept(desc, local)) nd_flush_local(d, local);
     idx = next;
   }
 }
 
 static NativeAddr nd_addr_materialize(NativeDirectTarget* d, NativeAddr in,
                                       NdAddrTemps* temps, MemAccess mem) {
   NativeAddr out = in;
   memset(temps, 0, sizeof *temps);
   temps->base = REG_NONE;
   temps->index = REG_NONE;
   /* A base/index that arrives already in a register is a pinned live cache reg
    * (the addr builders are the only producers of REG-kind storage addresses).
    * Record it so the temps release unpins it afterward — without storing or
    * invalidating, leaving the cache entry intact. */
   if (out.base_kind == NATIVE_ADDR_BASE_REG) {
     temps->base = out.base.reg;
     temps->base_cls = (NativeAllocClass)out.cls;
   }
   if (out.index_kind == NATIVE_ADDR_INDEX_REG) {
     temps->index = out.index.reg;
     temps->index_cls = (NativeAllocClass)out.index_cls;
   }
   if (out.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeAllocClass cls = (NativeAllocClass)out.cls;
     Reg r = nd_scratch_acquire(d, cls);
     NativeLoc dst = nd_loc_reg(d, out.base_type, cls, r);
     nd_load_frame_to_reg(d, dst, out.base.frame, out.base_type);
     out.base_kind = NATIVE_ADDR_BASE_REG;
     out.base.reg = r;
     temps->base = r;
     temps->base_cls = cls;
   }
   if (out.index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE) {
     NativeAllocClass cls = (NativeAllocClass)out.index_cls;
     Reg r = nd_scratch_acquire(d, cls);
     NativeLoc dst = nd_loc_reg(d, out.index_type, cls, r);
     nd_load_frame_to_reg(d, dst, out.index.frame, out.index_type);
     out.index_kind = NATIVE_ADDR_INDEX_REG;
     out.index.reg = r;
     temps->index = r;
     temps->index_cls = cls;
   }
   if (d->native && d->native->addr_legal &&
       !d->native->addr_legal(d->native, &out, mem)) {
     NativeAllocClass cls = NATIVE_REG_INT;
     Reg r = nd_scratch_acquire(d, cls);
     NativeLoc dst = nd_loc_reg(
         d, out.base_type ? out.base_type : builtin_id(KIT_CG_BUILTIN_I64), cls,
         r);
     ND_REQUIRE_NATIVE(d, load_addr, "target does not materialize addresses");
     d->native->load_addr(d->native, dst, out);
     nd_addr_temps_release(d, temps);
     memset(temps, 0, sizeof *temps);
     temps->base = r;
     temps->index = REG_NONE;
     temps->base_cls = cls;
     out = (NativeAddr){.base_kind = NATIVE_ADDR_BASE_REG,
                        .base.reg = r,
                        .cls = (u8)cls,
                        .base_type = dst.type};
     if (d->native && d->native->addr_legal &&
         !d->native->addr_legal(d->native, &out, mem))
       nd_panic(d, "native address is not legal");
   }
   return out;
 }
 
 static void nd_addr_temps_release(NativeDirectTarget* d,
                                   const NdAddrTemps* temps) {
   if (temps->base != REG_NONE)
     nd_scratch_release(d, temps->base_cls, temps->base);
   if (temps->index != REG_NONE)
     nd_scratch_release(d, temps->index_cls, temps->index);
 }
 
 static NativeLoc nd_materialize_loc(NativeDirectTarget* d, NativeLoc src,
                                     NativeAllocClass cls, KitCgTypeId type) {
   Reg r;
   NativeLoc dst;
   KitCgTypeId dt;
   if (src.kind == NATIVE_LOC_REG) return src;
   dt = type ? type : src.type;
   r = nd_scratch_acquire_span(d, cls, nd_reg_span(d, cls, dt));
   dst = nd_loc_reg(d, dt, cls, r);
   nd_copy_to_reg(d, dst, src);
   return dst;
 }
 
 static void nd_copy_to_reg(NativeDirectTarget* d, NativeLoc dst,
                            NativeLoc src) {
   if (dst.kind != NATIVE_LOC_REG) nd_panic(d, "copy destination is not a reg");
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_REG:
       if (src.v.reg != dst.v.reg || src.cls != dst.cls) {
         if (d->native->move_rr)
           d->native->move_rr(d->native, native_reg_loc_of(dst),
                              native_reg_loc_of(src));
         else {
           ND_REQUIRE_NATIVE(d, move, "target does not emit register moves");
           d->native->move(d->native, dst, src);
         }
       }
       break;
     case NATIVE_LOC_FRAME:
       nd_load_frame_to_reg(d, dst, src.v.frame, dst.type);
       break;
     case NATIVE_LOC_STACK: {
       MemAccess mem = nd_type_mem(d, dst.type);
       NativeAddr addr = {.base_kind = NATIVE_ADDR_BASE_FRAME,
                          .base.frame = src.v.stack.slot,
                          .base_type = dst.type,
                          .offset = src.v.stack.offset};
       ND_REQUIRE_NATIVE(d, load, "target does not emit loads");
       d->native->load(d->native, dst, addr, mem);
       break;
     }
     case NATIVE_LOC_IMM:
       ND_REQUIRE_NATIVE(d, load_imm, "target does not emit immediates");
       d->native->load_imm(d->native, dst, src.v.imm);
       break;
     case NATIVE_LOC_GLOBAL: {
       NativeAddr addr = {.base_kind = NATIVE_ADDR_BASE_GLOBAL,
                          .base.global = {.sym = src.v.global.sym,
                                          .addend = src.v.global.addend},
                          .base_type = dst.type};
       ND_REQUIRE_NATIVE(d, load_addr, "target does not materialize addresses");
       d->native->load_addr(d->native, dst, addr);
       break;
     }
     case NATIVE_LOC_ADDR: {
       NdAddrTemps temps;
       MemAccess mem = nd_scalar_mem(dst.type, d->base.c->target.ptr_size,
                                     d->base.c->target.ptr_align);
       NativeAddr addr = nd_addr_materialize(d, src.v.addr, &temps, mem);
       ND_REQUIRE_NATIVE(d, load_addr, "target does not materialize addresses");
       d->native->load_addr(d->native, dst, addr);
       nd_addr_temps_release(d, &temps);
       break;
     }
     default:
       nd_panic(d, "cannot materialize native location");
   }
 }
 
 static void nd_write_loc(NativeDirectTarget* d, NativeLoc dst, NativeLoc src,
                          MemAccess mem) {
   switch ((NativeLocKind)dst.kind) {
     case NATIVE_LOC_REG:
       nd_copy_to_reg(d, dst, src);
       break;
     case NATIVE_LOC_FRAME: {
       NativeLoc val =
           nd_materialize_loc(d, src, (NativeAllocClass)dst.cls, dst.type);
       nd_store_reg_to_frame(d, dst.v.frame, dst.type, val);
       nd_release_materialized(d, val);
       break;
     }
     case NATIVE_LOC_STACK: {
       NativeLoc val =
           nd_materialize_loc(d, src, (NativeAllocClass)dst.cls, dst.type);
       NativeAddr addr = {.base_kind = NATIVE_ADDR_BASE_FRAME,
                          .base.frame = dst.v.stack.slot,
                          .base_type = dst.type,
                          .offset = dst.v.stack.offset};
       ND_REQUIRE_NATIVE(d, store, "target does not emit stores");
       d->native->store(d->native, addr, val, mem);
       nd_release_materialized(d, val);
       break;
     }
     case NATIVE_LOC_ADDR: {
       NdAddrTemps temps;
       NativeAddr addr = nd_addr_materialize(d, dst.v.addr, &temps, mem);
       NativeAllocClass cls = nd_class_for_type(d, src.type);
       NativeLoc val = nd_materialize_loc(d, src, cls, src.type);
       ND_REQUIRE_NATIVE(d, store, "target does not emit stores");
       d->native->store(d->native, addr, val, mem);
       nd_release_materialized(d, val);
       nd_addr_temps_release(d, &temps);
       break;
     }
     default:
       nd_panic(d, "unsupported write destination");
   }
 }
 
 static void nd_release_materialized(NativeDirectTarget* d, NativeLoc loc) {
   if (loc.kind == NATIVE_LOC_REG) {
     u32 span = nd_reg_span(d, (NativeAllocClass)loc.cls, loc.type), k;
     for (k = 0; k < span; ++k)
       nd_scratch_release(d, (NativeAllocClass)loc.cls, (Reg)(loc.v.reg + k));
   }
 }
 
 /* Spill cached locals that back an INDIRECT operand's address before it is read
  * from their frame homes. Compute ops normally receive only LOCAL/IMM/GLOBAL
  * operands; this keeps the rare INDIRECT case correct without flushing all. */
 static void nd_flush_operand_addr_locals(NativeDirectTarget* d, Operand op) {
   if (op.kind != OPK_INDIRECT) return;
   nd_flush_local(d, op.v.ind.base);
   if (op.v.ind.index != CG_LOCAL_NONE) nd_flush_local(d, op.v.ind.index);
 }
 
 /* Materialize OP's value into a register and return it — the tcc gv(rc) analog:
  * "generalize this value into a register, doing nothing if it is already there."
  *
  *   want == NULL : ANY register of OP's class (gv with a generic class). A cache
  *                  hit returns the live register pinned, with no code; otherwise
  *                  a fresh scratch is acquired and OP loaded into it. The caller
  *                  releases the pin with nd_release_materialized.
  *   want != NULL : that SPECIFIC, already-acquired+pinned register (gv into a
  *                  fixed reg). A cache-resident OP costs the one unavoidable
  *                  reg-reg move; a memory / immediate / global / indirect OP is
  *                  loaded straight into *want (no scratch round-trip). *want
  *                  carries the destination type/width the copy uses, which may
  *                  differ from OP's (e.g. a narrowing copy), so it — not OP —
  *                  drives nd_copy_to_reg. Returns *want.
  *
  * Both modes share one residence resolution (cache hit, or width-mismatch flush
  * then memory), so there is a single soundness argument for getting a value into
  * a register at -O0. */
 static NativeLoc nd_gv(NativeDirectTarget* d, Operand op, const NativeLoc* want) {
   NativeAllocClass cls = nd_class_for_type(d, op.type);
   if (op.kind == OPK_LOCAL) {
     NativeDirectLocal* l = nd_local(d, op.v.local);
     if (l->reg != REG_NONE && op.type == l->type && nd_local_cacheable(d, l)) {
       nd_touch_local(d, l);
       if (!want) {
         /* Cache hit: pin and reuse the live register, no reload. */
         d->scratch_used[l->cls] |= 1u << l->reg;
         return nd_loc_reg(d, op.type, (NativeAllocClass)l->cls, l->reg);
       }
       nd_copy_to_reg(d, *want,
                      nd_loc_reg(d, op.type, (NativeAllocClass)l->cls, l->reg));
       return *want;
     }
     /* A live entry under a different access width must reach memory before we
      * bypass the cache for this access. */
     if (l->reg != REG_NONE) nd_flush_local(d, op.v.local);
   }
   nd_flush_operand_addr_locals(d, op);
   if (!want)
     return nd_materialize_loc(d, nd_loc_operand(d, op), cls, op.type);
   nd_copy_to_reg(d, *want, nd_loc_operand(d, op));
   return *want;
 }
 
 static NativeLoc nd_materialize_operand(NativeDirectTarget* d, Operand op) {
   return nd_gv(d, op, NULL);
 }
 
 /* Materialize OP straight into the already-acquired register WANT (gv into a
  * fixed reg), skipping the scratch round-trip + separate move a memory /
  * immediate / global / indirect source would otherwise make. WANT must already
  * be pinned by the caller. */
 static void nd_materialize_operand_into(NativeDirectTarget* d, NativeLoc want,
                                         Operand op) {
   nd_gv(d, op, &want);
 }
 
 static NativeLoc nd_dst_scratch(NativeDirectTarget* d, Operand dst) {
   NativeAllocClass cls = nd_class_for_type(d, dst.type);
   Reg r = nd_scratch_acquire_span(d, cls, nd_reg_span(d, cls, dst.type));
   return nd_loc_reg(d, dst.type, cls, r);
 }
 
 /* Arithmetic/compare RHS: keep a constant operand as an immediate when the
  * target can encode it for `use` (so no scratch register is spent
  * materializing it), mirroring the optimizer's operand_imm_or_reg. Falls back
  * to a register when there is no imm_legal hook (e.g. a recording mock target)
  * or the constant is not target-legal for this op. */
 static NativeLoc nd_rhs_imm_or_reg(NativeDirectTarget* d, NativeImmUse use,
                                    u32 sub, Operand b) {
   if (b.kind == OPK_IMM && d->native->imm_legal &&
       d->native->imm_legal(d->native, use, sub, b.type, b.v.imm))
     return nd_loc_imm(b.v.imm, b.type);
   return nd_materialize_operand(d, b);
 }
 
 /* Register a pure-compute op writes its result into. For a cacheable local that
  * is the local's cache register (reused or freshly allocated), pinned for the
  * instruction; nd_dst_writeback then marks it dirty without storing. Otherwise
  * a scratch temporary that nd_dst_writeback spills to the frame home. */
 static NativeLoc nd_dst_reg(NativeDirectTarget* d, Operand dst) {
   if (dst.kind == OPK_LOCAL) {
     NativeDirectLocal* l = nd_local(d, dst.v.local);
     if (dst.type == l->type && nd_local_cacheable(d, l)) {
       Reg r = l->reg;
       if (r == REG_NONE) {
         r = nd_cache_alloc(d, (NativeAllocClass)l->cls);
         if (r != REG_NONE) {
           d->reg_owner[l->cls][r] = dst.v.local;
           l->reg = r;
           nd_cache_link(d, dst.v.local);
         }
       }
       if (r != REG_NONE) {
         d->scratch_used[l->cls] |= 1u << r; /* pin for the instruction */
         nd_touch_local(d, l);
         return nd_loc_reg(d, dst.type, (NativeAllocClass)l->cls, r);
       }
     }
   }
   return nd_dst_scratch(d, dst);
 }
 
 static void nd_dst_writeback(NativeDirectTarget* d, Operand dst, NativeLoc dr) {
   if (dst.kind == OPK_LOCAL) {
     NativeDirectLocal* l = nd_local(d, dst.v.local);
     if (dr.kind == NATIVE_LOC_REG && l->reg == dr.v.reg &&
         dst.type == l->type && nd_local_cacheable(d, l)) {
       l->dirty = 1;
       l->load_zext = 0; /* a fresh value; nd_load re-sets this for narrow loads */
       l->load_sext = 0; /* "" (re-set only by an MF_SEXT_LOAD load) */
       d->scratch_used[l->cls] &= ~(1u << dr.v.reg); /* unpin, keep cached */
       return;
     }
     /* Bypassing the cache: drop any stale entry, then spill to the home. */
     if (l->reg != REG_NONE) nd_invalidate_local(d, dst.v.local);
   }
   nd_store_operand_from_reg(d, dst, dr);
   nd_release_materialized(d, dr);
 }
 
 /* Coalescing rename: SRC (a transient flagged dead — OPK_FLAG_KILL) holds its
  * live value in a cache register; transfer ownership of that register to DST so
  * a copy needs no mov and a convert can run in place. Returns the now-DST-owned
  * register (pinned for the consuming instruction; the caller passes it to
  * nd_dst_writeback to unpin + mark dirty), or REG_NONE when a rename is not
  * applicable and the caller must take its normal materialize path.
  *
  * Applicable only when: SRC is cached at its own (storage) width so the register
  * truly holds SRC's value; SRC and DST are both cacheable scalars in the SAME
  * register class (a width change within the class — e.g. a sxtb — is fine: the
  * caller's convert rewrites the register in place; a class change GPR<->FPR is
  * not, that needs a real cross-bank move). The register's bits become DST's, so
  * DST is marked dirty (its home is now stale) and pinned; SRC is dropped without
  * write-back (it is dead). A pre-existing DST cache entry is discarded first
  * (the copy/convert supersedes it). Sound because OPK_FLAG_KILL is set only from
  * api_temp_dead, which confirms no live value-stack reference to SRC remains. */
 static Reg nd_rename_killed_to_dst(NativeDirectTarget* d, Operand dst,
                                    Operand src) {
   NativeDirectLocal* sl;
   NativeDirectLocal* dl;
   Reg r;
   if (!(src.flags & OPK_FLAG_KILL) || src.kind != OPK_LOCAL ||
       dst.kind != OPK_LOCAL || src.v.local == dst.v.local)
     return REG_NONE;
   sl = nd_local(d, src.v.local);
   dl = nd_local(d, dst.v.local);
   if (sl->reg == REG_NONE || sl->type != src.type || !nd_local_cacheable(d, sl))
     return REG_NONE;
   if (sl->cls != dl->cls || !nd_local_cacheable(d, dl)) return REG_NONE;
   r = sl->reg;
   /* Detach SRC from R without writing it back (dead) and discard any stale DST
    * cache entry (its old value is about to be overwritten). */
   nd_invalidate_local(d, src.v.local);
   if (dl->reg != REG_NONE) nd_invalidate_local(d, dst.v.local);
   /* R is now DST's: take ownership, mark dirty (home stale), pin for the op. */
   d->reg_owner[dl->cls][r] = dst.v.local;
   dl->reg = r;
   dl->dirty = 1;
   nd_cache_link(d, dst.v.local);
   d->scratch_used[dl->cls] |= 1u << r;
   nd_touch_local(d, dl);
   return r;
 }
 
 /* After an op has materialized and consumed OP, drop OP's cache entry WITHOUT a
  * write-back when it carries OPK_FLAG_KILL — a transient the cg layer proved dead
  * after this op (api_temp_dead). The op already read OP's value, so storing it at
  * the next barrier would be wasted and holding its register inflates pressure;
  * dropping it now both removes that dead spill and frees the register immediately
  * (fewer eviction spills + reloads downstream). Sound for the same reason
  * nd_rename_killed_to_dst is: OPK_FLAG_KILL is set only from api_temp_dead, which
  * confirms no live value-stack reference to OP remains. A non-local or non-cached
  * operand is a no-op (nd_invalidate_local early-returns on REG_NONE). The caller
  * must have already unpinned OP's materialized register (nd_release_materialized);
  * the cg layer never flags an operand that is also the op's destination. */
 static void nd_drop_killed_operand(NativeDirectTarget* d, Operand op) {
   if ((op.flags & OPK_FLAG_KILL) && op.kind == OPK_LOCAL)
     nd_invalidate_local(d, op.v.local);
 }
 
 static void nd_store_operand_from_reg(NativeDirectTarget* d, Operand dst,
                                       NativeLoc src) {
   if (dst.kind != OPK_LOCAL) nd_panic(d, "destination is not a semantic local");
   /* This writes SRC to the local's frame home, bypassing the value cache (the
    * result was produced in a scratch reg, e.g. a load / address-of). Any live
    * cache entry for the local is now stale and must be dropped — the home write
    * supersedes it. Drop without storing; storing back would clobber the new
    * home value. Runs after SRC is produced, so a dst that was its own address
    * base has already been consumed. */
   {
     NativeDirectLocal* l = nd_local(d, dst.v.local);
     if (l->reg != REG_NONE) nd_invalidate_local(d, dst.v.local);
     nd_store_reg_to_frame(d, nd_home(d, dst.v.local), dst.type, src);
   }
 }
 
 static void nd_func_begin(CgTarget* t, const CGFuncDesc* fd) {
   NativeDirectTarget* d = nd_of(t);
   d->func = fd;
   d->nlocals = 0;
   d->ntransient = 0;
   d->nlabels = 0;
   d->nscopes = 0;
   d->max_outgoing = 0;
   d->use_tick = 0;
   d->cache_head = -1;
   d->cache_tail = -1;
   d->ncached = 0;
   memset(d->scratch_used, 0, sizeof d->scratch_used);
   memset(d->callee_saved_used, 0, sizeof d->callee_saved_used);
   memset(d->reg_owner, 0, sizeof d->reg_owner);
   if (d->native && d->native->func_begin) d->native->func_begin(d->native, fd);
 }
 
 static void nd_func_end(CgTarget* t) {
   NativeDirectTarget* d = nd_of(t);
   NativeFramePatchState frame;
   memset(&frame, 0, sizeof frame);
   frame.max_outgoing = d->max_outgoing;
   if (d->reg_info && d->reg_info->ndt_caller_saved_only) {
     /* Caller-saved-only target (e.g. aa64): the -O0 path never allocates a
      * callee-saved register — nd_scratch_acquire and the local cache restrict
      * themselves to caller-saved regs. The prologue therefore never
      * saves/restores any callee-save, which is what lets the backend reserve
      * only a tiny tcc-style deferred-`sub` prologue region. Assert it so a
      * future regression can't silently overflow that region. */
     for (u32 cls = 0; cls < NATIVE_REG_CLASS_COUNT; ++cls) {
       if (d->callee_saved_used[cls])
         nd_panic(d, "ndt_caller_saved_only target used a callee-saved register");
     }
   } else {
     /* Default: NDT may use callee-saved registers as extra scratch under
      * pressure; have the backend reserve save slots for them. The
      * optimizer/known-frame path is separate and always callee-save-capable. */
     u32 ncallee_classes = 0;
     for (u32 cls = 0; cls < NATIVE_REG_CLASS_COUNT; ++cls) {
       if (d->callee_saved_used[cls]) ncallee_classes = cls + 1u;
     }
     if (ncallee_classes) {
       if (!d->native || !d->native->reserve_callee_saves)
         nd_panic(d, "target cannot preserve callee-saved scratch registers");
       d->native->reserve_callee_saves(d->native, d->callee_saved_used,
                                       ncallee_classes);
     }
   }
   if (d->native && d->native->note_frame_state)
     d->native->note_frame_state(d->native, &frame);
   if (d->native && d->native->patch_apply) d->native->patch_apply(d->native);
   if (d->native && d->native->func_end) d->native->func_end(d->native);
   d->func = NULL;
 }
 
 /* Reclaim dead transient temp slots at a statement boundary. cg calls this only
  * when its value stack is empty, so every transient minted since the last
  * reclaim is provably dead: nothing on the stack references it.
  *
  * For each such temp we DROP (not spill) any live cache entry — a pure-compute
  * expression statement can leave a transient cached+dirty, and spilling it would
  * write a dead value into a home we are about to recycle, then a later
  * nd_flush_all could resurrect it into a different temp now sharing that slot.
  * Then we return its frame home to the free list for the next statement's temps.
  *
  * An address-taken transient is excluded conservatively: nd_local_addr already
  * flushed it and marked it uncacheable, and a pointer to its home may have
  * escaped, so we keep its slot live (it is never cached here). At -O0 no C
  * construct takes the address of a compiler temp, so this branch is belt-and-
  * suspenders. Non-transient locals (declared locals/params) are untouched —
  * their cached/dirty values legitimately live across statements. */
 static void nd_reclaim_temps(CgTarget* t) {
   NativeDirectTarget* d = nd_of(t);
   for (u32 i = 0; i < d->ntransient; ++i) {
     CGLocal local = d->transient_locals[i];
     NativeDirectLocal* l = nd_local(d, local);
     if (l->address_taken) continue;
     nd_invalidate_local(d, local); /* drop cache entry, no write-back */
     if (d->native && d->native->release_frame_slot &&
         l->home != NATIVE_FRAME_SLOT_NONE)
       d->native->release_frame_slot(d->native, l->home);
   }
   d->ntransient = 0;
 }
 
 static void nd_alias(CgTarget* t, ObjSymId alias_sym, ObjSymId target_sym,
                      KitCgTypeId type) {
   (void)t;
   (void)alias_sym;
   (void)target_sym;
   (void)type;
 }
 
 static CGLocal nd_local_new(CgTarget* t, const CGLocalDesc* desc) {
   return nd_alloc_local(nd_of(t), desc);
 }
 
 static void nd_local_addr(CgTarget* t, Operand dst, const CGLocalDesc* desc,
                           CGLocal local) {
   NativeDirectTarget* d = nd_of(t);
   NativeDirectLocal* l = nd_local(d, local);
   Operand lv;
   (void)desc;
   /* Targeted flush: only this local escapes. Spill+drop its entry so the home
    * is authoritative for the address computation, then mark it uncacheable. The
    * rest of the cache is unaffected (other cached locals stay non-escaped). */
   nd_flush_local(d, local);
   l->address_taken = 1;
   l->flags |= CG_LOCAL_ADDR_TAKEN;
   memset(&lv, 0, sizeof lv);
   lv.kind = OPK_LOCAL;
   lv.type = l->type;
   lv.v.local = local;
   {
     /* Route the &local address through the write-back cache (see nd_addr_of):
      * write it into dst's cache register rather than a scratch copied to the
      * home, killing the mov and keeping it resident for an immediate consumer. */
     NativeLoc reg = nd_dst_reg(d, dst);
     ND_REQUIRE_NATIVE(d, load_addr, "target does not materialize addresses");
     d->native->load_addr(d->native, reg, nd_addr_storage(d, lv));
     nd_dst_writeback(d, dst, reg);
   }
 }
 
 static CGLocal nd_param(CgTarget* t, const CGParamDesc* desc) {
   NativeDirectTarget* d = nd_of(t);
   NativeDirectLocal* l;
   NativeFrameSlotDesc fsd;
   CGLocal id;
   nd_grow_locals(d, d->nlocals + 1u);
   id = d->nlocals + 1u;
   l = &d->locals[d->nlocals++];
   memset(l, 0, sizeof *l);
   l->type = desc->type;
   l->size = desc->size;
   l->align = desc->align;
   l->flags = desc->flags;
   l->reg = REG_NONE;
   l->address_taken = (desc->flags & CG_LOCAL_ADDR_TAKEN) != 0;
   l->memory_required = (desc->flags & CG_LOCAL_MEMORY_REQUIRED) != 0;
   l->cls = (u8)nd_class_for_type(d, desc->type);
   fsd = nd_slot_desc_param(desc);
   l->home = nd_alloc_frame_slot(d, &fsd);
   if (d->ops && d->ops->bind_param) d->ops->bind_param(d, desc, id, l);
   return id;
 }
 
 static int nd_local_debug_loc(CgTarget* t, CGLocal local, CGDebugLoc* out) {
   NativeDirectTarget* d = nd_of(t);
   NativeDirectLocal* l;
   if (!out) return 0;
   memset(out, 0, sizeof *out);
   if (!d->native || !d->native->frame_slot_debug_loc) return 0;
   l = nd_local(d, local);
   if (l->home == NATIVE_FRAME_SLOT_NONE) return 0;
   return d->native->frame_slot_debug_loc(d->native, l->home, out);
 }
 
 static Label nd_label_new(CgTarget* t) { return nd_label_new_raw(nd_of(t)); }
 
 static void nd_label_place(CgTarget* t, Label label) {
   NativeDirectTarget* d = nd_of(t);
   nd_flush_all(d);
   ND_REQUIRE_NATIVE(d, label_place, "target does not place labels");
   d->native->label_place(d->native, nd_mc_label(d, label));
 }
 
 static void nd_jump(CgTarget* t, Label label) {
   NativeDirectTarget* d = nd_of(t);
   nd_flush_all(d);
   ND_REQUIRE_NATIVE(d, jump, "target does not emit jumps");
   d->native->jump(d->native, nd_mc_label(d, label));
 }
 
 static void nd_cmp_branch(CgTarget* t, CmpOp op, Operand a, Operand b,
                           Label label) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc ar, br;
   /* Materialize the compare operands BEFORE spilling the live-across set. A
    * cache-resident operand is then read straight from its register instead of
    * being spilled by the flush and immediately reloaded for the compare; a dead
    * one (OPK_FLAG_KILL) is dropped without a write-back so the flush skips its
    * store entirely. The operands stay pinned across the flush — their values
    * survive in their registers (a spill store does not clobber them) — and the
    * flush's stores are emitted before the compare, so they cannot clobber its
    * condition flags. The post-branch cache state is still empty (the flush spills
    * everything live), so control-flow-merge correctness is unchanged. */
   ar = nd_materialize_operand(d, a);
   br = nd_rhs_imm_or_reg(d, NATIVE_IMM_CMP, (u32)op, b);
   nd_drop_killed_operand(d, a);
   nd_drop_killed_operand(d, b);
   nd_flush_all(d);
   ND_REQUIRE_NATIVE(d, cmp_branch, "target does not emit compare branches");
   d->native->cmp_branch(d->native, op, ar, br, nd_mc_label(d, label));
   nd_release_materialized(d, br);
   nd_release_materialized(d, ar);
 }
 
 /* Cmp-and-branch chain with the selector pinned in ONE register across every
  * case. This emits the same shape as cg_lower_switch_default (one CMP_EQ per
  * case against cases[i].value at selector_type, then a jump to the default if
  * any), but materializes the selector ONCE — before the flush — instead of
  * letting each per-case cmp_branch reload it from its home (the flush empties
  * the cache, so the shared lowering would re-materialize + re-flush per case).
  *
  * Soundness: after the initial flush the cache is empty. The chain falls through
  * case-to-case with that same empty state — sel is a pinned scratch reg, not a
  * cache entry, so nothing is re-cached between cases. Each case body and the
  * default are reached through nd_label_place / nd_jump, which flush an
  * already-empty cache. So the cache is empty along every out-edge: merge-correct
  * exactly as the shared lowering is. The selector's value survives the flush in
  * its pinned register (a spill store does not clobber it). Other backends and
  * opt's IR replay still use cg_lower_switch_default unchanged. */
 static void nd_switch(CgTarget* t, const CGSwitchDesc* desc) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc sel;
   if (desc->ncases == 0) {
     nd_flush_all(d);
     if (desc->default_label != LABEL_NONE) {
       ND_REQUIRE_NATIVE(d, jump, "target does not emit jumps");
       d->native->jump(d->native, nd_mc_label(d, desc->default_label));
     }
     return;
   }
   sel = nd_materialize_operand(d, desc->selector);
   nd_flush_all(d);
   ND_REQUIRE_NATIVE(d, cmp_branch, "target does not emit compare branches");
   for (u32 i = 0; i < desc->ncases; ++i) {
     /* Same per-case immediate cg_lower_switch_default builds with api_op_imm:
      * cases[i].value interpreted at selector_type. Constructed inline (the imm
      * helper is not declared in this TU). */
     Operand imm = {.kind = OPK_IMM,
                    .type = desc->selector_type,
                    .v.imm = (i64)desc->cases[i].value};
     NativeLoc br = nd_rhs_imm_or_reg(d, NATIVE_IMM_CMP, (u32)CMP_EQ, imm);
     d->native->cmp_branch(d->native, CMP_EQ, sel, br,
                           nd_mc_label(d, desc->cases[i].label));
     nd_release_materialized(d, br);
   }
   if (desc->default_label != LABEL_NONE) {
     ND_REQUIRE_NATIVE(d, jump, "target does not emit jumps");
     d->native->jump(d->native, nd_mc_label(d, desc->default_label));
   }
   nd_release_materialized(d, sel);
 }
 
 static void nd_indirect_branch(CgTarget* t, Operand addr,
                                const Label* valid_targets, u32 ntargets) {
   NativeDirectTarget* d = nd_of(t);
   MCLabel* native_targets;
   NativeLoc addr_reg;
   /* Materialize the target address BEFORE the flush (mirrors nd_cmp_branch): a
    * cache-resident address is read from its register instead of being spilled
    * by the flush and immediately reloaded. The reg stays pinned across the
    * flush — its value survives (a spill store does not clobber it) — and the
    * post-branch cache is still empty, so merge correctness is unchanged. */
   addr_reg = nd_materialize_operand(d, addr);
   nd_drop_killed_operand(d, addr);
   nd_flush_all(d);
   ND_REQUIRE_NATIVE(d, indirect_branch,
                     "target does not emit indirect branches");
   native_targets =
       ntargets == 0 ? NULL
       : ntargets <= ND_LBL_BUF
           ? d->lblbuf
           : nd_arena(d, sizeof(*native_targets) * ntargets, _Alignof(MCLabel));
   for (u32 i = 0; i < ntargets; ++i)
     native_targets[i] = nd_mc_label(d, valid_targets[i]);
   d->native->indirect_branch(d->native, addr_reg, native_targets, ntargets);
   nd_release_materialized(d, addr_reg);
 }
 
 static void nd_load_label_addr(CgTarget* t, Operand dst, Label label) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc reg;
   nd_flush_all(d);
   reg = nd_dst_reg(d, dst);
   ND_REQUIRE_NATIVE(d, load_label_addr,
                     "target does not materialize label addresses");
   d->native->load_label_addr(d->native, reg, nd_mc_label(d, label));
   nd_dst_writeback(d, dst, reg);
 }
 
 static int nd_local_static_data_begin(CgTarget* t,
                                       const CGLocalStaticDataDesc* desc) {
   NativeDirectTarget* d = nd_of(t);
   Sym name;
   SecKind kind;
   u16 flags;
   if (!d->native || !d->native->mc || !desc) return 0;
   if (d->local_static_active) nd_panic(d, "nested local static data");
   if (desc->attrs.section) {
     name = (Sym)desc->attrs.section;
     kind =
         (desc->attrs.flags & KIT_CG_DATADEF_READONLY) ? SEC_RODATA : SEC_DATA;
     flags = (desc->attrs.flags & KIT_CG_DATADEF_READONLY)
                 ? SF_ALLOC
                 : (SF_ALLOC | SF_WRITE);
   } else if (desc->attrs.flags & KIT_CG_DATADEF_READONLY) {
     name = pool_intern_slice(t->c->global, SLICE_LIT(".rodata"));
     kind = SEC_RODATA;
     flags = SF_ALLOC;
   } else {
     name = pool_intern_slice(t->c->global, SLICE_LIT(".data"));
     kind = SEC_DATA;
     flags = SF_ALLOC | SF_WRITE;
   }
   d->local_static_sec =
       obj_section(t->obj, name, kind, flags, desc->align ? desc->align : 1u);
   d->local_static_base =
       obj_align_to(t->obj, d->local_static_sec, desc->align ? desc->align : 1u);
   d->local_static_size = 0;
   d->local_static_sym = desc->sym;
   d->local_static_active = 1;
   return 1;
 }
 
 static void nd_local_static_data_write(CgTarget* t, const u8* data, u64 len) {
   NativeDirectTarget* d = nd_of(t);
   u8 zero[64];
   u64 orig_len = len;
   if (!d->local_static_active || !len) return;
   if (data) {
     obj_write(t->obj, d->local_static_sec, data, (size_t)len);
   } else {
     memset(zero, 0, sizeof zero);
     while (len >= sizeof zero) {
       obj_write(t->obj, d->local_static_sec, zero, sizeof zero);
       len -= sizeof zero;
     }
     if (len) obj_write(t->obj, d->local_static_sec, zero, (size_t)len);
   }
   d->local_static_size += (u32)orig_len;
 }
 
 static void nd_local_static_data_label_addr(CgTarget* t, Label target,
                                             i64 addend, u32 width,
                                             u32 address_space) {
   NativeDirectTarget* d = nd_of(t);
   u32 off;
   u8 zero[8];
   RelocKind kind;
   (void)address_space;
   if (!d->local_static_active)
     nd_panic(d, "label address outside local static data");
   /* A jump-table / label-address slot is one target pointer wide: 8 bytes
    * (R_ABS64) on a 64-bit target, 4 bytes (R_ABS32) on rv32/ELFCLASS32. */
   if (width == 8u)
     kind = R_ABS64;
   else if (width == 4u)
     kind = R_ABS32;
   else {
     nd_panic(d, "unsupported local static label address width");
     return;
   }
   memset(zero, 0, sizeof zero);
   off = d->local_static_base + d->local_static_size;
   obj_write(t->obj, d->local_static_sec, zero, width);
   mc_emit_label_data_reloc(d->native->mc, d->local_static_sec, off,
                                        nd_mc_label(d, target), kind, width,
                                        addend);
   d->local_static_size += width;
 }
 
 static void nd_local_static_data_end(CgTarget* t) {
   NativeDirectTarget* d = nd_of(t);
   if (!d->local_static_active) return;
   obj_symbol_define_live(t->obj, d->local_static_sym, d->local_static_sec,
                          d->local_static_base, d->local_static_size);
   d->local_static_active = 0;
   d->local_static_sec = OBJ_SEC_NONE;
   d->local_static_sym = OBJ_SYM_NONE;
   d->local_static_base = 0;
   d->local_static_size = 0;
 }
 
 static const char* nd_data_label_addr_unsupported_msg(CgTarget* t) {
   (void)t;
   return NULL;
 }
 
 static CGScope nd_scope_begin(CgTarget* t, const CGScopeDesc* desc) {
   NativeDirectTarget* d = nd_of(t);
   NativeDirectScope* s;
   CGScope id;
   nd_grow_scopes(d, d->nscopes + 1u);
   id = d->nscopes + 1u;
   s = &d->scopes[d->nscopes++];
   memset(s, 0, sizeof *s);
   s->kind = desc->kind;
   s->owns_break = desc->break_label == LABEL_NONE;
   s->break_label = desc->break_label ? desc->break_label : nd_label_new_raw(d);
   s->continue_label = desc->continue_label;
   if (desc->kind == SCOPE_LOOP && s->continue_label == LABEL_NONE)
     s->continue_label = nd_label_new_raw(d);
   return id;
 }
 
 static NativeDirectScope* nd_scope(NativeDirectTarget* d, CGScope scope) {
   if (scope == CG_SCOPE_NONE || scope > d->nscopes) nd_panic(d, "bad scope");
   return &d->scopes[scope - 1u];
 }
 
 static void nd_scope_end(CgTarget* t, CGScope scope) {
   NativeDirectTarget* d = nd_of(t);
   NativeDirectScope* s = nd_scope(d, scope);
   if (s->owns_break) nd_label_place(t, s->break_label);
 }
 
 static void nd_break_to(CgTarget* t, CGScope scope) {
   nd_jump(t, nd_scope(nd_of(t), scope)->break_label);
 }
 
 static void nd_continue_to(CgTarget* t, CGScope scope) {
   NativeDirectScope* s = nd_scope(nd_of(t), scope);
   if (s->continue_label == LABEL_NONE)
     nd_panic(nd_of(t), "continue_to on scope without continue label");
   nd_jump(t, s->continue_label);
 }
 
 static int nd_is_wide64_int(NativeDirectTarget* d, KitCgTypeId ty);
 static int nd_is_soft_double(NativeDirectTarget* d, KitCgTypeId ty);
 
 static void nd_load_imm(CgTarget* t, Operand dst, i64 imm) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc reg;
   if (nd_is_wide64_int(d, dst.type))
     nd_panic(d,
              "64-bit integer immediate reached the backend un-lowered "
              "(cg should materialize it as two 32-bit lanes)");
   reg = nd_dst_reg(d, dst);
   ND_REQUIRE_NATIVE(d, load_imm, "target does not emit immediates");
   d->native->load_imm(d->native, reg, imm);
   nd_dst_writeback(d, dst, reg);
 }
 
 static void nd_load_const(CgTarget* t, Operand dst, ConstBytes cbytes) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc reg;
   if (nd_is_wide64_int(d, dst.type) || nd_is_soft_double(d, dst.type))
     nd_panic(d,
              "8-byte constant reached the backend un-lowered (cg should "
              "materialize it as two 32-bit lanes)");
   reg = nd_dst_reg(d, dst);
   ND_REQUIRE_NATIVE(d, load_const, "target does not emit byte constants");
   d->native->load_const(d->native, reg, cbytes);
   nd_dst_writeback(d, dst, reg);
 }
 
 static void nd_copy(CgTarget* t, Operand dst, Operand src) {
   NativeDirectTarget* d = nd_of(t);
   u64 size = dst.type ? cg_type_size(t->c, dst.type) : 0;
   if (size > (u64)t->c->target.ptr_size) {
     NdAddrTemps dt, st;
     AggregateAccess access;
     /* Aggregate copy: addresses are built cache-aware (a directly-addressed
      * cached local is flushed in nd_addr_storage), so no whole-cache flush. */
     memset(&access, 0, sizeof access);
     access.type = dst.type;
     access.size = (u32)size;
     access.align =
         dst.type ? cg_type_align(t->c, dst.type) : (u32)t->c->target.ptr_align;
     access.mem.type = dst.type;
     access.mem.size = access.size;
     access.mem.align = access.align;
     NativeAddr da =
         nd_addr_materialize(d, nd_addr_storage(d, dst), &dt, access.mem);
     NativeAddr sa =
         nd_addr_materialize(d, nd_addr_storage(d, src), &st, access.mem);
     ND_REQUIRE_NATIVE(d, copy_bytes, "target does not copy bytes");
     d->native->copy_bytes(d->native, da, sa, access);
     nd_addr_temps_release(d, &st);
     nd_addr_temps_release(d, &dt);
     return;
   }
   /* Coalesce `x = t` when t is a dead transient resident in a register: rename
    * its register to x (transfer ownership) instead of materialize + mov. Only
    * for an exact same-type copy (the register bits become x's unchanged). */
   if (dst.type == src.type) {
     Reg r = nd_rename_killed_to_dst(d, dst, src);
     if (r != REG_NONE) {
       /* Register already holds the value as x's; just unpin + mark dirty. */
       nd_dst_writeback(d, dst,
                        nd_loc_reg(d, dst.type, nd_class_for_type(d, dst.type),
                                   r));
       return;
     }
   }
   /* Acquire the destination register first, then materialize the source
    * straight into it: a memory/immediate/global source loads directly into dr
    * (no scratch + mov), a cached source costs the one unavoidable reg-reg move. */
   NativeLoc dr = nd_dst_reg(d, dst);
   nd_materialize_operand_into(d, dr, src);
   nd_dst_writeback(d, dst, dr);
 }
 
 /* Bit-fields ride the generic load/store (mem.bf_width != 0); this impl
  * translates them to the physical NativeTarget bitfield_load/store below. */
 static void nd_bitfield_load(CgTarget* t, Operand dst, Operand record_addr,
                              BitFieldAccess access);
 static void nd_bitfield_store(CgTarget* t, Operand record_addr, Operand src,
                               BitFieldAccess access);
 
 static void nd_load(CgTarget* t, Operand dst, Operand addr, MemAccess mem) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   u64 size;
   if (mem.bf_width != 0) {
     nd_bitfield_load(t, dst, addr, bf_from_mem(mem));
     return;
   }
   size = mem.size ? mem.size : (mem.type ? cg_type_size(t->c, mem.type) : 0);
   /* No value-cache flush (2c): the cache holds only non-escaped locals, which no
    * pointer can alias, so a volatile access — foreign memory — can neither observe
    * a cached value's stale home nor overwrite it. The arch barrier hook still
    * fires so an instrumentation seam can order the access. */
   if (mem.flags & MF_VOLATILE)
     nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_VOLATILE);
   NativeAddr naddr =
       nd_addr_materialize(d, nd_addr_storage(d, addr), &temps, mem);
   if (size > (u64)t->c->target.ptr_size) {
     NdAddrTemps dt;
     AggregateAccess access;
     memset(&access, 0, sizeof access);
     access.type = mem.type ? mem.type : dst.type;
     access.size = (u32)size;
     access.align = mem.align;
     access.mem = mem;
     NativeAddr da = nd_addr_materialize(d, nd_addr_storage(d, dst), &dt, mem);
     ND_REQUIRE_NATIVE(d, copy_bytes, "target does not copy bytes");
     d->native->copy_bytes(d->native, da, naddr, access);
     nd_addr_temps_release(d, &dt);
     nd_addr_temps_release(d, &temps);
     return;
   }
   /* The loaded value goes through the write-back cache (like a compute result)
    * rather than straight to the frame home: a load feeds an immediate consumer
    * far more often than not, so keeping it resident kills the store-then-reload
    * pair. nd_dst_reg avoids the address temps (still pinned here) and falls back
    * to a scratch + home store via nd_dst_writeback when dst is not cacheable. */
   NativeLoc reg = nd_dst_reg(d, dst);
   ND_REQUIRE_NATIVE(d, load, "target does not emit loads");
   d->native->load(d->native, reg, naddr, mem);
   nd_dst_writeback(d, dst, reg);
   /* A narrow integer load fills the whole register: a plain ldrb/ldrh/ldr-w
    * zero-extends, and (when the backend advertises load_sext and the access
    * is MF_SEXT_LOAD) a sign-extending ldrsb/ldrsh sign-extends. Either way a
    * subsequent matching widen (CV_ZEXT / CV_SEXT) is a no-op, so record which
    * one this load produced — register-resident only — so nd_convert can drop the
    * extend. (nd_dst_writeback just cleared both flags for this local.) */
   if (dst.kind == OPK_LOCAL && size < (u64)t->c->target.ptr_size &&
       nd_class_for_type(d, dst.type) == NATIVE_REG_INT &&
       reg.kind == NATIVE_LOC_REG) {
     NativeDirectLocal* l = nd_local(d, dst.v.local);
     if (l->reg == reg.v.reg) {
       if ((mem.flags & MF_SEXT_LOAD) && d->reg_info &&
           d->reg_info->load_sext)
         l->load_sext = 1;
       else
         l->load_zext = 1;
     }
   }
   nd_addr_temps_release(d, &temps);
 }
 
 static void nd_store(CgTarget* t, Operand addr, Operand src, MemAccess mem) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   u64 size;
   if (mem.bf_width != 0) {
     nd_bitfield_store(t, addr, src, bf_from_mem(mem));
     return;
   }
   size = mem.size ? mem.size : (mem.type ? cg_type_size(t->c, mem.type) : 0);
   /* No value-cache flush (2c, see nd_load): a store through a pointer cannot
    * alias a cached non-escaped local. The store target is foreign memory, so
    * there is no dst local entry to invalidate; SRC is read via
    * nd_materialize_operand. The arch barrier hook still fires. */
   if (mem.flags & MF_VOLATILE)
     nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_VOLATILE);
   NativeAddr naddr =
       nd_addr_materialize(d, nd_addr_storage(d, addr), &temps, mem);
   if (size > (u64)t->c->target.ptr_size) {
     NdAddrTemps st;
     AggregateAccess access;
     memset(&access, 0, sizeof access);
     access.type = mem.type ? mem.type : src.type;
     access.size = (u32)size;
     access.align = mem.align;
     access.mem = mem;
     NativeAddr sa = nd_addr_materialize(d, nd_addr_storage(d, src), &st, mem);
     ND_REQUIRE_NATIVE(d, copy_bytes, "target does not copy bytes");
     d->native->copy_bytes(d->native, naddr, sa, access);
     nd_addr_temps_release(d, &st);
     nd_addr_temps_release(d, &temps);
     return;
   }
   NativeLoc val = nd_materialize_operand(d, src);
   ND_REQUIRE_NATIVE(d, store, "target does not emit stores");
   d->native->store(d->native, naddr, val, mem);
   nd_release_materialized(d, val);
   nd_addr_temps_release(d, &temps);
   /* The store consumed SRC; if it was a dead transient, drop it (no write-back)
    * so it is not flush-stored at the next barrier (eager dead-operand drop). */
   nd_drop_killed_operand(d, src);
 }
 
 static void nd_addr_of(CgTarget* t, Operand dst, Operand lv) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   MemAccess mem = nd_scalar_mem(dst.type, d->base.c->target.ptr_size,
                                 d->base.c->target.ptr_align);
   NativeAddr naddr;
   /* Targeted: only an OPK_LOCAL lvalue escapes here — flush+mark just that
    * local (its home becomes the authoritative address source). An INDIRECT
    * lvalue's address is computed from base/index, which nd_addr_storage now
    * reads from the cache directly; a GLOBAL needs nothing. The dst home write
    * is handled by nd_store_operand_from_reg's invalidation. */
   if (lv.kind == OPK_LOCAL) {
     NativeDirectLocal* l = nd_local(d, lv.v.local);
     nd_flush_local(d, lv.v.local);
     l->address_taken = 1;
     l->flags |= CG_LOCAL_ADDR_TAKEN;
   }
   naddr = nd_addr_materialize(d, nd_addr_storage(d, lv), &temps, mem);
   /* Route the address through the write-back register cache (like nd_load):
    * load_addr writes straight into the dst local's cache register instead of a
    * scratch that is then copied to the home, killing the mov and keeping the
    * address resident for an immediate consumer (e.g. the base of the load it
    * feeds). nd_dst_reg falls back to a scratch + home store via nd_dst_writeback
    * when dst is not cacheable. The address temps are still pinned, and
    * nd_cache_alloc skips pinned regs, so the cache reg never aliases base/index. */
   NativeLoc reg = nd_dst_reg(d, dst);
   ND_REQUIRE_NATIVE(d, load_addr, "target does not materialize addresses");
   d->native->load_addr(d->native, reg, naddr);
   nd_dst_writeback(d, dst, reg);
   nd_addr_temps_release(d, &temps);
 }
 
 static void nd_tls_addr_of(CgTarget* t, Operand dst, ObjSymId sym, i64 addend) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc reg;
   nd_flush_all(d);
   reg = nd_dst_scratch(d, dst);
   ND_REQUIRE_NATIVE(d, tls_addr_of,
                     "target does not materialize TLS addresses");
   d->native->tls_addr_of(d->native, reg, sym, addend);
   nd_store_operand_from_reg(d, dst, reg);
   nd_release_materialized(d, reg);
 }
 
 static void nd_copy_bytes(CgTarget* t, Operand dst_addr, Operand src_addr,
                           AggregateAccess access) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps dt, st;
   NativeAddr dst;
   /* Pointer-target memory; addresses are cache-aware. No whole-cache flush. */
   dst = nd_addr_materialize(d, nd_addr_pointer(d, dst_addr), &dt, access.mem);
   NativeAddr src =
       nd_addr_materialize(d, nd_addr_pointer(d, src_addr), &st, access.mem);
   ND_REQUIRE_NATIVE(d, copy_bytes, "target does not copy bytes");
   d->native->copy_bytes(d->native, dst, src, access);
   nd_addr_temps_release(d, &st);
   nd_addr_temps_release(d, &dt);
 }
 
 static void nd_set_bytes(CgTarget* t, Operand dst_addr, Operand byte_value,
                          AggregateAccess access) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   NativeAddr dst;
   NativeLoc byte;
   /* Pointer-target memory; addresses are cache-aware. No whole-cache flush. */
   dst =
       nd_addr_materialize(d, nd_addr_pointer(d, dst_addr), &temps, access.mem);
   byte = nd_materialize_operand(d, byte_value);
   ND_REQUIRE_NATIVE(d, set_bytes, "target does not set bytes");
   d->native->set_bytes(d->native, dst, byte, access);
   nd_release_materialized(d, byte);
   nd_addr_temps_release(d, &temps);
 }
 
 static void nd_bitfield_load(CgTarget* t, Operand dst, Operand record_addr,
                              BitFieldAccess access) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   NativeAddr addr;
   NativeLoc reg;
   /* Record (pointer-target) memory; addresses are cache-aware. The dst home
    * write is handled by nd_store_operand_from_reg's invalidation. */
   addr = nd_addr_materialize(d, nd_addr_storage(d, record_addr), &temps,
                              access.storage);
   reg = nd_dst_scratch(d, dst);
   ND_REQUIRE_NATIVE(d, bitfield_load, "target does not load bitfields");
   d->native->bitfield_load(d->native, reg, addr, access);
   nd_store_operand_from_reg(d, dst, reg);
   nd_release_materialized(d, reg);
   nd_addr_temps_release(d, &temps);
 }
 
 static void nd_bitfield_store(CgTarget* t, Operand record_addr, Operand src,
                               BitFieldAccess access) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   NativeAddr addr;
   NativeLoc val;
   /* Record (pointer-target) memory; addresses are cache-aware, SRC reads the
    * cache. No whole-cache flush. */
   addr = nd_addr_materialize(d, nd_addr_storage(d, record_addr), &temps,
                              access.storage);
   val = nd_materialize_operand(d, src);
   ND_REQUIRE_NATIVE(d, bitfield_store, "target does not store bitfields");
   d->native->bitfield_store(d->native, addr, val, access);
   nd_release_materialized(d, val);
   nd_addr_temps_release(d, &temps);
 }
 
 /* Last line of defense against an unlowered split-scalar/soft-float op reaching
  * the machine backend. The cg-layer gates in src/cg/arith.c route split i64
  * mul/div/shift and all soft-double arith/convert/compare to runtime calls; if
  * one escapes, the native backend would silently emit wrong code. */
 static int nd_is_split_wide8_scalar(NativeDirectTarget* d, KitCgTypeId ty) {
   return abi_cg_scalar_split_lane_size(d->base.c->abi, ty) == 4u &&
          native_type_size(d->native, ty) == 8u;
 }
 
 static int nd_is_wide64_int(NativeDirectTarget* d, KitCgTypeId ty) {
   if (!nd_is_split_wide8_scalar(d, ty)) return 0;
   if (kit_cg_type_int_width((KitCompiler*)d->base.c, ty) == 0) return 0;
   return 1;
 }
 
 static int nd_is_soft_double(NativeDirectTarget* d, KitCgTypeId ty) {
   if (!nd_is_split_wide8_scalar(d, ty)) return 0;
   return kit_cg_type_float_width((KitCompiler*)d->base.c, ty) == 64;
 }
 
 static void nd_binop(CgTarget* t, BinOp op, Operand dst, Operand a, Operand b) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc ar;
   NativeLoc br;
   NativeLoc dr;
   /* No split-lane 8-byte value reaches a single-register op: the cg layer
    * lowers i64 add/sub/and/or/xor to inline 2-word lane sequences and
    * mul/div/rem/shift to __*di3 runtime calls (src/cg/arith.c). Anything that
    * slips through here would silently compute only the low word, so fail
    * loudly instead. */
   if (nd_is_wide64_int(d, a.type) || nd_is_wide64_int(d, dst.type)) {
     nd_panic(
         d,
         "64-bit integer arithmetic reached the backend un-lowered "
         "(cg should emit a 2-word lane sequence or a __*di3 runtime call)");
   }
   if (nd_is_soft_double(d, a.type) || nd_is_soft_double(d, dst.type)) {
     nd_panic(d,
              "soft-float double arithmetic reached the backend un-lowered "
              "(should be a __*df3 runtime call)");
   }
   ar = nd_materialize_operand(d, a);
   br = nd_rhs_imm_or_reg(d, NATIVE_IMM_BINOP, (u32)op, b);
   dr = nd_dst_reg(d, dst);
   /* §E.3: cross the post-materialization operands as 16 B NativeRegLoc when the
    * backend installs the narrow hook (every native arch does); fall back to the
    * fat 48 B hook otherwise. Byte-identical — the arch reconstructs the same
    * NativeLoc. */
   if (d->native->binop_rr)
     d->native->binop_rr(d->native, op, native_reg_loc_of(dr),
                         native_reg_loc_of(ar), native_reg_loc_of(br));
   else {
     ND_REQUIRE_NATIVE(d, binop, "target does not emit binary ops");
     d->native->binop(d->native, op, dr, ar, br);
   }
   nd_dst_writeback(d, dst, dr);
   nd_release_materialized(d, br);
   nd_release_materialized(d, ar);
   /* Eager dead-operand drop: a/b flagged dead by the cg layer (OPK_FLAG_KILL) are
    * consumed now — drop their cache entries without a write-back so they are not
    * flush-stored at the next barrier and their registers free immediately. */
   nd_drop_killed_operand(d, b);
   nd_drop_killed_operand(d, a);
 }
 
 static void nd_unop(CgTarget* t, UnOp op, Operand dst, Operand a) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc ar;
   NativeLoc dr;
   /* i64 neg/bnot stay inline as register pairs, and soft-double FNEG stays
    * inline as a high-word sign-bit flip (v1), so both are allowlisted. Any
    * OTHER soft-double unop reaching the backend is an unlowered escape. */
   if (nd_is_wide64_int(d, a.type) || nd_is_wide64_int(d, dst.type)) {
     nd_panic(d,
              "64-bit integer unary op reached the backend un-lowered "
              "(cg should emit a 2-word lane sequence)");
   }
   if (op != UO_FNEG &&
       (nd_is_soft_double(d, a.type) || nd_is_soft_double(d, dst.type))) {
     nd_panic(d, "soft-float double unary op reached the backend un-lowered");
   }
   ar = nd_materialize_operand(d, a);
   dr = nd_dst_reg(d, dst);
   ND_REQUIRE_NATIVE(d, unop, "target does not emit unary ops");
   d->native->unop(d->native, op, dr, ar);
   nd_dst_writeback(d, dst, dr);
   nd_release_materialized(d, ar);
   nd_drop_killed_operand(d, a);
 }
 
 static void nd_cmp(CgTarget* t, CmpOp op, Operand dst, Operand a, Operand b) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc ar;
   NativeLoc br;
   NativeLoc dr;
   /* i64 compares are lowered to inline 2-word lane sequences and soft-double
    * compares to __*df2 runtime calls (src/cg/arith.c); neither reaches a single
    * GPR compare here. */
   if (nd_is_wide64_int(d, a.type) || nd_is_wide64_int(d, b.type)) {
     nd_panic(d,
              "64-bit integer compare reached the backend un-lowered "
              "(cg should emit a 2-word lane sequence)");
   }
   if (nd_is_soft_double(d, a.type) || nd_is_soft_double(d, b.type)) {
     nd_panic(d,
              "soft-float double compare reached the backend un-lowered "
              "(should be a __*df2 runtime call)");
   }
   ar = nd_materialize_operand(d, a);
   br = nd_rhs_imm_or_reg(d, NATIVE_IMM_CMP, (u32)op, b);
   dr = nd_dst_reg(d, dst);
   if (d->native->cmp_rr)
     d->native->cmp_rr(d->native, op, native_reg_loc_of(dr),
                       native_reg_loc_of(ar), native_reg_loc_of(br));
   else {
     ND_REQUIRE_NATIVE(d, cmp, "target does not emit compares");
     d->native->cmp(d->native, op, dr, ar, br);
   }
   nd_dst_writeback(d, dst, dr);
   nd_release_materialized(d, br);
   nd_release_materialized(d, ar);
   nd_drop_killed_operand(d, b);
   nd_drop_killed_operand(d, a);
 }
 
 static void nd_convert(CgTarget* t, ConvKind op, Operand dst, Operand src) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc sr;
   NativeLoc dr;
   /* i64<->i32 sext/zext/trunc are lowered to inline lane ops (src/cg/arith.c
    * api_try_wide8_convert) and i64<->float / soft-double conversions to runtime
    * calls; none reaches a single-register convert here. */
   if (nd_is_wide64_int(d, src.type) || nd_is_wide64_int(d, dst.type)) {
     nd_panic(d,
              "64-bit integer conversion reached the backend un-lowered "
              "(cg should emit a 2-word lane sequence or a runtime call)");
   }
   if (nd_is_soft_double(d, src.type) || nd_is_soft_double(d, dst.type)) {
     nd_panic(d,
              "soft-float double conversion reached the backend un-lowered "
              "(should be a runtime call)");
   }
   /* Coalesce a convert whose source is a dead, register-resident transient of
    * the same register class: rename the source's register to the destination
    * and convert it in place (dr == sr). Saves the source materialize-into-a-
    * pinned-reg + a separate destination scratch, and lets a width-preserving
    * convert (e.g. a zext/sext/bitcast that is a no-op once dst==src reg)
    * collapse to nothing in the arch move/convert path. A class-crossing convert
    * (GPR<->FPR fmov) is excluded by nd_rename_killed_to_dst's same-class guard
    * and falls through below. */
   /* A CV_ZEXT (resp. CV_SEXT) whose source came straight from a narrow integer
    * load is a no-op: the load already zero-extended (resp. sign-extended, via an
    * MF_SEXT_LOAD ldrsb/ldrsh that filled the whole register) the value, so the
    * widen is redundant for any wider destination. Detect it before the rename,
    * which transfers (and would clear) the source's flag. */
   int ext_noop = 0;
   if (src.kind == OPK_LOCAL && (op == CV_ZEXT || op == CV_SEXT)) {
     NativeDirectLocal* sl = nd_local(d, src.v.local);
     u8 has = op == CV_ZEXT ? sl->load_zext : sl->load_sext;
     ext_noop = sl->reg != REG_NONE && has && src.type == sl->type;
   }
   {
     Reg r = nd_rename_killed_to_dst(d, dst, src);
     if (r != REG_NONE) {
       NativeAllocClass cls = nd_class_for_type(d, dst.type);
       sr = nd_loc_reg(d, src.type, cls, r);
       dr = nd_loc_reg(d, dst.type, cls, r);
       /* Renaming the source register to the destination already placed the
        * extended value where the result wants it — skip the redundant extend;
        * the upper bits are provably the correct zero/sign fill. */
       if (!ext_noop) {
         if (d->native->convert_rr)
           d->native->convert_rr(d->native, op, native_reg_loc_of(dr),
                                 native_reg_loc_of(sr));
         else {
           ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
           d->native->convert(d->native, op, dr, sr);
         }
       }
       nd_dst_writeback(d, dst, dr);
       return;
     }
   }
   sr = nd_materialize_operand(d, src);
   dr = nd_dst_reg(d, dst);
   if (d->native->convert_rr)
     d->native->convert_rr(d->native, op, native_reg_loc_of(dr),
                           native_reg_loc_of(sr));
   else {
     ND_REQUIRE_NATIVE(d, convert, "target does not emit converts");
     d->native->convert(d->native, op, dr, sr);
   }
   nd_dst_writeback(d, dst, dr);
   nd_release_materialized(d, sr);
 }
 
 static void nd_call(CgTarget* t, const CGCallDesc* desc) {
   NativeDirectTarget* d = nd_of(t);
   NativeCallPhase plan;
   NativeCallDesc nd;
   NativeLoc* args;
   NativeLoc* results;
   NativeLoc callee_tmp;
   int release_callee_tmp = 0;
   u32 nresults = desc->result != CG_LOCAL_NONE ? 1u : 0u;
   memset(&plan, 0, sizeof plan);
   memset(&nd, 0, sizeof nd);
   memset(&callee_tmp, 0, sizeof callee_tmp);
   args = nd_loc_buf(d, d->argbuf, ND_ARG_BUF, desc->nargs);
   results = nd_loc_buf(d, d->retbuf, ND_RET_BUF, nresults);
   /* Source each argument cache-aware BEFORE spilling. A cached arg that is dead
    * after the call (arg_dead_mask) flows from its live register straight into
    * the ABI arg register — the backend's parallel-copy scheduler
    * (native_arg_shuffle) resolves reg<->arg-reg conflicts — avoiding the
    * spill-to-home + reload round-trip the old flush-all forced. Pin its register
    * so the callee materialize below cannot evict it as a scratch victim. A live
    * (or uncached) arg keeps the home source and is spilled by the flush below. */
   for (u32 i = 0; i < desc->nargs; ++i) {
     NativeDirectLocal* l = nd_local(d, desc->args[i]);
     int dead = i < 64u && ((desc->arg_dead_mask >> i) & 1u);
     if (dead && l->reg != REG_NONE) {
       args[i] = nd_loc_reg(d, l->type, (NativeAllocClass)l->cls, l->reg);
       d->scratch_used[l->cls] |= 1u << l->reg;
     } else {
       args[i] = nd_loc_frame(d, desc->args[i], 0);
     }
   }
   /* An indirect callee is read from its home; make it authoritative before the
    * selective flush (the cg only ever produces an OPK_LOCAL or OPK_GLOBAL
    * callee). */
   if (desc->callee.kind == OPK_LOCAL) nd_flush_local(d, desc->callee.v.local);
   /* Spill the live-across set: everything cached except the kept (dead) args. */
   nd_flush_all_except_kept_args(d, desc);
   /* Place the scalar result. With ndt_result_reg_stable (aa64/rv64) the result
    * is cached directly in the ABI result register after the call — no mov, no
    * home store; feed the plan a placeholder dst (never applied, the post-call
    * step always skips rets[0]) so no home slot is allocated for a result
    * consumed before the next flush. Without it (x86-64: RAX is an implicit
    * div/mul operand, so a result left there is clobbered before its consumer)
    * claim a general cache register now and let the post-call move write the
    * result into it. A non-cacheable result (aggregate / sret) keeps the home. */
   int result_in_abi_reg = 0;
   if (nresults) {
     NativeDirectLocal* rl = nd_local(d, desc->result);
     if (!nd_local_cacheable(d, rl)) {
       results[0] = nd_loc_frame(d, desc->result, 0);
     } else if (d->reg_info && d->reg_info->ndt_result_reg_stable) {
       result_in_abi_reg = 1;
       results[0] = nd_loc_reg(d, rl->type, (NativeAllocClass)rl->cls, 0);
     } else {
       Reg r = nd_cache_alloc(d, (NativeAllocClass)rl->cls);
       if (r != REG_NONE) {
         d->reg_owner[rl->cls][r] = desc->result;
         rl->reg = r;
         rl->dirty = 1;
         nd_cache_link(d, desc->result);
         /* Pin until the post-call move so the callee materialize cannot evict
          * this not-yet-valid entry as a scratch victim. */
         d->scratch_used[rl->cls] |= 1u << r;
         results[0] = nd_loc_reg(d, rl->type, (NativeAllocClass)rl->cls, r);
       } else {
         results[0] = nd_loc_frame(d, desc->result, 0);
       }
     }
   }
   nd_barrier(d, NATIVE_DIRECT_BARRIER_CALL | NATIVE_DIRECT_BARRIER_MEMORY);
   nd.fn_type = desc->fn_type;
   nd.callee = nd_loc_operand(d, desc->callee);
   if (nd.callee.kind == NATIVE_LOC_FRAME) {
     callee_tmp = nd_materialize_loc(
         d, nd.callee, (NativeAllocClass)nd.callee.cls, nd.callee.type);
     nd.callee = callee_tmp;
     release_callee_tmp = 1;
   }
   nd.args = args;
   nd.results = results;
   nd.nargs = desc->nargs;
   nd.nresults = nresults;
   nd.flags = desc->flags;
   nd.tail_policy = desc->tail_policy;
   nd.inline_policy = desc->inline_policy;
 
   if (d->ops && d->ops->marshal_call)
     d->ops->marshal_call(d, &nd, &plan);
   else {
     ND_REQUIRE_NATIVE(d, marshal_call, "target does not marshal calls");
     d->native->marshal_call(d->native, &nd, &plan);
   }
   if (plan.stack_arg_size > d->max_outgoing)
     d->max_outgoing = plan.stack_arg_size;
   for (u32 i = 0; i < plan.nargs; ++i)
     nd_write_loc(d, plan.args[i].dst, plan.args[i].src, plan.args[i].mem);
   if (d->ops && d->ops->emit_call)
     d->ops->emit_call(d, &plan);
   else {
     ND_REQUIRE_NATIVE(d, emit_call, "target does not emit calls");
     d->native->emit_call(d->native, &plan);
   }
   /* The call clobbered the caller-saved registers, so the (dead-after) args'
    * cached values are stale: unpin and drop their entries without writing back.
    * args[i] is a reg only for args that were sourced from the cache. */
   for (u32 i = 0; i < desc->nargs; ++i) {
     if (args[i].kind == NATIVE_LOC_REG)
       d->scratch_used[args[i].cls] &= ~(1u << args[i].v.reg);
     nd_invalidate_local(d, desc->args[i]);
   }
   /* Stable-result-register path: cache the result directly in the ABI result
    * register the call left it in (plan.rets[0].src) and skip its home store.
    * That register may now be an ordinary cache-pool member (x0 on aa64 is -O0
    * allocable), but it is free here regardless: the live-across flush + the
    * dead-arg invalidation above cleared it, and the guard below evicts any
    * residual owner before claiming it. The next call's flush spills it if still
    * live, or drops it if dead. */
   u32 ret_start = 0;
   if (result_in_abi_reg) {
     NativeDirectLocal* rl = nd_local(d, desc->result);
     NativeLoc src = plan.rets[0].src;
     if (plan.nrets != 1u || src.kind != NATIVE_LOC_REG ||
         (NativeAllocClass)src.cls != (NativeAllocClass)rl->cls)
       nd_panic(d, "cacheable scalar result not returned in one register");
     if (d->reg_owner[rl->cls][src.v.reg] != CG_LOCAL_NONE)
       nd_invalidate_local(d, d->reg_owner[rl->cls][src.v.reg]);
     d->reg_owner[rl->cls][src.v.reg] = desc->result;
     rl->reg = src.v.reg;
     rl->dirty = 1;
     nd_cache_link(d, desc->result);
     nd_touch_local(d, rl);
     ret_start = 1u;
   }
   for (u32 i = ret_start; i < plan.nrets; ++i)
     nd_write_loc(d, plan.rets[i].dst, plan.rets[i].src, plan.rets[i].mem);
   /* Non-stable path: the result was pre-claimed in a general cache register and
    * the move above wrote the ABI reg into it; unpin it so it behaves as an
    * ordinary write-back entry. */
   if (nresults && !result_in_abi_reg && results[0].kind == NATIVE_LOC_REG)
     d->scratch_used[results[0].cls] &= ~(1u << results[0].v.reg);
   if (release_callee_tmp)
     nd_scratch_release(d, (NativeAllocClass)callee_tmp.cls, callee_tmp.v.reg);
 }
 
 static const char* nd_tail_call_unrealizable_reason(CgTarget* t,
                                                     const CGCallDesc* desc) {
   NativeDirectTarget* d = nd_of(t);
   if (d->ops && d->ops->tail_call_unrealizable_reason)
     return d->ops->tail_call_unrealizable_reason(d, desc);
   return "target does not expose direct tail-call lowering";
 }
 
 static void nd_ret(CgTarget* t, CGLocal value) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc loc;
   const NativeLoc* locp = NULL;
   NativeCallPhaseRet* rets = NULL;
   u32 nrets = 0;
   /* The emit_ret ops path reads `value` from its home, so it needs the cache
    * spilled first (no production arch installs it today). */
   if (d->ops && d->ops->emit_ret) {
     nd_flush_all(d);
     if (value != CG_LOCAL_NONE) nd_home(d, value); /* arch reads value from home */
     d->ops->emit_ret(d, value);
     return;
   }
   /* Default path: at a return every cached value is dead (the frame is about to
    * be torn down), so source the return value straight from its live register
    * when cached and drop the cache without spilling. This removes the
    * per-function `stur <result>,[home]; ldur x0,[home]` round-trip and the dead
    * writebacks the old flush-all emitted. A non-cached value (multi-part /
    * aggregate / memory-required) is already authoritative in its home. */
   if (value != CG_LOCAL_NONE) {
     NativeDirectLocal* l = nd_local(d, value);
     loc = l->reg != REG_NONE
               ? nd_loc_reg(d, l->type, (NativeAllocClass)l->cls, l->reg)
               : nd_loc_frame(d, value, 0);
     locp = &loc;
   }
   nd_drop_all(d);
   ND_REQUIRE_NATIVE(d, marshal_ret, "target does not marshal returns");
   d->native->marshal_ret(d->native, d->func, locp, &rets, &nrets);
   for (u32 i = 0; i < nrets; ++i)
     nd_write_loc(d, rets[i].dst, rets[i].src, rets[i].mem);
   ND_REQUIRE_NATIVE(d, ret, "target does not emit returns");
   d->native->ret(d->native);
 }
 
 static void nd_unreachable(CgTarget* t) {
   NativeDirectTarget* d = nd_of(t);
   nd_flush_all(d);
   ND_REQUIRE_NATIVE(d, trap, "target does not emit traps");
   d->native->trap(d->native);
 }
 
 static void nd_alloca(CgTarget* t, Operand dst, Operand size, u32 align) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc sr, dr;
   nd_flush_all(d);
   sr = nd_materialize_operand(d, size);
   dr = nd_dst_scratch(d, dst);
   ND_REQUIRE_NATIVE(d, alloca_, "target does not emit alloca");
   d->native->alloca_(d->native, dr, sr, align);
   nd_store_operand_from_reg(d, dst, dr);
   nd_release_materialized(d, dr);
   nd_release_materialized(d, sr);
 }
 
 static void nd_va_start(CgTarget* t, Operand ap_addr) {
   NativeDirectTarget* d = nd_of(t);
   nd_flush_all(d);
   /* The arch va_* / asm helpers read operand locals' homes directly; with lazy
    * homing a never-spilled transient operand (or a fresh store-target temp) has
    * no home yet, so materialize one before crossing into arch code. */
   nd_home_operand(d, ap_addr);
   if (!d->ops || !d->ops->va_start_)
     nd_panic(d, "target does not emit va_start");
   d->ops->va_start_(d, ap_addr);
 }
 
 static void nd_va_arg(CgTarget* t, Operand dst, Operand ap_addr,
                       KitCgTypeId type) {
   NativeDirectTarget* d = nd_of(t);
   nd_flush_all(d);
   nd_home_operand(d, dst); /* fresh store-target temp: home it before arch reads it */
   nd_home_operand(d, ap_addr);
   if (!d->ops || !d->ops->va_arg_) nd_panic(d, "target does not emit va_arg");
   d->ops->va_arg_(d, dst, ap_addr, type);
 }
 
 static void nd_va_end(CgTarget* t, Operand ap_addr) {
   NativeDirectTarget* d = nd_of(t);
   nd_flush_all(d);
   nd_home_operand(d, ap_addr);
   if (!d->ops || !d->ops->va_end_) nd_panic(d, "target does not emit va_end");
   d->ops->va_end_(d, ap_addr);
 }
 
 static void nd_va_copy(CgTarget* t, Operand dst_ap_addr, Operand src_ap_addr) {
   NativeDirectTarget* d = nd_of(t);
   nd_flush_all(d);
   nd_home_operand(d, dst_ap_addr);
   nd_home_operand(d, src_ap_addr);
   if (!d->ops || !d->ops->va_copy_) nd_panic(d, "target does not emit va_copy");
   d->ops->va_copy_(d, dst_ap_addr, src_ap_addr);
 }
 
 static void nd_atomic_load(CgTarget* t, Operand dst, Operand addr,
                            MemAccess mem, KitCgMemOrder order) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   /* No value-cache flush (2c): the atomic accesses foreign memory, which cannot
    * alias a cached non-escaped local; the arch barrier hook still orders it. */
   nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_ATOMIC);
   NativeAddr naddr =
       nd_addr_materialize(d, nd_addr_pointer(d, addr), &temps, mem);
   NativeLoc dr = nd_dst_scratch(d, dst);
   ND_REQUIRE_NATIVE(d, atomic_load, "target does not emit atomic loads");
   d->native->atomic_load(d->native, dr, naddr, mem, order);
   nd_store_operand_from_reg(d, dst, dr);
   nd_release_materialized(d, dr);
   nd_addr_temps_release(d, &temps);
 }
 
 static void nd_atomic_store(CgTarget* t, Operand addr, Operand src,
                             MemAccess mem, KitCgMemOrder order) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   /* No value-cache flush (2c): foreign atomic memory cannot alias a cached
    * non-escaped local. */
   nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_ATOMIC);
   NativeAddr naddr =
       nd_addr_materialize(d, nd_addr_pointer(d, addr), &temps, mem);
   NativeLoc sr = nd_materialize_operand(d, src);
   ND_REQUIRE_NATIVE(d, atomic_store, "target does not emit atomic stores");
   d->native->atomic_store(d->native, naddr, sr, mem, order);
   nd_release_materialized(d, sr);
   nd_addr_temps_release(d, &temps);
 }
 
 static void nd_atomic_rmw(CgTarget* t, KitCgAtomicOp op, Operand dst,
                           Operand addr, Operand val, MemAccess mem,
                           KitCgMemOrder order) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   /* No value-cache flush (2c): foreign atomic memory cannot alias a cached
    * non-escaped local. */
   nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_ATOMIC);
   NativeAddr naddr =
       nd_addr_materialize(d, nd_addr_pointer(d, addr), &temps, mem);
   NativeLoc vr = nd_materialize_operand(d, val);
   NativeLoc dr = nd_dst_scratch(d, dst);
   ND_REQUIRE_NATIVE(d, atomic_rmw, "target does not emit atomic rmw");
   d->native->atomic_rmw(d->native, op, dr, naddr, vr, mem, order);
   nd_store_operand_from_reg(d, dst, dr);
   nd_release_materialized(d, dr);
   nd_release_materialized(d, vr);
   nd_addr_temps_release(d, &temps);
 }
 
 static void nd_atomic_cas(CgTarget* t, Operand prior, Operand ok, Operand addr,
                           Operand expected, Operand desired, MemAccess mem,
                           KitCgMemOrder success, KitCgMemOrder failure) {
   NativeDirectTarget* d = nd_of(t);
   NdAddrTemps temps;
   /* No value-cache flush (2c): foreign atomic memory cannot alias a cached
    * non-escaped local. */
   nd_barrier(d, NATIVE_DIRECT_BARRIER_MEMORY | NATIVE_DIRECT_BARRIER_ATOMIC);
   NativeAddr naddr =
       nd_addr_materialize(d, nd_addr_pointer(d, addr), &temps, mem);
   NativeLoc er = nd_materialize_operand(d, expected);
   NativeLoc dr = nd_materialize_operand(d, desired);
   NativeLoc pr = nd_dst_scratch(d, prior);
   NativeLoc kr = nd_dst_scratch(d, ok);
   ND_REQUIRE_NATIVE(d, atomic_cas,
                     "target does not emit atomic compare-exchange");
   d->native->atomic_cas(d->native, pr, kr, naddr, er, dr, mem, success,
                         failure);
   nd_store_operand_from_reg(d, prior, pr);
   nd_store_operand_from_reg(d, ok, kr);
   nd_release_materialized(d, kr);
   nd_release_materialized(d, pr);
   nd_release_materialized(d, dr);
   nd_release_materialized(d, er);
   nd_addr_temps_release(d, &temps);
 }
 
 static void nd_fence(CgTarget* t, KitCgMemOrder order) {
   NativeDirectTarget* d = nd_of(t);
   /* No value-cache flush (2c): a fence orders memory visible to other agents; a
    * cached local is non-escaped, so no other agent can observe its deferred home
    * write. The arch fence instruction is still emitted. */
   ND_REQUIRE_NATIVE(d, fence, "target does not emit fences");
   d->native->fence(d->native, order);
 }
 
 static void nd_intrinsic(CgTarget* t, IntrinKind kind, Operand* dsts, u32 ndst,
                          const Operand* args, u32 narg) {
   NativeDirectTarget* d = nd_of(t);
   NativeLoc* ndsts = nd_loc_buf(d, d->retbuf, ND_RET_BUF, ndst);
   NativeLoc* nargs = nd_loc_buf(d, d->argbuf, ND_ARG_BUF, narg);
   nd_flush_all(d);
   ND_REQUIRE_NATIVE(d, intrinsic, "target does not emit compiler intrinsics");
   for (u32 i = 0; i < ndst; ++i) ndsts[i] = nd_dst_scratch(d, dsts[i]);
   for (u32 i = 0; i < narg; ++i) {
     nargs[i] = args[i].kind == OPK_IMM ? nd_loc_operand(d, args[i])
                                        : nd_materialize_operand(d, args[i]);
   }
   d->native->intrinsic(d->native, kind, ndsts, ndst, nargs, narg);
   for (u32 i = 0; i < ndst; ++i) {
     nd_store_operand_from_reg(d, dsts[i], ndsts[i]);
     nd_release_materialized(d, ndsts[i]);
   }
   for (u32 i = 0; i < narg; ++i) nd_release_materialized(d, nargs[i]);
 }
 
 static void nd_asm_block(CgTarget* t, const char* tmpl,
                          const AsmConstraint* outs, u32 nout, Operand* out_ops,
                          const AsmConstraint* ins, u32 nin,
                          const Operand* in_ops, const Sym* clobbers, u32 nclob,
                          u32 clobber_abi_sets) {
   NativeDirectTarget* d = nd_of(t);
   nd_flush_all(d);
   nd_barrier(d,
              NATIVE_DIRECT_BARRIER_INLINE_ASM | NATIVE_DIRECT_BARRIER_MEMORY);
   /* Outputs (incl. fresh reg-constraint store-targets) and inputs are bound by
    * the arch helper reading their homes directly; materialize lazy homes now. */
   for (u32 i = 0; i < nout; ++i) nd_home_operand(d, out_ops[i]);
   for (u32 i = 0; i < nin; ++i) nd_home_operand(d, in_ops[i]);
   if (d->ops && d->ops->asm_block) {
     d->ops->asm_block(d, tmpl, outs, nout, out_ops, ins, nin, in_ops, clobbers,
                       nclob, clobber_abi_sets);
     return;
   }
   nd_panic(d, "target does not emit inline asm");
 }
 
 static int nd_asm_is_reg_constraint(CgTarget* t, const char* constraint) {
   NativeDirectTarget* d = nd_of(t);
   return native_asm_constraint_is_reg(d->native, constraint);
 }
 
 static void nd_file_scope_asm(CgTarget* t, const char* src, size_t len) {
   NativeDirectTarget* d = nd_of(t);
   ND_REQUIRE_NATIVE(d, file_scope_asm, "target does not emit file-scope asm");
   d->native->file_scope_asm(d->native, src, len);
 }
 
 static void nd_set_loc(CgTarget* t, SrcLoc loc) {
   NativeDirectTarget* d = nd_of(t);
   d->loc = loc;
   if (d->native && d->native->set_loc) d->native->set_loc(d->native, loc);
 }
 
 static void nd_finalize(CgTarget* t) {
   NativeDirectTarget* d = nd_of(t);
   if (d->native && d->native->finalize) d->native->finalize(d->native);
 }
 
 static void nd_destroy(CgTarget* t) {
   NativeDirectTarget* d = nd_of(t);
   if (d->native && d->native->destroy) d->native->destroy(d->native);
 }
 
 CgTarget* native_direct_target_new(Compiler* c, ObjBuilder* obj,
                                    const NativeDirectTargetConfig* cfg) {
   NativeDirectTarget* d;
   if (!c || !cfg || !cfg->native)
     compiler_panic(c, (SrcLoc){0, 0, 0},
                    "native_direct_target_new: missing native target");
   d = arena_znew(c->tu, NativeDirectTarget);
   if (!d) return NULL;
   d->base.c = c;
   d->base.obj = obj;
   d->magic = NATIVE_DIRECT_MAGIC;
   d->native = cfg->native;
   d->ops = cfg->ops;
   d->user = cfg->user;
 
   /* Resolve register/class info once; it is constant for the program. */
   d->reg_info = cfg->native ? cfg->native->regs : NULL;
   native_reg_info_validate(c, d->reg_info);
   for (u32 i = 0; i < NATIVE_REG_CLASS_COUNT; ++i) d->class_info[i] = NULL;
   if (d->reg_info) {
     const NativeRegInfo* ri = d->reg_info;
     for (u32 i = 0; i < ri->nclasses; ++i) {
       u32 cls = ri->classes[i].cls;
       if (cls < NATIVE_REG_CLASS_COUNT) d->class_info[cls] = &ri->classes[i];
     }
   }
 
   /* Precompute, per class, the -O0 value-cache register pool (in allocable
    * order), constant for the program. On ndt_caller_saved_only targets the pool
    * is the caller-saved subset of allocable (the cache then never forces a
    * callee-save spill into the deferred prologue); otherwise it is the full
    * allocable list, with any callee-save in it reported to the backend on use
    * (nd_note_reg_used) and reserved by reserve_callee_saves. */
   {
     int caller_only = d->reg_info && d->reg_info->ndt_caller_saved_only;
     for (u32 cls = 0; cls < NATIVE_REG_CLASS_COUNT; ++cls) {
       const NativeAllocClassInfo* ci = d->class_info[cls];
       u32 mask, n = 0;
       if (!ci) continue;
       mask = nd_caller_saved_mask(d, (NativeAllocClass)cls);
       for (u32 i = 0; i < ci->ndt_allocable_count; ++i) {
         Reg r = ci->ndt_allocable[i];
         if (r >= 32u) continue;
         if (caller_only && !(mask & (1u << r))) continue;
         d->cache_pool[cls][n++] = r;
       }
       d->ncache_pool[cls] = n;
     }
   }
 
   d->base.func_begin = nd_func_begin;
   d->base.func_end = nd_func_end;
   d->base.alias = nd_alias;
   d->base.local = nd_local_new;
   d->base.local_addr = nd_local_addr;
   d->base.param = nd_param;
   d->base.local_debug_loc = nd_local_debug_loc;
   d->base.reclaim_temps = nd_reclaim_temps;
   d->base.label_new = nd_label_new;
   d->base.label_place = nd_label_place;
   d->base.jump = nd_jump;
   d->base.cmp_branch = nd_cmp_branch;
   d->base.switch_ = nd_switch;
   d->base.indirect_branch = nd_indirect_branch;
   d->base.load_label_addr = nd_load_label_addr;
   d->base.local_static_data_begin = nd_local_static_data_begin;
   d->base.local_static_data_write = nd_local_static_data_write;
   d->base.local_static_data_label_addr = nd_local_static_data_label_addr;
   d->base.local_static_data_end = nd_local_static_data_end;
   d->base.data_label_addr_unsupported_msg = nd_data_label_addr_unsupported_msg;
   d->base.scope_begin = nd_scope_begin;
   d->base.scope_end = nd_scope_end;
   d->base.break_to = nd_break_to;
   d->base.continue_to = nd_continue_to;
   d->base.load_imm = nd_load_imm;
   d->base.load_const = nd_load_const;
   d->base.copy = nd_copy;
   d->base.load = nd_load;
   d->base.store = nd_store;
   d->base.addr_of = nd_addr_of;
   d->base.tls_addr_of = nd_tls_addr_of;
   d->base.copy_bytes = nd_copy_bytes;
   d->base.set_bytes = nd_set_bytes;
   d->base.binop = nd_binop;
   d->base.unop = nd_unop;
   d->base.cmp = nd_cmp;
   d->base.convert = nd_convert;
   d->base.untyped_values = 1; /* register-resident values: no-op bitcasts elide */
   d->base.call = nd_call;
   d->base.tail_call_unrealizable_reason = nd_tail_call_unrealizable_reason;
   d->base.ret = nd_ret;
   d->base.unreachable = nd_unreachable;
   d->base.alloca_ = nd_alloca;
   d->base.va_start_ = nd_va_start;
   d->base.va_arg_ = nd_va_arg;
   d->base.va_end_ = nd_va_end;
   d->base.va_copy_ = nd_va_copy;
   d->base.atomic_load = nd_atomic_load;
   d->base.atomic_store = nd_atomic_store;
   d->base.atomic_rmw = nd_atomic_rmw;
   d->base.atomic_cas = nd_atomic_cas;
   d->base.fence = nd_fence;
   d->base.intrinsic = nd_intrinsic;
   d->base.asm_is_reg_constraint = nd_asm_is_reg_constraint;
   d->base.asm_block = nd_asm_block;
   d->base.file_scope_asm = nd_file_scope_asm;
   d->base.set_loc = nd_set_loc;
   d->base.finalize = nd_finalize;
   d->base.destroy = nd_destroy;
   return &d->base;
 }
 
 NativeTarget* native_direct_target_native(CgTarget* t) {
   NativeDirectTarget* d = t ? nd_of(t) : NULL;
   return d && d->magic == NATIVE_DIRECT_MAGIC ? d->native : NULL;
 }
 
 CgTarget* native_direct_backend_make(Compiler* c, ObjBuilder* o,
                                      const KitCodeOptions* opts,
                                      NativeTargetCtor ctor,
                                      const NativeOps* ops) {
   MCEmitter* mc = NULL;
   Debug* debug = NULL;
   CgTarget* t;
   NativeTarget* native;
   NativeDirectTargetConfig cfg;
   if (cg_mc_debug_new(c, o, opts, &mc, &debug) != KIT_OK) return NULL;
   native = ctor(c, o, mc);
   if (!native) return NULL;
   if (opts) native->disabled_backend_features = opts->disabled_backend_features;
   memset(&cfg, 0, sizeof cfg);
   cfg.native = native;
   cfg.ops = ops;
   t = native_direct_target_new(c, o, &cfg);
   if (t) t->debug = debug;
   return t;
 }
 
 CgTarget* native_direct_semantic_target_new(Compiler* c, ObjBuilder* o,
                                             MCEmitter* mc,
                                             NativeTargetCtor ctor,
                                             const NativeOps* ops) {
   NativeTarget* native;
   NativeDirectTargetConfig cfg;
   if (!mc) mc = mc_new(c, o);
   native = ctor(c, o, mc);
   if (!native) return NULL;
   memset(&cfg, 0, sizeof cfg);
   cfg.native = native;
   cfg.ops = ops;
   return native_direct_target_new(c, o, &cfg);
 }