kit

lower.c (18046B)

---

 /* Func (post opt_run_o1_interp) -> InterpFunc bytecode loader.
  *
  * Walks blocks in f->emit_order, emits one fixed-width InterpInsn per non-no-op
  * Inst, bump-allocates frame-slot offsets, resolves branch targets from block
  * ids to code pcs, and collects the global symbols the function references (so
  * their names survive past the obj's lifetime). Unsupported ops are lowered to
  * IOP_TRAP and the function is flagged rejected with a reason — the engine then
  * reports a clean "interp: <op> not supported" rather than miscompiling. */
 
 #include <string.h>
 
 #include "abi/abi.h"
 /* cg/ir.h must precede any header that pulls opt/ir.h: opt/ir.h aliases
  * `Operand`/`CGCallDesc`/... to their Opt* forms via macros, so the semantic
  * cg structs (CgIrLocalStatic*Aux, reused verbatim as the opt aux pointers)
  * have to be parsed first. This mirrors opt/opt.h's include order. */
 #include "cg/cgtarget.h"
 #include "cg/ir.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/core.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "interp/interp.h"
 #include "obj/obj.h"
 #include "opt/ir.h"
 
 typedef struct Lower {
   InterpProgram* p;
   Compiler* c;
   Func* f;
   const ObjBuilder* obj;
   InterpFunc* fn;
   const char* reject;
 } Lower;
 
 static void type_wf(Compiler* c, KitCgTypeId t, u16* w, u8* fp) {
   ABITypeInfo ti = abi_cg_type_info(c->abi, t);
   *w = (u16)ti.size;
   *fp = (ti.scalar_kind == ABI_SC_FLOAT) ? 1u : 0u;
 }
 
 static int is_agg_or_large(Compiler* c, KitCgTypeId t) {
   return cg_type_is_aggregate(c, t) || abi_cg_sizeof(c->abi, t) > 8u;
 }
 
 /* Does this IROp produce a bytecode record? */
 static int inst_emits(u16 op) {
   switch ((IROp)op) {
     case IR_NOP:
     case IR_CONST_I:
     case IR_CONST_BYTES:
     case IR_PHI:
     case IR_PARAM_DECL:
     case IR_SCOPE_BEGIN:
     case IR_SCOPE_END:
       return 0;
     default:
       return 1;
   }
 }
 
 /* Map an IROp to its InterpOp, choosing aggregate-specialized handlers. On an
  * unsupported op, records a reject reason and returns IOP_TRAP. */
 static InterpOp map_op(Lower* lw, const Inst* in) {
   Compiler* c = lw->c;
   switch ((IROp)in->op) {
     case IR_LOAD_IMM:
       return IOP_LOAD_IMM;
     case IR_LOAD_CONST:
       return IOP_LOAD_CONST;
     case IR_COPY:
       return is_agg_or_large(c, in->opnds[0].type) ? IOP_COPY_AGG : IOP_COPY;
     case IR_LOAD:
       return is_agg_or_large(c, in->opnds[0].type) ? IOP_LOAD_AGG : IOP_LOAD;
     case IR_STORE:
       return is_agg_or_large(c, in->opnds[1].type) ? IOP_STORE_AGG : IOP_STORE;
     case IR_ADDR_OF:
       return IOP_ADDR_OF;
     case IR_TLS_ADDR_OF:
       return IOP_TLS_ADDR;
     case IR_AGG_COPY:
       return IOP_AGG_COPY;
     case IR_AGG_SET:
       return IOP_AGG_SET;
     case IR_BITFIELD_LOAD:
       return IOP_BITFIELD_LOAD;
     case IR_BITFIELD_STORE:
       return IOP_BITFIELD_STORE;
     case IR_BINOP:
       return IOP_BINOP;
     case IR_UNOP:
       return IOP_UNOP;
     case IR_CMP:
       return IOP_CMP;
     case IR_CONVERT:
       return IOP_CONVERT;
     case IR_CALL:
       return IOP_CALL;
     case IR_BR:
     case IR_BREAK_TO:
     case IR_CONTINUE_TO:
       return IOP_BR;
     case IR_CONDBR:
       return IOP_CONDBR;
     case IR_CMP_BRANCH:
       return IOP_CMP_BRANCH;
     case IR_SWITCH:
       return IOP_SWITCH;
     case IR_INDIRECT_BRANCH:
       return IOP_INDIRECT_BR;
     case IR_LOAD_LABEL_ADDR:
       return IOP_LOAD_LABEL_ADDR;
     case IR_RET: {
       IRRetAux* aux = (IRRetAux*)in->extra.aux;
       return (aux && aux->present) ? IOP_RET : IOP_RET_VOID;
     }
     case IR_UNREACHABLE:
       return IOP_UNREACHABLE;
     case IR_ALLOCA:
       return IOP_ALLOCA;
     case IR_VA_START:
       return IOP_VA_START;
     case IR_VA_ARG:
       return IOP_VA_ARG;
     case IR_VA_END:
       return IOP_VA_END;
     case IR_VA_COPY:
       return IOP_VA_COPY;
     case IR_ATOMIC_LOAD:
       return IOP_ATOMIC_LOAD;
     case IR_ATOMIC_STORE:
       return IOP_ATOMIC_STORE;
     case IR_ATOMIC_RMW:
       return IOP_ATOMIC_RMW;
     case IR_ATOMIC_CAS:
       return IOP_ATOMIC_CAS;
     case IR_FENCE:
       return IOP_FENCE;
     case IR_INTRINSIC:
       return IOP_INTRINSIC;
     case IR_ASM_BLOCK:
       lw->reject = "inline asm";
       return IOP_TRAP;
     case IR_LOCAL_STATIC_DATA_BEGIN:
     case IR_LOCAL_STATIC_DATA_WRITE:
     case IR_LOCAL_STATIC_DATA_LABEL_ADDR:
     case IR_LOCAL_STATIC_DATA_END:
       /* Function-scope static data (incl. dense-switch jump tables and
        * computed-goto label arrays) is materialized into an interp-private
        * blob at lower time (lower_static_blobs) and the blob's symbol is
        * resolved to that buffer; the stream ops themselves are pure markers. */
       return IOP_NOP;
     default:
       lw->reject = "unhandled IR op";
       return IOP_TRAP;
   }
 }
 
 /* Record (deduped) a global symbol the function references, resolving its name
  * from the obj while it is alive. */
 static void note_global(Lower* lw, ObjSymId sym) {
   InterpFunc* fn = lw->fn;
   const ObjSym* s;
   u32 i;
   if (sym == OBJ_SYM_NONE || !lw->obj) return;
   for (i = 0; i < fn->nglobals; ++i)
     if (fn->globals[i].sym == sym) return;
   s = obj_symbol_get(lw->obj, sym);
   if (!s) return;
   fn->globals[fn->nglobals].sym = sym;
   fn->globals[fn->nglobals].name = pool_slice(lw->c->global, s->name);
   fn->globals[fn->nglobals].cached = NULL;
   fn->globals[fn->nglobals].resolved = 0;
   fn->nglobals++;
 }
 
 static void note_operand_globals(Lower* lw, const Operand* op) {
   if (op->kind == OPK_GLOBAL) note_global(lw, op->v.global.sym);
 }
 
 static void note_inst_globals(Lower* lw, const Inst* in) {
   u32 i;
   for (i = 0; i < in->nopnds; ++i) note_operand_globals(lw, &in->opnds[i]);
   if ((IROp)in->op == IR_CALL && in->extra.aux) {
     IRCallAux* aux = (IRCallAux*)in->extra.aux;
     note_operand_globals(lw, &aux->desc.callee);
   } else if ((IROp)in->op == IR_TLS_ADDR_OF && in->extra.aux) {
     IRTlsAux* aux = (IRTlsAux*)in->extra.aux;
     note_global(lw, aux->sym);
   }
 }
 
 /* Block placement: code pc per reachable block, in emit_order. */
 static u32 block_pc_of(InterpFunc* fn, u32 block) {
   if (block >= fn->nblocks) return INTERP_PC_NONE;
   return fn->block_pc[block];
 }
 
 /* Point a (referenced) global symbol at an already-resolved host address,
  * overriding lazy name resolution. Used for interp-private static blobs. */
 static void register_blob_global(InterpFunc* fn, ObjSymId sym, void* ptr) {
   u32 i;
   if (sym == OBJ_SYM_NONE) return;
   for (i = 0; i < fn->nglobals; ++i) {
     if (fn->globals[i].sym == sym) {
       fn->globals[i].cached = ptr;
       fn->globals[i].resolved = 1;
       return;
     }
   }
   fn->globals[fn->nglobals].sym = sym;
   fn->globals[fn->nglobals].name.s = NULL;
   fn->globals[fn->nglobals].name.len = 0;
   fn->globals[fn->nglobals].cached = ptr;
   fn->globals[fn->nglobals].resolved = 1;
   fn->nglobals++;
 }
 
 /* Materialize every function-scope static data blob (regular static locals,
  * dense-switch jump tables, computed-goto label arrays) into an interp-private,
  * program-lifetime buffer and bind the blob symbol to it.
  *
  * WRITE records contribute their literal bytes (zero-fill when has_data==0).
  * LABEL_ADDR records contribute a *bytecode pc* for the target block: the
  * interpreter addresses code by InterpInsn index, so a table that the program
  * later walks with IR_LOAD + IR_INDIRECT_BRANCH must hold interp pcs, not the
  * native code-label addresses the parallel object/JIT path bakes in. (kit
  * forbids data-symbol relocations inside function-local statics, so WRITE bytes
  * + code-label pcs are the only contents possible.) The BEGIN/WRITE/LABEL_ADDR/
  * END stream ops lower to IOP_NOP — they are fully consumed here.
  *
  * Runs after block placement (block pcs known) and after the globals table is
  * allocated; the per-blob sym is also referenced by an OPK_GLOBAL operand, so
  * the table has room and Pass B's note_global dedups against the entry added
  * here. */
 static void lower_static_blobs(Lower* lw) {
   InterpFunc* fn = lw->fn;
   Func* f = lw->f;
   Arena* a = f->arena ? f->arena : lw->c->tu;
   u32 b;
   for (b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     u32 k;
     for (k = 0; k < bl->ninsts; ++k) {
       const Inst* in = &bl->insts[k];
       CgIrLocalStaticBeginAux* beg;
       ObjSymId sym;
       u32 total = 0u, off = 0u, j;
       u8* buf;
       if ((IROp)in->op != IR_LOCAL_STATIC_DATA_BEGIN) continue;
       beg = (CgIrLocalStaticBeginAux*)in->extra.aux;
       sym = beg ? beg->desc.sym : OBJ_SYM_NONE;
       for (j = k + 1u; j < bl->ninsts; ++j) {
         IROp op = (IROp)bl->insts[j].op;
         if (op == IR_LOCAL_STATIC_DATA_END) break;
         if (op == IR_LOCAL_STATIC_DATA_WRITE) {
           CgIrLocalStaticWriteAux* w =
               (CgIrLocalStaticWriteAux*)bl->insts[j].extra.aux;
           total += w ? (u32)w->len : 0u;
         } else if (op == IR_LOCAL_STATIC_DATA_LABEL_ADDR) {
           CgIrLocalStaticLabelAux* la =
               (CgIrLocalStaticLabelAux*)bl->insts[j].extra.aux;
           total += la ? la->width : 0u;
         }
       }
       buf = total ? arena_array(a, u8, total) : NULL;
       for (j = k + 1u; j < bl->ninsts; ++j) {
         IROp op = (IROp)bl->insts[j].op;
         if (op == IR_LOCAL_STATIC_DATA_END) break;
         if (op == IR_LOCAL_STATIC_DATA_WRITE) {
           CgIrLocalStaticWriteAux* w =
               (CgIrLocalStaticWriteAux*)bl->insts[j].extra.aux;
           u32 len = w ? (u32)w->len : 0u;
           if (len) {
             if (w->has_data && w->data)
               memcpy(buf + off, w->data, len);
             else
               memset(buf + off, 0, len);
           }
           off += len;
         } else if (op == IR_LOCAL_STATIC_DATA_LABEL_ADDR) {
           CgIrLocalStaticLabelAux* la =
               (CgIrLocalStaticLabelAux*)bl->insts[j].extra.aux;
           u32 width = la ? la->width : 0u;
           u64 val = la ? (u64)block_pc_of(fn, (u32)la->target) : 0u;
           u32 bi;
           for (bi = 0u; bi < width && bi < 8u; ++bi)
             buf[off + bi] = (u8)(val >> (bi * 8u));
           for (bi = 8u; bi < width; ++bi) buf[off + bi] = 0u;
           off += width;
         }
       }
       register_blob_global(fn, sym, buf);
     }
   }
 }
 
 InterpFunc* interp_lower(InterpProgram* p, Func* f, ObjSymId sym, Slice name,
                          const ObjBuilder* obj) {
   Compiler* c = p->c;
   Arena* a = f->arena ? f->arena : c->tu;
   Lower lw;
   InterpFunc* fn;
   u32 i, b, pc;
   u32 ncode = 0, nopnd_total = 0;
   u32 max_slot_id = 0;
   u32 off;
   u32 nswitch = 0, swi = 0;
 
   memset(&lw, 0, sizeof lw);
   lw.p = p;
   lw.c = c;
   lw.f = f;
   lw.obj = obj;
 
   fn = arena_znew(a, InterpFunc);
   fn->prog = p;
   fn->f = f;
   fn->sym = sym;
   fn->name = name;
   fn->npregs = f->npregs ? f->npregs : 1u;
   fn->nblocks = f->nblocks;
   fn->ok = 1;
   lw.fn = fn;
 
   /* block_pc, default unreachable. */
   fn->block_pc = arena_array(a, u32, f->nblocks ? f->nblocks : 1u);
   for (b = 0; b < f->nblocks; ++b) fn->block_pc[b] = INTERP_PC_NONE;
 
   /* Pass A: place blocks in emit_order, count records + operands + switches. */
   for (i = 0; i < f->emit_order_n; ++i) {
     b = f->emit_order[i];
     if (b >= f->nblocks) continue;
     if (fn->block_pc[b] != INTERP_PC_NONE) continue; /* placed already */
     fn->block_pc[b] = ncode;
     {
       Block* bl = &f->blocks[b];
       u32 k;
       for (k = 0; k < bl->ninsts; ++k) {
         const Inst* in = &bl->insts[k];
         if (!inst_emits(in->op)) continue;
         ncode++;
         nopnd_total += in->nopnds;
         if ((IROp)in->op == IR_SWITCH) nswitch++;
       }
     }
   }
   (void)nopnd_total;
 
   fn->code = arena_zarray(a, InterpInsn, ncode ? ncode : 1u);
   fn->ncode = ncode;
   fn->switches = nswitch ? arena_zarray(a, InterpSwitch, nswitch) : NULL;
   fn->nswitches = nswitch;
   /* Over-allocate the globals table: at most one per operand + one per call. */
   {
     u32 cap = 0;
     for (i = 0; i < f->nblocks; ++i) {
       Block* bl = &f->blocks[i];
       u32 k;
       for (k = 0; k < bl->ninsts; ++k) cap += bl->insts[k].nopnds + 1u;
     }
     fn->globals = arena_zarray(a, InterpGlobal, cap ? cap : 1u);
     fn->nglobals = 0;
   }
 
   /* Frame slots: bump-allocate non-alloca slots honoring align. */
   for (i = 0; i < f->nframe_slots; ++i)
     if (f->frame_slots[i].id > max_slot_id) max_slot_id = f->frame_slots[i].id;
   fn->nslots = max_slot_id + 1u;
   fn->slot_off = arena_zarray(a, u32, fn->nslots);
   fn->frame_align = 16u;
   off = 0;
   for (i = 0; i < f->nframe_slots; ++i) {
     IRFrameSlot* fs = &f->frame_slots[i];
     u32 align = fs->align ? fs->align : 1u;
     u32 size = fs->size;
     if (fs->kind == FS_ALLOCA) continue; /* dynamic, allocated at OP_ALLOCA */
     if (align > fn->frame_align) fn->frame_align = align;
     off = (off + align - 1u) & ~(align - 1u);
     fn->slot_off[fs->id] = off;
     off += size ? size : 1u;
   }
   fn->frame_bytes = (off + 15u) & ~15u;
 
   /* Materialize function-scope static data into interp-private buffers and bind
    * their symbols (block pcs are now known for any label-address tables). */
   lower_static_blobs(&lw);
 
   /* Pass B: emit records, resolving branch targets to pcs. */
   pc = 0;
   for (i = 0; i < f->emit_order_n; ++i) {
     Block* bl;
     u32 k;
     b = f->emit_order[i];
     if (b >= f->nblocks) continue;
     if (fn->block_pc[b] != pc) {
       /* A block placed at a different pc (duplicate in emit_order) — skip its
        * second appearance to keep pc aligned with Pass A. */
       if (fn->block_pc[b] != INTERP_PC_NONE && fn->block_pc[b] < pc) continue;
     }
     bl = &f->blocks[b];
     for (k = 0; k < bl->ninsts; ++k) {
       const Inst* in = &bl->insts[k];
       InterpInsn* rec;
       if (!inst_emits(in->op)) continue;
       note_inst_globals(&lw, in);
       rec = &fn->code[pc++];
       rec->inst = in;
       rec->op = (u32)map_op(&lw, in);
       rec->dst = (in->nopnds > 0 && in->opnds[0].kind == OPK_REG)
                      ? in->opnds[0].v.reg
                      : 0u;
       switch ((InterpOp)rec->op) {
         case IOP_LOAD_IMM:
           rec->imm = in->extra.imm;
           type_wf(c, in->opnds[0].type, &rec->w0, &rec->fp0);
           break;
         case IOP_COPY:
         case IOP_LOAD:
         case IOP_ADDR_OF:
           type_wf(c, in->opnds[0].type, &rec->w0, &rec->fp0);
           if ((IROp)in->op == IR_LOAD) rec->w0 = (u16)in->extra.mem.size;
           break;
         case IOP_STORE:
           rec->w0 = (u16)in->extra.mem.size;
           type_wf(c, in->opnds[1].type, &rec->w1, &rec->fp1);
           rec->fp0 = rec->fp1;
           break;
         case IOP_BINOP:
         case IOP_UNOP:
           rec->sub = (u32)in->extra.imm;
           type_wf(c, in->opnds[0].type, &rec->w0, &rec->fp0);
           break;
         case IOP_CMP:
           rec->sub = (u32)in->extra.imm;
           type_wf(c, in->opnds[1].type, &rec->w0, &rec->fp0);
           break;
         case IOP_CONVERT:
           rec->sub = (u32)in->extra.imm;
           type_wf(c, in->opnds[0].type, &rec->w0, &rec->fp0);
           type_wf(c, in->opnds[1].type, &rec->w1, &rec->fp1);
           break;
         case IOP_ALLOCA:
           rec->imm = in->extra.imm; /* alignment */
           break;
         case IOP_CALL: {
           /* A realized tail call is the block terminator: the CGCallDesc has
            * CG_CALL_TAIL, or equivalently the call is the last emitting inst
            * of a successor-less block (it returns the callee's result
            * directly). Detect both. */
           IRCallAux* aux = (IRCallAux*)in->extra.aux;
           u8 is_tail = aux && (aux->desc.flags & CG_CALL_TAIL);
           if (!is_tail && bl->nsucc == 0) {
             u32 j;
             u8 last = 1;
             for (j = k + 1u; j < bl->ninsts; ++j)
               if (inst_emits(bl->insts[j].op)) {
                 last = 0;
                 break;
               }
             is_tail = last;
           }
           rec->tail = is_tail;
           break;
         }
         case IOP_BR:
           rec->t0 = block_pc_of(fn, bl->nsucc > 0 ? bl->succ[0] : 0xffffffffu);
           break;
         case IOP_CONDBR:
         case IOP_CMP_BRANCH:
           if ((IROp)in->op == IR_CMP_BRANCH) {
             rec->sub = (u32)in->extra.imm;
             type_wf(c, in->opnds[0].type, &rec->w0, &rec->fp0);
           }
           rec->t0 = block_pc_of(fn, bl->nsucc > 0 ? bl->succ[0] : 0xffffffffu);
           rec->t1 = block_pc_of(fn, bl->nsucc > 1 ? bl->succ[1] : 0xffffffffu);
           break;
         case IOP_SWITCH: {
           IRSwitchAux* aux = (IRSwitchAux*)in->extra.aux;
           InterpSwitch* sw = &fn->switches[swi];
           u32 ci;
           rec->t0 = swi;
           swi++;
           sw->aux = aux;
           sw->sel_type = aux ? aux->selector_type : 0;
           sw->ncases = aux ? aux->ncases : 0;
           sw->case_pc = sw->ncases ? arena_array(a, u32, sw->ncases) : NULL;
           for (ci = 0; ci < sw->ncases; ++ci)
             sw->case_pc[ci] = block_pc_of(fn, aux->cases[ci].block);
           sw->default_pc =
               aux ? block_pc_of(fn, aux->default_block) : INTERP_PC_NONE;
           break;
         }
         case IOP_LOAD_LABEL_ADDR:
           rec->t0 = block_pc_of(fn, (u32)in->extra.imm);
           break;
         case IOP_RET:
         case IOP_RET_VOID:
           break;
         case IOP_ATOMIC_RMW: {
           IRAtomicAux* aux = (IRAtomicAux*)in->extra.aux;
           rec->sub = aux ? aux->op : 0u;
           rec->w0 = aux ? (u16)aux->mem.size : 8u;
           break;
         }
         case IOP_ATOMIC_LOAD:
         case IOP_ATOMIC_STORE: {
           IRAtomicAux* aux = (IRAtomicAux*)in->extra.aux;
           rec->w0 = aux ? (u16)aux->mem.size : 8u;
           break;
         }
         case IOP_ATOMIC_CAS: {
           IRCasAux* aux = (IRCasAux*)in->extra.aux;
           rec->w0 = aux ? (u16)aux->mem.size : 8u;
           break;
         }
         case IOP_TRAP:
           if (fn->ok) {
             fn->ok = 0;
             fn->reject_reason = lw.reject ? lw.reject : "unsupported op";
           }
           break;
         default:
           break;
       }
     }
   }
 
   return fn;
 }