ir_emit.c (36697B)
1 #include <string.h> 2 3 #include "cg/ir.h" 4 5 typedef Operand CgSemOperand; 6 typedef CGCallDesc CgSemCallDesc; 7 typedef CGFuncDesc CgSemFuncDesc; 8 typedef CGParamDesc CgSemParamDesc; 9 typedef CGScopeDesc CgSemScopeDesc; 10 11 #include "arch/wasm/internal.h" 12 #include "cg/type.h" 13 #include "core/heap.h" 14 15 void wasm_func_begin(CGTarget*, const CGFuncDesc*); 16 void wasm_func_end(CGTarget*); 17 void wasm_alias(CGTarget*, ObjSymId, ObjSymId, KitCgTypeId); 18 CGLocalStorage wasm_param(CGTarget*, const CGParamDesc*); 19 CGLocalStorage wasm_local(CGTarget*, const CGLocalDesc*); 20 Label wasm_label_new(CGTarget*); 21 void wasm_label_place(CGTarget*, Label); 22 void wasm_jump(CGTarget*, Label); 23 void wasm_cmp_branch(CGTarget*, CmpOp, Operand, Operand, Label); 24 void wasm_switch(CGTarget*, const CGSwitchDesc*); 25 CGScope wasm_scope_begin(CGTarget*, const CGScopeDesc*); 26 void wasm_scope_end(CGTarget*, CGScope); 27 void wasm_break_to(CGTarget*, CGScope); 28 void wasm_continue_to(CGTarget*, CGScope); 29 void wasm_set_loc(CGTarget*, SrcLoc); 30 void wasm_load_imm(CGTarget*, Operand, i64); 31 void wasm_load_const(CGTarget*, Operand, ConstBytes); 32 void wasm_copy(CGTarget*, Operand, Operand); 33 void wasm_load(CGTarget*, Operand, Operand, MemAccess); 34 void wasm_store(CGTarget*, Operand, Operand, MemAccess); 35 void wasm_addr_of(CGTarget*, Operand, Operand); 36 void wasm_copy_bytes(CGTarget*, Operand, Operand, AggregateAccess); 37 void wasm_set_bytes(CGTarget*, Operand, Operand, AggregateAccess); 38 void wasm_binop(CGTarget*, BinOp, Operand, Operand, Operand); 39 void wasm_unop(CGTarget*, UnOp, Operand, Operand); 40 void wasm_cmp(CGTarget*, CmpOp, Operand, Operand, Operand); 41 void wasm_convert(CGTarget*, ConvKind, Operand, Operand); 42 void wasm_call(CGTarget*, const CGCallDesc*); 43 void wasm_ret(CGTarget*, const CGABIValue*); 44 void wasm_unreachable(CGTarget*); 45 void wasm_alloca(CGTarget*, Operand, Operand, u32); 46 void wasm_va_start(CGTarget*, Operand); 47 void wasm_va_arg(CGTarget*, Operand, Operand, KitCgTypeId); 48 void wasm_va_end(CGTarget*, Operand); 49 void wasm_va_copy(CGTarget*, Operand, Operand); 50 void wasm_atomic_load(CGTarget*, Operand, Operand, MemAccess, KitCgMemOrder); 51 void wasm_atomic_store(CGTarget*, Operand, Operand, MemAccess, KitCgMemOrder); 52 void wasm_atomic_rmw(CGTarget*, KitCgAtomicOp, Operand, Operand, Operand, 53 MemAccess, KitCgMemOrder); 54 void wasm_atomic_cas(CGTarget*, Operand, Operand, Operand, Operand, Operand, 55 MemAccess, KitCgMemOrder, KitCgMemOrder); 56 void wasm_fence(CGTarget*, KitCgMemOrder); 57 void wasm_intrinsic(CGTarget*, IntrinKind, Operand*, u32, const Operand*, u32); 58 void wasm_asm_block(CGTarget*, const char*, const AsmConstraint*, u32, Operand*, 59 const AsmConstraint*, u32, const Operand*, const Sym*, u32); 60 61 typedef struct WasmIrEmitter { 62 WTarget* target; 63 FrameSlot* local_slots; 64 KitCgTypeId* local_types; /* CG type of each local id, parallel to slots */ 65 u32 local_slots_n; 66 Reg next_temp_reg; 67 CGScope* scope_map; 68 u32 scope_map_n; 69 } WasmIrEmitter; 70 71 static void wasm_ir_fail(WasmIrEmitter* e, SrcLoc loc, const char* msg) { 72 compiler_panic(e->target->c, loc, "%s", msg); 73 } 74 75 static RegClass wasm_ir_class_for_type(WTarget* t, KitCgTypeId type) { 76 ABITypeInfo info; 77 if (!type) return RC_INT; 78 info = abi_cg_type_info(t->c->abi, type); 79 return info.scalar_kind == ABI_SC_FLOAT ? RC_FP : RC_INT; 80 } 81 82 /* An aggregate (record/array) value lives in linear memory on wasm: a scalar 83 * COPY of it must be lowered to a byte-wise memcpy between the two homes. */ 84 static int wasm_ir_is_aggregate(WTarget* t, KitCgTypeId ty) { 85 ABITypeInfo info; 86 if (!ty) return 0; 87 info = abi_cg_type_info(t->c->abi, ty); 88 if (info.scalar_kind == ABI_SC_PTR) return 0; 89 if (info.size == 0) return 0; /* genuine void */ 90 return info.scalar_kind == ABI_SC_VOID || info.size > 8u; 91 } 92 93 static void wasm_ir_bind_reg(WasmIrEmitter* e, Reg reg, u32 wasm_local, 94 KitCgTypeId type) { 95 WTarget* t = e->target; 96 Heap* h = t->c->ctx->heap; 97 if (reg == REG_NONE) return; 98 if (reg >= t->reg_cap) { 99 u32 nc = t->reg_cap ? t->reg_cap : 64u; 100 while (nc <= reg) nc *= 2u; 101 u32* regs = (u32*)h->realloc(h, t->reg_to_local, sizeof(u32) * t->reg_cap, 102 sizeof(u32) * nc, _Alignof(u32)); 103 KitCgTypeId* types = (KitCgTypeId*)h->realloc( 104 h, t->reg_type, sizeof(KitCgTypeId) * t->reg_cap, 105 sizeof(KitCgTypeId) * nc, _Alignof(KitCgTypeId)); 106 u8* cls = (u8*)h->realloc(h, t->reg_cls, t->reg_cap, nc, 1); 107 if (!regs || !types || !cls) 108 compiler_panic(t->c, (SrcLoc){0, 0, 0}, "wasm IR emit: out of memory"); 109 for (u32 i = t->reg_cap; i < nc; ++i) { 110 regs[i] = 0xffffffffu; 111 types[i] = 0; 112 cls[i] = 0; 113 } 114 t->reg_to_local = regs; 115 t->reg_type = types; 116 t->reg_cls = cls; 117 t->reg_cap = nc; 118 } 119 /* A NONE type carries no wasm value type. This shows up for variadic call 120 * arguments, whose operands are untyped at the call site. Don't let it 121 * clobber a binding the locals/params pass already gave a real type. */ 122 if (!type && t->reg_to_local[reg] != 0xffffffffu && t->reg_type[reg]) return; 123 t->reg_to_local[reg] = wasm_local; 124 t->reg_type[reg] = type; 125 t->reg_cls[reg] = (u8)wasm_ir_class_for_type(t, type); 126 } 127 128 static void wasm_ir_bind_value_local(WasmIrEmitter* e, CGLocal local, 129 KitCgTypeId type, FrameSlot slot) { 130 WSlot* s; 131 if (slot == FRAME_SLOT_NONE) return; 132 s = &e->target->slots[slot - 1u]; 133 if (s->kind == W_SLOT_LOCAL) 134 wasm_ir_bind_reg(e, (Reg)local, s->wasm_local, type); 135 } 136 137 static Operand wasm_ir_value_op(WasmIrEmitter* e, CgSemOperand in) { 138 Operand out; 139 memset(&out, 0, sizeof out); 140 out.kind = in.kind; 141 out.type = in.type; 142 out.cls = (u8)wasm_ir_class_for_type(e->target, in.type); 143 switch ((OpKind)in.kind) { 144 case OPK_IMM: 145 out.v.imm = in.v.imm; 146 return out; 147 case OPK_LOCAL: 148 out.kind = OPK_REG; 149 out.v.reg = (Reg)in.v.local; 150 return out; 151 case OPK_GLOBAL: 152 out.v.global.sym = in.v.global.sym; 153 out.v.global.addend = in.v.global.addend; 154 return out; 155 case OPK_INDIRECT: 156 out.v.ind.base = (Reg)in.v.ind.base; 157 out.v.ind.index = 158 in.v.ind.index == CG_LOCAL_NONE ? REG_NONE : (Reg)in.v.ind.index; 159 out.v.ind.log2_scale = in.v.ind.log2_scale; 160 out.v.ind.ofs = in.v.ind.ofs; 161 return out; 162 } 163 return out; 164 } 165 166 static Reg wasm_ir_temp_reg(WasmIrEmitter* e) { return e->next_temp_reg++; } 167 168 static Operand wasm_ir_source_op(WasmIrEmitter* e, CgSemOperand in, 169 SrcLoc loc) { 170 Operand out; 171 if (in.kind == OPK_LOCAL && in.v.local < e->local_slots_n) { 172 FrameSlot slot = e->local_slots[in.v.local]; 173 if (slot != FRAME_SLOT_NONE) { 174 WSlot* s = &e->target->slots[slot - 1u]; 175 if (s->kind == W_SLOT_LOCAL) { 176 wasm_ir_bind_value_local(e, in.v.local, in.type, slot); 177 } else { 178 MemAccess mem; 179 Operand addr; 180 memset(&mem, 0, sizeof mem); 181 memset(&addr, 0, sizeof addr); 182 out = wasm_ir_value_op(e, in); 183 out.v.reg = wasm_ir_temp_reg(e); 184 addr.kind = OPK_LOCAL; 185 addr.type = in.type; 186 addr.v.frame_slot = slot; 187 mem.type = in.type; 188 mem.size = s->size; 189 mem.align = s->align; 190 wasm_load((CGTarget*)&e->target->base, out, addr, mem); 191 return out; 192 } 193 } 194 } 195 (void)loc; 196 return wasm_ir_value_op(e, in); 197 } 198 199 /* A value-producing instruction whose CG destination is an address-taken 200 * (stack-homed) local cannot write a wasm local directly: the variable lives 201 * in linear memory so its address is meaningful. Route the def through a temp 202 * reg and store it back to the slot once the underlying op has run. */ 203 typedef struct WasmIrDest { 204 int spill; 205 Operand tmp; /* OPK_REG temp the op writes into */ 206 Operand store_addr; /* OPK_LOCAL frame slot to store back to */ 207 MemAccess mem; 208 } WasmIrDest; 209 210 static Operand wasm_ir_dest_op(WasmIrEmitter* e, CgSemOperand in, 211 WasmIrDest* d) { 212 memset(d, 0, sizeof *d); 213 if (in.kind == OPK_LOCAL && in.v.local < e->local_slots_n) { 214 FrameSlot slot = e->local_slots[in.v.local]; 215 if (slot != FRAME_SLOT_NONE) { 216 WSlot* s = &e->target->slots[slot - 1u]; 217 if (s->kind != W_SLOT_LOCAL) { 218 Operand tmp = wasm_ir_value_op(e, in); 219 tmp.kind = OPK_REG; 220 tmp.v.reg = wasm_ir_temp_reg(e); 221 d->spill = 1; 222 d->tmp = tmp; 223 memset(&d->store_addr, 0, sizeof d->store_addr); 224 d->store_addr.kind = OPK_LOCAL; 225 d->store_addr.type = in.type; 226 d->store_addr.v.frame_slot = slot; 227 memset(&d->mem, 0, sizeof d->mem); 228 d->mem.type = in.type; 229 d->mem.size = s->size; 230 d->mem.align = s->align; 231 return tmp; 232 } 233 } 234 } 235 return wasm_ir_value_op(e, in); 236 } 237 238 static void wasm_ir_dest_finish(WasmIrEmitter* e, const WasmIrDest* d) { 239 if (d->spill) 240 wasm_store((CGTarget*)&e->target->base, d->store_addr, d->tmp, d->mem); 241 } 242 243 static Operand wasm_ir_addr_op(WasmIrEmitter* e, CgSemOperand in, SrcLoc loc) { 244 Operand out; 245 if (in.kind != OPK_LOCAL) return wasm_ir_value_op(e, in); 246 if (in.v.local == CG_LOCAL_NONE || in.v.local >= e->local_slots_n || 247 e->local_slots[in.v.local] == FRAME_SLOT_NONE) { 248 wasm_ir_fail(e, loc, "wasm IR emit: unknown local address"); 249 } 250 memset(&out, 0, sizeof out); 251 out.kind = OPK_LOCAL; 252 out.type = in.type; 253 out.v.frame_slot = e->local_slots[in.v.local]; 254 return out; 255 } 256 257 /* Lower an aggregate move to a memory.copy between two linear-memory homes. 258 * Both `dst` and `src` are lvalue operands (frame slots, indirect `[ptr]` 259 * addressing, or global symbols); copy_bytes wants each endpoint as a 260 * pointer-valued register, so materialize the effective address of each with 261 * addr_of first. `ty` names the aggregate being moved. */ 262 static void wasm_ir_emit_agg_move(WasmIrEmitter* e, CgSemOperand dst, 263 CgSemOperand src, KitCgTypeId ty, 264 SrcLoc loc) { 265 CGTarget* t = (CGTarget*)&e->target->base; 266 AggregateAccess agg; 267 KitCgTypeId pty = cg_type_ptr_to(e->target->c, ty); 268 Operand adst = wasm_ir_addr_op(e, dst, loc); 269 Operand asrc = wasm_ir_addr_op(e, src, loc); 270 Operand dreg, sreg; 271 memset(&dreg, 0, sizeof dreg); 272 memset(&sreg, 0, sizeof sreg); 273 dreg.kind = sreg.kind = OPK_REG; 274 dreg.type = sreg.type = pty; 275 dreg.cls = sreg.cls = (u8)RC_INT; 276 dreg.v.reg = wasm_ir_temp_reg(e); 277 sreg.v.reg = wasm_ir_temp_reg(e); 278 wasm_addr_of(t, dreg, adst); 279 wasm_addr_of(t, sreg, asrc); 280 memset(&agg, 0, sizeof agg); 281 agg.type = ty; 282 agg.size = (u32)abi_cg_sizeof(e->target->c->abi, ty); 283 agg.align = (u32)abi_cg_alignof(e->target->c->abi, ty); 284 wasm_copy_bytes(t, dreg, sreg, agg); 285 } 286 287 static CGScope wasm_ir_scope_lookup(WasmIrEmitter* e, CGScope recorded, 288 SrcLoc loc) { 289 if ((u32)recorded >= e->scope_map_n || !e->scope_map[recorded]) 290 wasm_ir_fail(e, loc, "wasm IR emit: unknown recorded scope"); 291 return e->scope_map[recorded]; 292 } 293 294 static void wasm_ir_bind_scope(WasmIrEmitter* e, CGScope recorded, 295 CGScope emitted, SrcLoc loc) { 296 if ((u32)recorded >= e->scope_map_n) 297 wasm_ir_fail(e, loc, "wasm IR emit: recorded scope out of range"); 298 e->scope_map[recorded] = emitted; 299 } 300 301 static const ABIArgInfo* wasm_ir_param_abi(const ABIFuncInfo* abi, u32 index) { 302 return abi && index < abi->nparams ? &abi->params[index] : NULL; 303 } 304 305 static CGABIValue wasm_ir_abi_value(WasmIrEmitter* e, CGLocal local, 306 KitCgTypeId type, const ABIArgInfo* abi, 307 int address, int source, SrcLoc loc) { 308 CGABIValue out; 309 CgSemOperand sem; 310 memset(&out, 0, sizeof out); 311 memset(&sem, 0, sizeof sem); 312 sem.kind = OPK_LOCAL; 313 sem.type = type; 314 sem.v.local = local; 315 out.type = type; 316 out.abi = abi; 317 out.storage = address ? wasm_ir_addr_op(e, sem, loc) 318 : source ? wasm_ir_source_op(e, sem, loc) 319 : wasm_ir_value_op(e, sem); 320 return out; 321 } 322 323 static void wasm_ir_emit_call(WasmIrEmitter* e, const CgIrInst* in) { 324 const CgIrCallAux* aux = (const CgIrCallAux*)in->extra.aux; 325 const CgSemCallDesc* src = &aux->desc; 326 Heap* h = e->target->c->ctx->heap; 327 const ABIFuncInfo* abi = abi_cg_func_info(e->target->c->abi, src->fn_type); 328 CGCallDesc d; 329 CGABIValue* args = NULL; 330 KitCgTypeId ret_type = cg_type_func_ret_id(e->target->c, src->fn_type); 331 memset(&d, 0, sizeof d); 332 d.fn_type = src->fn_type; 333 d.callee = wasm_ir_value_op(e, src->callee); 334 d.nargs = src->nargs; 335 d.flags = src->flags; 336 d.tail_policy = src->tail_policy; 337 d.inline_policy = src->inline_policy; 338 d.abi = abi; 339 if (src->nargs) { 340 args = (CGABIValue*)h->alloc(h, sizeof(*args) * src->nargs, 341 _Alignof(CGABIValue)); 342 if (!args) wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 343 for (u32 i = 0; i < src->nargs; ++i) { 344 const ABIArgInfo* ai = wasm_ir_param_abi(abi, i); 345 KitCgTypeId ty = KIT_CG_TYPE_NONE; 346 const CgType* fty = cg_type_get(e->target->c, src->fn_type); 347 if (fty && fty->kind == KIT_CG_TYPE_FUNC && i < fty->func.nparams) 348 ty = fty->func.params[i].type; 349 /* Variadic arguments have no signature slot; recover the type from the 350 * passed local so the call's vararg packing knows its wasm value type. */ 351 if (ty == KIT_CG_TYPE_NONE && src->args[i] != CG_LOCAL_NONE && 352 (u32)src->args[i] < e->local_slots_n) 353 ty = e->local_types[src->args[i]]; 354 args[i] = 355 wasm_ir_abi_value(e, src->args[i], ty, ai, 356 ai && ai->kind == ABI_ARG_INDIRECT, 1, in->loc); 357 } 358 } 359 d.args = args; 360 if (src->result != CG_LOCAL_NONE) { 361 d.ret = 362 wasm_ir_abi_value(e, src->result, ret_type, abi ? &abi->ret : NULL, 363 abi && abi->has_sret, 0, in->loc); 364 } 365 wasm_call((CGTarget*)&e->target->base, &d); 366 if (args) h->free(h, args, sizeof(*args) * src->nargs); 367 } 368 369 static void wasm_ir_emit_ret(WasmIrEmitter* e, const CgIrFunc* f, 370 const CgIrInst* in) { 371 const CgIrRetAux* aux = (const CgIrRetAux*)in->extra.aux; 372 const ABIFuncInfo* abi = abi_cg_func_info(e->target->c->abi, f->desc.fn_type); 373 KitCgTypeId ret_type = cg_type_func_ret_id(e->target->c, f->desc.fn_type); 374 CGABIValue ret; 375 if (!aux || !aux->present) { 376 wasm_ret((CGTarget*)&e->target->base, NULL); 377 return; 378 } 379 ret = wasm_ir_abi_value(e, aux->value, ret_type, abi ? &abi->ret : NULL, 380 abi && abi->has_sret, 1, in->loc); 381 wasm_ret((CGTarget*)&e->target->base, &ret); 382 } 383 384 static void wasm_ir_emit_switch(WasmIrEmitter* e, const CgIrInst* in) { 385 const CgIrSwitchAux* aux = (const CgIrSwitchAux*)in->extra.aux; 386 CGSwitchDesc d; 387 memset(&d, 0, sizeof d); 388 d.selector = wasm_ir_source_op(e, in->opnds[0], in->loc); 389 d.selector_type = aux->selector_type; 390 d.default_label = aux->default_label; 391 d.cases = aux->cases; 392 d.ncases = aux->ncases; 393 d.hint = aux->hint; 394 d.opt_level = aux->opt_level; 395 wasm_switch((CGTarget*)&e->target->base, &d); 396 } 397 398 /* Bitfields have no native wasm insert/extract, so lower to load + shift/mask 399 * + store over the storage unit. All arithmetic runs in i64 regardless of 400 * storage width: the load zero-extends into i64 (i64.load{8,16,32}_u), the 401 * store truncates back (i64.store{8,16,32}), and a uniform 64-bit shift count 402 * keeps the field-extraction math width-agnostic. storage_offset is always 0 403 * here — the frontend folds it into record_addr. */ 404 #define WASM_BF_REG_BITS 64u 405 406 static Operand wasm_ir_temp_i64(WasmIrEmitter* e) { 407 Operand o; 408 memset(&o, 0, sizeof o); 409 o.kind = OPK_REG; 410 o.type = builtin_id(KIT_CG_BUILTIN_I64); 411 o.cls = (u8)RC_INT; 412 o.v.reg = wasm_ir_temp_reg(e); 413 return o; 414 } 415 416 static Operand wasm_ir_imm_i64(i64 v) { 417 Operand o; 418 memset(&o, 0, sizeof o); 419 o.kind = OPK_IMM; 420 o.type = builtin_id(KIT_CG_BUILTIN_I64); 421 o.cls = (u8)RC_INT; 422 o.v.imm = v; 423 return o; 424 } 425 426 /* Storage-unit access: i64 value, exactly storage_size bytes wide. */ 427 static MemAccess wasm_ir_bf_storage_mem(const BitFieldAccess* bf) { 428 MemAccess mem = bf->storage; 429 mem.type = builtin_id(KIT_CG_BUILTIN_I64); 430 mem.size = bf->storage.size ? bf->storage.size : 4u; 431 return mem; 432 } 433 434 static void wasm_ir_emit_bitfield_load(WasmIrEmitter* e, const CgIrInst* in) { 435 CGTarget* t = (CGTarget*)&e->target->base; 436 const CgIrBitFieldAux* aux = (const CgIrBitFieldAux*)in->extra.aux; 437 const BitFieldAccess* bf = &aux->access; 438 u32 width = bf->bit_width ? bf->bit_width : 1u; 439 u32 lsb = bf->bit_offset; 440 u32 left = WASM_BF_REG_BITS - lsb - width; /* bits above the field */ 441 u32 right = WASM_BF_REG_BITS - width; /* slide field back to bit 0 */ 442 Operand addr = wasm_ir_addr_op(e, in->opnds[1], in->loc); 443 Operand val = wasm_ir_temp_i64(e); 444 WasmIrDest d; 445 Operand dst; 446 447 /* Load the storage unit, slide the field to the top of the i64, then back 448 * down — arithmetic shift sign-extends a signed field, logical zero-extends 449 * an unsigned one. */ 450 wasm_load(t, val, addr, wasm_ir_bf_storage_mem(bf)); 451 if (left) wasm_binop(t, BO_SHL, val, val, wasm_ir_imm_i64((i64)left)); 452 if (right) 453 wasm_binop(t, bf->signed_ ? BO_SHR_S : BO_SHR_U, val, val, 454 wasm_ir_imm_i64((i64)right)); 455 dst = wasm_ir_dest_op(e, in->opnds[0], &d); 456 /* Narrow to the field's wasm value type; a no-op copy when dst is i64. */ 457 wasm_convert(t, CV_TRUNC, dst, val); 458 wasm_ir_dest_finish(e, &d); 459 } 460 461 static void wasm_ir_emit_bitfield_store(WasmIrEmitter* e, const CgIrInst* in) { 462 CGTarget* t = (CGTarget*)&e->target->base; 463 const CgIrBitFieldAux* aux = (const CgIrBitFieldAux*)in->extra.aux; 464 const BitFieldAccess* bf = &aux->access; 465 u32 width = bf->bit_width ? bf->bit_width : 1u; 466 u32 lsb = bf->bit_offset; 467 u64 ones = (width >= WASM_BF_REG_BITS) ? ~(u64)0 : (((u64)1 << width) - 1u); 468 u64 mask = ones << lsb; 469 MemAccess mem = wasm_ir_bf_storage_mem(bf); 470 Operand addr = wasm_ir_addr_op(e, in->opnds[0], in->loc); 471 Operand cur = wasm_ir_temp_i64(e); 472 473 /* Read-modify-write: clear the field bits, OR in the masked/shifted value. */ 474 wasm_load(t, cur, addr, mem); 475 wasm_binop(t, BO_AND, cur, cur, wasm_ir_imm_i64((i64)~mask)); 476 if (in->opnds[1].kind == OPK_IMM) { 477 u64 v = ((u64)in->opnds[1].v.imm & ones) << lsb; 478 wasm_binop(t, BO_OR, cur, cur, wasm_ir_imm_i64((i64)v)); 479 } else { 480 Operand src = wasm_ir_source_op(e, in->opnds[1], in->loc); 481 Operand staged = wasm_ir_temp_i64(e); 482 wasm_convert(t, CV_ZEXT, staged, src); /* widen field value to i64 */ 483 wasm_binop(t, BO_AND, staged, staged, wasm_ir_imm_i64((i64)ones)); 484 if (lsb) wasm_binop(t, BO_SHL, staged, staged, wasm_ir_imm_i64((i64)lsb)); 485 wasm_binop(t, BO_OR, cur, cur, staged); 486 } 487 wasm_store(t, addr, cur, mem); 488 } 489 490 static void wasm_ir_emit_inst(WasmIrEmitter* e, const CgIrFunc* f, 491 const CgIrInst* in) { 492 CGTarget* t = (CGTarget*)&e->target->base; 493 wasm_set_loc(t, in->loc); 494 switch ((CgIrOp)in->op) { 495 case CG_IR_NOP: 496 return; 497 case CG_IR_LABEL: 498 wasm_label_place(t, (Label)in->extra.imm); 499 return; 500 case CG_IR_LOAD_IMM: { 501 WasmIrDest d; 502 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 503 wasm_load_imm(t, dst, in->extra.imm); 504 wasm_ir_dest_finish(e, &d); 505 return; 506 } 507 case CG_IR_LOAD_CONST: { 508 WasmIrDest d; 509 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 510 wasm_load_const(t, dst, in->extra.cbytes); 511 wasm_ir_dest_finish(e, &d); 512 return; 513 } 514 case CG_IR_COPY: { 515 WasmIrDest d; 516 Operand src, dst; 517 if (wasm_ir_is_aggregate(e->target, in->opnds[0].type)) { 518 wasm_ir_emit_agg_move(e, in->opnds[0], in->opnds[1], in->opnds[0].type, 519 in->loc); 520 return; 521 } 522 src = wasm_ir_source_op(e, in->opnds[1], in->loc); 523 dst = wasm_ir_dest_op(e, in->opnds[0], &d); 524 wasm_copy(t, dst, src); 525 wasm_ir_dest_finish(e, &d); 526 return; 527 } 528 case CG_IR_LOAD: { 529 WasmIrDest d; 530 Operand addr, dst; 531 if (wasm_ir_is_aggregate(e->target, in->opnds[0].type)) { 532 /* Aggregate load: the source operand is the address of the aggregate 533 * (an indirect `[ptr]` or a global symbol), so its effective address 534 * is the source home. Lower to memory.copy into the destination's 535 * home rather than a scalar wasm load. */ 536 wasm_ir_emit_agg_move(e, in->opnds[0], in->opnds[1], in->opnds[0].type, 537 in->loc); 538 return; 539 } 540 addr = wasm_ir_addr_op(e, in->opnds[1], in->loc); 541 dst = wasm_ir_dest_op(e, in->opnds[0], &d); 542 wasm_load(t, dst, addr, in->extra.mem); 543 wasm_ir_dest_finish(e, &d); 544 return; 545 } 546 case CG_IR_STORE: { 547 Operand addr = wasm_ir_addr_op(e, in->opnds[0], in->loc); 548 Operand src = wasm_ir_source_op(e, in->opnds[1], in->loc); 549 wasm_store(t, addr, src, in->extra.mem); 550 return; 551 } 552 case CG_IR_ADDR_OF: { 553 WasmIrDest d; 554 Operand addr = wasm_ir_addr_op(e, in->opnds[1], in->loc); 555 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 556 wasm_addr_of(t, dst, addr); 557 wasm_ir_dest_finish(e, &d); 558 return; 559 } 560 case CG_IR_TLS_ADDR_OF: { 561 /* Wasm has no thread-local storage: a module instance owns one linear 562 * memory, so a thread-local resolves to a fixed data address. Lower to 563 * the symbol's (addend-adjusted) linear-memory address, exactly like a 564 * non-TLS addr_of of a global. */ 565 const CgIrTlsAux* aux = (const CgIrTlsAux*)in->extra.aux; 566 WasmIrDest d; 567 Operand src; 568 Operand dst; 569 memset(&src, 0, sizeof src); 570 src.kind = OPK_GLOBAL; 571 src.type = in->opnds[0].type; 572 src.v.global.sym = aux->sym; 573 src.v.global.addend = aux->addend; 574 dst = wasm_ir_dest_op(e, in->opnds[0], &d); 575 wasm_addr_of(t, dst, src); 576 wasm_ir_dest_finish(e, &d); 577 return; 578 } 579 case CG_IR_AGG_COPY: { 580 const CgIrAggAux* aux = (const CgIrAggAux*)in->extra.aux; 581 Operand dst = wasm_ir_source_op(e, in->opnds[0], in->loc); 582 Operand src = wasm_ir_source_op(e, in->opnds[1], in->loc); 583 wasm_copy_bytes(t, dst, src, aux->access); 584 return; 585 } 586 case CG_IR_AGG_SET: { 587 const CgIrAggAux* aux = (const CgIrAggAux*)in->extra.aux; 588 Operand dst = wasm_ir_source_op(e, in->opnds[0], in->loc); 589 Operand byte = wasm_ir_source_op(e, in->opnds[1], in->loc); 590 wasm_set_bytes(t, dst, byte, aux->access); 591 return; 592 } 593 case CG_IR_BITFIELD_LOAD: 594 wasm_ir_emit_bitfield_load(e, in); 595 return; 596 case CG_IR_BITFIELD_STORE: 597 wasm_ir_emit_bitfield_store(e, in); 598 return; 599 case CG_IR_BINOP: { 600 WasmIrDest d; 601 Operand a = wasm_ir_source_op(e, in->opnds[1], in->loc); 602 Operand b = wasm_ir_source_op(e, in->opnds[2], in->loc); 603 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 604 wasm_binop(t, (BinOp)in->extra.imm, dst, a, b); 605 wasm_ir_dest_finish(e, &d); 606 return; 607 } 608 case CG_IR_UNOP: { 609 WasmIrDest d; 610 Operand a = wasm_ir_source_op(e, in->opnds[1], in->loc); 611 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 612 wasm_unop(t, (UnOp)in->extra.imm, dst, a); 613 wasm_ir_dest_finish(e, &d); 614 return; 615 } 616 case CG_IR_CMP: { 617 WasmIrDest d; 618 Operand a = wasm_ir_source_op(e, in->opnds[1], in->loc); 619 Operand b = wasm_ir_source_op(e, in->opnds[2], in->loc); 620 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 621 wasm_cmp(t, (CmpOp)in->extra.imm, dst, a, b); 622 wasm_ir_dest_finish(e, &d); 623 return; 624 } 625 case CG_IR_CONVERT: { 626 WasmIrDest d; 627 Operand a = wasm_ir_source_op(e, in->opnds[1], in->loc); 628 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 629 wasm_convert(t, (ConvKind)in->extra.imm, dst, a); 630 wasm_ir_dest_finish(e, &d); 631 return; 632 } 633 case CG_IR_CALL: 634 wasm_ir_emit_call(e, in); 635 return; 636 case CG_IR_RET: 637 wasm_ir_emit_ret(e, f, in); 638 return; 639 case CG_IR_UNREACHABLE: 640 wasm_unreachable(t); 641 return; 642 case CG_IR_BR: 643 wasm_jump(t, (Label)in->extra.imm); 644 return; 645 case CG_IR_CMP_BRANCH: { 646 const CgIrCmpBranchAux* aux = (const CgIrCmpBranchAux*)in->extra.aux; 647 /* Use source_op, not value_op: a compared operand may be an 648 * address-taken local that lives in linear memory (e.g. the `expected` 649 * out-param of __atomic_compare_exchange), which must be loaded rather 650 * than read as a bare wasm local. */ 651 wasm_cmp_branch(t, aux->op, wasm_ir_source_op(e, in->opnds[0], in->loc), 652 wasm_ir_source_op(e, in->opnds[1], in->loc), aux->target); 653 return; 654 } 655 case CG_IR_SWITCH: 656 wasm_ir_emit_switch(e, in); 657 return; 658 case CG_IR_INDIRECT_BRANCH: 659 wasm_ir_fail(e, in->loc, 660 "wasm target: indirect_branch (computed goto) not yet " 661 "implemented"); 662 return; 663 case CG_IR_LOAD_LABEL_ADDR: 664 wasm_ir_fail(e, in->loc, 665 "wasm target: load_label_addr (&&label) not yet " 666 "implemented"); 667 return; 668 case CG_IR_LOCAL_STATIC_DATA_BEGIN: 669 case CG_IR_LOCAL_STATIC_DATA_WRITE: 670 case CG_IR_LOCAL_STATIC_DATA_LABEL_ADDR: 671 case CG_IR_LOCAL_STATIC_DATA_END: 672 wasm_ir_fail(e, in->loc, 673 "wasm target: function-local static data not yet " 674 "implemented"); 675 return; 676 case CG_IR_SCOPE_BEGIN: { 677 const CgIrScopeAux* aux = (const CgIrScopeAux*)in->extra.aux; 678 CGScopeDesc d; 679 memset(&d, 0, sizeof d); 680 d.kind = aux->desc.kind; 681 d.break_label = aux->desc.break_label; 682 d.continue_label = aux->desc.continue_label; 683 d.result_type = aux->desc.result_type; 684 wasm_ir_bind_scope(e, aux->scope, wasm_scope_begin(t, &d), in->loc); 685 return; 686 } 687 case CG_IR_SCOPE_END: 688 wasm_scope_end(t, 689 wasm_ir_scope_lookup(e, (CGScope)in->extra.imm, in->loc)); 690 return; 691 case CG_IR_BREAK_TO: 692 wasm_break_to(t, 693 wasm_ir_scope_lookup(e, (CGScope)in->extra.imm, in->loc)); 694 return; 695 case CG_IR_CONTINUE_TO: 696 wasm_continue_to( 697 t, wasm_ir_scope_lookup(e, (CGScope)in->extra.imm, in->loc)); 698 return; 699 case CG_IR_ALLOCA: { 700 WasmIrDest d; 701 Operand size = wasm_ir_source_op(e, in->opnds[1], in->loc); 702 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 703 wasm_alloca(t, dst, size, (u32)in->extra.imm); 704 wasm_ir_dest_finish(e, &d); 705 return; 706 } 707 case CG_IR_VA_START: 708 wasm_va_start(t, wasm_ir_source_op(e, in->opnds[0], in->loc)); 709 return; 710 case CG_IR_VA_ARG: { 711 WasmIrDest d; 712 Operand ap = wasm_ir_source_op(e, in->opnds[1], in->loc); 713 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 714 wasm_va_arg(t, dst, ap, (KitCgTypeId)in->extra.imm); 715 wasm_ir_dest_finish(e, &d); 716 return; 717 } 718 case CG_IR_VA_END: 719 wasm_va_end(t, wasm_ir_source_op(e, in->opnds[0], in->loc)); 720 return; 721 case CG_IR_VA_COPY: { 722 Operand src = wasm_ir_source_op(e, in->opnds[1], in->loc); 723 Operand dst = wasm_ir_source_op(e, in->opnds[0], in->loc); 724 wasm_va_copy(t, dst, src); 725 return; 726 } 727 case CG_IR_ATOMIC_LOAD: { 728 const CgIrAtomicAux* aux = (const CgIrAtomicAux*)in->extra.aux; 729 WasmIrDest d; 730 Operand addr = wasm_ir_source_op(e, in->opnds[1], in->loc); 731 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 732 wasm_atomic_load(t, dst, addr, aux->mem, aux->order); 733 wasm_ir_dest_finish(e, &d); 734 return; 735 } 736 case CG_IR_ATOMIC_STORE: { 737 const CgIrAtomicAux* aux = (const CgIrAtomicAux*)in->extra.aux; 738 Operand addr = wasm_ir_source_op(e, in->opnds[0], in->loc); 739 Operand val = wasm_ir_source_op(e, in->opnds[1], in->loc); 740 wasm_atomic_store(t, addr, val, aux->mem, aux->order); 741 return; 742 } 743 case CG_IR_ATOMIC_RMW: { 744 const CgIrAtomicAux* aux = (const CgIrAtomicAux*)in->extra.aux; 745 WasmIrDest d; 746 Operand addr = wasm_ir_source_op(e, in->opnds[1], in->loc); 747 Operand val = wasm_ir_source_op(e, in->opnds[2], in->loc); 748 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 749 wasm_atomic_rmw(t, aux->op, dst, addr, val, aux->mem, aux->order); 750 wasm_ir_dest_finish(e, &d); 751 return; 752 } 753 case CG_IR_ATOMIC_CAS: { 754 const CgIrAtomicAux* aux = (const CgIrAtomicAux*)in->extra.aux; 755 WasmIrDest dprior, dok; 756 Operand addr = wasm_ir_source_op(e, in->opnds[2], in->loc); 757 Operand expected = wasm_ir_source_op(e, in->opnds[3], in->loc); 758 Operand desired = wasm_ir_source_op(e, in->opnds[4], in->loc); 759 Operand prior = wasm_ir_dest_op(e, in->opnds[0], &dprior); 760 Operand ok = wasm_ir_dest_op(e, in->opnds[1], &dok); 761 wasm_atomic_cas(t, prior, ok, addr, expected, desired, aux->mem, 762 aux->order, aux->failure); 763 wasm_ir_dest_finish(e, &dprior); 764 wasm_ir_dest_finish(e, &dok); 765 return; 766 } 767 case CG_IR_FENCE: 768 wasm_fence(t, (KitCgMemOrder)in->extra.imm); 769 return; 770 case CG_IR_INTRINSIC: { 771 const CgIrIntrinsicAux* aux = (const CgIrIntrinsicAux*)in->extra.aux; 772 Heap* h = e->target->c->ctx->heap; 773 Operand* dsts = NULL; 774 Operand* args = NULL; 775 WasmIrDest* dd = NULL; 776 if (aux->ndst) { 777 dsts = 778 (Operand*)h->alloc(h, sizeof(*dsts) * aux->ndst, _Alignof(Operand)); 779 dd = (WasmIrDest*)h->alloc(h, sizeof(*dd) * aux->ndst, 780 _Alignof(WasmIrDest)); 781 if (!dsts || !dd) 782 wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 783 } 784 if (aux->narg) { 785 args = 786 (Operand*)h->alloc(h, sizeof(*args) * aux->narg, _Alignof(Operand)); 787 if (!args) wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 788 for (u32 i = 0; i < aux->narg; ++i) 789 args[i] = wasm_ir_source_op(e, aux->args[i], in->loc); 790 } 791 for (u32 i = 0; i < aux->ndst; ++i) 792 dsts[i] = wasm_ir_dest_op(e, aux->dsts[i], &dd[i]); 793 wasm_intrinsic(t, aux->kind, dsts, aux->ndst, args, aux->narg); 794 for (u32 i = 0; i < aux->ndst; ++i) wasm_ir_dest_finish(e, &dd[i]); 795 if (dsts) h->free(h, dsts, sizeof(*dsts) * aux->ndst); 796 if (dd) h->free(h, dd, sizeof(*dd) * aux->ndst); 797 if (args) h->free(h, args, sizeof(*args) * aux->narg); 798 return; 799 } 800 case CG_IR_ASM_BLOCK: { 801 const CgIrAsmAux* aux = (const CgIrAsmAux*)in->extra.aux; 802 Heap* h = e->target->c->ctx->heap; 803 Operand* outs = NULL; 804 Operand* ins = NULL; 805 WasmIrDest* od = NULL; 806 if (aux->nout) { 807 outs = 808 (Operand*)h->alloc(h, sizeof(*outs) * aux->nout, _Alignof(Operand)); 809 od = (WasmIrDest*)h->alloc(h, sizeof(*od) * aux->nout, 810 _Alignof(WasmIrDest)); 811 if (!outs || !od) 812 wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 813 } 814 if (aux->nin) { 815 ins = (Operand*)h->alloc(h, sizeof(*ins) * aux->nin, _Alignof(Operand)); 816 if (!ins) wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 817 for (u32 i = 0; i < aux->nin; ++i) 818 ins[i] = wasm_ir_source_op(e, aux->in_ops[i], in->loc); 819 } 820 for (u32 i = 0; i < aux->nout; ++i) 821 outs[i] = wasm_ir_dest_op(e, aux->out_ops[i], &od[i]); 822 wasm_asm_block(t, aux->tmpl, aux->outs, aux->nout, outs, aux->ins, 823 aux->nin, ins, aux->clobbers, aux->nclob); 824 for (u32 i = 0; i < aux->nout; ++i) wasm_ir_dest_finish(e, &od[i]); 825 if (outs) h->free(h, outs, sizeof(*outs) * aux->nout); 826 if (od) h->free(h, od, sizeof(*od) * aux->nout); 827 if (ins) h->free(h, ins, sizeof(*ins) * aux->nin); 828 return; 829 } 830 } 831 wasm_ir_fail(e, in->loc, "wasm IR emit: unknown op"); 832 } 833 834 static void wasm_ir_emit_func(WTarget* t, const CgIrFunc* f) { 835 Heap* h = t->c->ctx->heap; 836 WasmIrEmitter e; 837 CGFuncDesc fd; 838 CGParamDesc* params = NULL; 839 memset(&e, 0, sizeof e); 840 e.target = t; 841 e.next_temp_reg = (Reg)(f->nlocals + 1u); 842 e.local_slots_n = f->nlocals + 1u; 843 e.local_slots = (FrameSlot*)h->alloc(h, sizeof(FrameSlot) * e.local_slots_n, 844 _Alignof(FrameSlot)); 845 if (!e.local_slots) 846 compiler_panic(t->c, f->desc.loc, "wasm IR emit: out of memory"); 847 for (u32 i = 0; i < e.local_slots_n; ++i) e.local_slots[i] = FRAME_SLOT_NONE; 848 e.local_types = (KitCgTypeId*)h->alloc( 849 h, sizeof(KitCgTypeId) * e.local_slots_n, _Alignof(KitCgTypeId)); 850 if (!e.local_types) 851 compiler_panic(t->c, f->desc.loc, "wasm IR emit: out of memory"); 852 for (u32 i = 0; i < e.local_slots_n; ++i) e.local_types[i] = KIT_CG_TYPE_NONE; 853 e.scope_map_n = f->nscopes + 1u; 854 if (e.scope_map_n) { 855 e.scope_map = (CGScope*)h->alloc(h, sizeof(CGScope) * e.scope_map_n, 856 _Alignof(CGScope)); 857 if (!e.scope_map) 858 compiler_panic(t->c, f->desc.loc, "wasm IR emit: out of memory"); 859 memset(e.scope_map, 0, sizeof(CGScope) * e.scope_map_n); 860 } 861 862 memset(&fd, 0, sizeof fd); 863 fd.sym = f->desc.sym; 864 fd.text_section_id = f->desc.text_section_id; 865 fd.group_id = f->desc.group_id; 866 fd.fn_type = f->desc.fn_type; 867 fd.result_type = f->desc.result_type; 868 fd.nparams = f->desc.nparams; 869 fd.loc = f->desc.loc; 870 fd.flags = f->desc.flags; 871 fd.inline_policy = f->desc.inline_policy; 872 fd.atomize = f->desc.atomize; 873 fd.abi = abi_cg_func_info(t->c->abi, f->desc.fn_type); 874 if (f->nparams) { 875 params = (CGParamDesc*)h->alloc(h, sizeof(*params) * f->nparams, 876 _Alignof(CGParamDesc)); 877 if (!params) 878 compiler_panic(t->c, f->desc.loc, "wasm IR emit: out of memory"); 879 for (u32 i = 0; i < f->nparams; ++i) { 880 const CgIrParam* src = &f->params[i]; 881 memset(¶ms[i], 0, sizeof params[i]); 882 params[i].index = src->desc.index; 883 params[i].name = src->desc.name; 884 params[i].type = src->desc.type; 885 params[i].size = src->desc.size; 886 params[i].align = src->desc.align; 887 params[i].flags = src->desc.flags; 888 params[i].loc = src->desc.loc; 889 params[i].abi = wasm_ir_param_abi(fd.abi, src->desc.index); 890 } 891 fd.params = params; 892 } 893 894 wasm_func_begin((CGTarget*)&t->base, &fd); 895 for (u32 i = 0; i < f->nlabels; ++i) 896 (void)wasm_label_new((CGTarget*)&t->base); 897 for (u32 i = 0; i < f->nparams; ++i) { 898 const CgIrParam* p = &f->params[i]; 899 CGLocalStorage st = wasm_param((CGTarget*)&t->base, ¶ms[i]); 900 if (p->local < e.local_slots_n) { 901 e.local_slots[p->local] = st.v.frame_slot; 902 e.local_types[p->local] = p->desc.type; 903 } 904 wasm_ir_bind_value_local(&e, p->local, p->desc.type, st.v.frame_slot); 905 } 906 for (u32 i = 0; i < f->nlocals; ++i) { 907 const CgIrLocal* l = &f->locals[i]; 908 if (l->is_param) continue; 909 CGLocalStorage st = wasm_local((CGTarget*)&t->base, &l->desc); 910 if (l->id < e.local_slots_n) { 911 e.local_slots[l->id] = st.v.frame_slot; 912 e.local_types[l->id] = l->desc.type; 913 } 914 wasm_ir_bind_value_local(&e, l->id, l->desc.type, st.v.frame_slot); 915 } 916 for (u32 i = 0; i < f->ninsts; ++i) wasm_ir_emit_inst(&e, f, &f->insts[i]); 917 wasm_func_end((CGTarget*)&t->base); 918 919 if (params) h->free(h, params, sizeof(*params) * f->nparams); 920 if (e.scope_map) h->free(h, e.scope_map, sizeof(CGScope) * e.scope_map_n); 921 h->free(h, e.local_types, sizeof(KitCgTypeId) * e.local_slots_n); 922 h->free(h, e.local_slots, sizeof(FrameSlot) * e.local_slots_n); 923 } 924 925 void wasm_emit_ir_module(WTarget* t, const CgIrModule* module) { 926 if (!t || !module) return; 927 /* Process aliases before emitting function bodies. Function linearization is 928 * eager (wasm_func_end), so a call through a function alias resolves its 929 * wasm func index at emit time; if the alias->target mapping isn't installed 930 * first, the call allocates a fresh empty func for the alias (bad body -> 931 * "function result type mismatch") instead of dispatching to the target. 932 * Data aliases only need the target's (section_id,value), shared earlier at 933 * the ObjBuilder layer, so they are equally safe here. */ 934 for (u32 i = 0; i < module->naliases; ++i) { 935 const CgIrAlias* a = &module->aliases[i]; 936 wasm_alias((CGTarget*)&t->base, a->alias_sym, a->target_sym, a->type); 937 } 938 for (u32 i = 0; i < module->nfuncs; ++i) { 939 wasm_ir_emit_func(t, module->funcs[i]); 940 } 941 }