ir_emit.c (36693B)
1 #include <string.h> 2 3 #include "cg/ir.h" 4 5 typedef Operand CgSemOperand; 6 typedef CGCallDesc CgSemCallDesc; 7 typedef CGFuncDesc CgSemFuncDesc; 8 typedef CGParamDesc CgSemParamDesc; 9 typedef CGScopeDesc CgSemScopeDesc; 10 11 #include "arch/wasm/internal.h" 12 #include "cg/type.h" 13 #include "core/heap.h" 14 15 void wasm_func_begin(CGTarget*, const CGFuncDesc*); 16 void wasm_func_end(CGTarget*); 17 void wasm_alias(CGTarget*, ObjSymId, ObjSymId, KitCgTypeId); 18 CGLocalStorage wasm_param(CGTarget*, const CGParamDesc*); 19 CGLocalStorage wasm_local(CGTarget*, const CGLocalDesc*); 20 Label wasm_label_new(CGTarget*); 21 void wasm_label_place(CGTarget*, Label); 22 void wasm_jump(CGTarget*, Label); 23 void wasm_cmp_branch(CGTarget*, CmpOp, Operand, Operand, Label); 24 void wasm_switch(CGTarget*, const CGSwitchDesc*); 25 CGScope wasm_scope_begin(CGTarget*, const CGScopeDesc*); 26 void wasm_scope_end(CGTarget*, CGScope); 27 void wasm_break_to(CGTarget*, CGScope); 28 void wasm_continue_to(CGTarget*, CGScope); 29 void wasm_set_loc(CGTarget*, SrcLoc); 30 void wasm_load_imm(CGTarget*, Operand, i64); 31 void wasm_load_const(CGTarget*, Operand, ConstBytes); 32 void wasm_copy(CGTarget*, Operand, Operand); 33 void wasm_load(CGTarget*, Operand, Operand, MemAccess); 34 void wasm_store(CGTarget*, Operand, Operand, MemAccess); 35 void wasm_addr_of(CGTarget*, Operand, Operand); 36 void wasm_copy_bytes(CGTarget*, Operand, Operand, AggregateAccess); 37 void wasm_set_bytes(CGTarget*, Operand, Operand, AggregateAccess); 38 void wasm_binop(CGTarget*, BinOp, Operand, Operand, Operand); 39 void wasm_unop(CGTarget*, UnOp, Operand, Operand); 40 void wasm_cmp(CGTarget*, CmpOp, Operand, Operand, Operand); 41 void wasm_convert(CGTarget*, ConvKind, Operand, Operand); 42 void wasm_call(CGTarget*, const CGCallDesc*); 43 void wasm_ret(CGTarget*, const CGABIValue*); 44 void wasm_unreachable(CGTarget*); 45 void wasm_alloca(CGTarget*, Operand, Operand, u32); 46 void wasm_va_start(CGTarget*, Operand); 47 void wasm_va_arg(CGTarget*, Operand, Operand, KitCgTypeId); 48 void wasm_va_end(CGTarget*, Operand); 49 void wasm_va_copy(CGTarget*, Operand, Operand); 50 void wasm_atomic_load(CGTarget*, Operand, Operand, MemAccess, KitCgMemOrder); 51 void wasm_atomic_store(CGTarget*, Operand, Operand, MemAccess, KitCgMemOrder); 52 void wasm_atomic_rmw(CGTarget*, KitCgAtomicOp, Operand, Operand, Operand, 53 MemAccess, KitCgMemOrder); 54 void wasm_atomic_cas(CGTarget*, Operand, Operand, Operand, Operand, Operand, 55 MemAccess, KitCgMemOrder, KitCgMemOrder); 56 void wasm_fence(CGTarget*, KitCgMemOrder); 57 void wasm_intrinsic(CGTarget*, IntrinKind, Operand*, u32, const Operand*, u32); 58 void wasm_asm_block(CGTarget*, const char*, const AsmConstraint*, u32, Operand*, 59 const AsmConstraint*, u32, const Operand*, const Sym*, u32); 60 61 typedef struct WasmIrEmitter { 62 WTarget* target; 63 FrameSlot* local_slots; 64 KitCgTypeId* local_types; /* CG type of each local id, parallel to slots */ 65 u32 local_slots_n; 66 Reg next_temp_reg; 67 CGScope* scope_map; 68 u32 scope_map_n; 69 } WasmIrEmitter; 70 71 static void wasm_ir_fail(WasmIrEmitter* e, SrcLoc loc, const char* msg) { 72 compiler_panic(e->target->c, loc, "%s", msg); 73 } 74 75 static RegClass wasm_ir_class_for_type(WTarget* t, KitCgTypeId type) { 76 ABITypeInfo info; 77 if (!type) return RC_INT; 78 info = abi_cg_type_info(t->c->abi, type); 79 return info.scalar_kind == ABI_SC_FLOAT ? RC_FP : RC_INT; 80 } 81 82 /* An aggregate (record/array) value lives in linear memory on wasm: a scalar 83 * COPY of it must be lowered to a byte-wise memcpy between the two homes. */ 84 static int wasm_ir_is_aggregate(WTarget* t, KitCgTypeId ty) { 85 ABITypeInfo info; 86 if (!ty) return 0; 87 info = abi_cg_type_info(t->c->abi, ty); 88 if (info.scalar_kind == ABI_SC_PTR) return 0; 89 if (info.size == 0) return 0; /* genuine void */ 90 return info.scalar_kind == ABI_SC_VOID || info.size > 8u; 91 } 92 93 static void wasm_ir_bind_reg(WasmIrEmitter* e, Reg reg, u32 wasm_local, 94 KitCgTypeId type) { 95 WTarget* t = e->target; 96 Heap* h = t->c->ctx->heap; 97 if (reg == REG_NONE) return; 98 if (reg >= t->reg_cap) { 99 u32 nc = t->reg_cap ? t->reg_cap : 64u; 100 while (nc <= reg) nc *= 2u; 101 u32* regs = (u32*)h->realloc(h, t->reg_to_local, sizeof(u32) * t->reg_cap, 102 sizeof(u32) * nc, _Alignof(u32)); 103 KitCgTypeId* types = (KitCgTypeId*)h->realloc( 104 h, t->reg_type, sizeof(KitCgTypeId) * t->reg_cap, 105 sizeof(KitCgTypeId) * nc, _Alignof(KitCgTypeId)); 106 u8* cls = (u8*)h->realloc(h, t->reg_cls, t->reg_cap, nc, 1); 107 if (!regs || !types || !cls) 108 compiler_panic(t->c, (SrcLoc){0, 0, 0}, "wasm IR emit: out of memory"); 109 for (u32 i = t->reg_cap; i < nc; ++i) { 110 regs[i] = 0xffffffffu; 111 types[i] = 0; 112 cls[i] = 0; 113 } 114 t->reg_to_local = regs; 115 t->reg_type = types; 116 t->reg_cls = cls; 117 t->reg_cap = nc; 118 } 119 /* A NONE type carries no wasm value type. This shows up for variadic call 120 * arguments, whose operands are untyped at the call site. Don't let it 121 * clobber a binding the locals/params pass already gave a real type. */ 122 if (!type && t->reg_to_local[reg] != 0xffffffffu && t->reg_type[reg]) return; 123 t->reg_to_local[reg] = wasm_local; 124 t->reg_type[reg] = type; 125 t->reg_cls[reg] = (u8)wasm_ir_class_for_type(t, type); 126 } 127 128 static void wasm_ir_bind_value_local(WasmIrEmitter* e, CGLocal local, 129 KitCgTypeId type, FrameSlot slot) { 130 WSlot* s; 131 if (slot == FRAME_SLOT_NONE) return; 132 s = &e->target->slots[slot - 1u]; 133 if (s->kind == W_SLOT_LOCAL) 134 wasm_ir_bind_reg(e, (Reg)local, s->wasm_local, type); 135 } 136 137 static Operand wasm_ir_value_op(WasmIrEmitter* e, CgSemOperand in) { 138 Operand out; 139 memset(&out, 0, sizeof out); 140 out.kind = in.kind; 141 out.type = in.type; 142 out.cls = (u8)wasm_ir_class_for_type(e->target, in.type); 143 switch ((OpKind)in.kind) { 144 case OPK_IMM: 145 out.v.imm = in.v.imm; 146 return out; 147 case OPK_LOCAL: 148 out.kind = OPK_REG; 149 out.v.reg = (Reg)in.v.local; 150 return out; 151 case OPK_GLOBAL: 152 out.v.global.sym = in.v.global.sym; 153 out.v.global.addend = in.v.global.addend; 154 return out; 155 case OPK_INDIRECT: 156 out.v.ind.base = (Reg)in.v.ind.base; 157 out.v.ind.index = 158 in.v.ind.index == CG_LOCAL_NONE ? REG_NONE : (Reg)in.v.ind.index; 159 out.v.ind.log2_scale = in.v.ind.log2_scale; 160 out.v.ind.ofs = in.v.ind.ofs; 161 return out; 162 } 163 return out; 164 } 165 166 static Reg wasm_ir_temp_reg(WasmIrEmitter* e) { return e->next_temp_reg++; } 167 168 static Operand wasm_ir_source_op(WasmIrEmitter* e, CgSemOperand in, 169 SrcLoc loc) { 170 Operand out; 171 if (in.kind == OPK_LOCAL && in.v.local < e->local_slots_n) { 172 FrameSlot slot = e->local_slots[in.v.local]; 173 if (slot != FRAME_SLOT_NONE) { 174 WSlot* s = &e->target->slots[slot - 1u]; 175 if (s->kind == W_SLOT_LOCAL) { 176 wasm_ir_bind_value_local(e, in.v.local, in.type, slot); 177 } else { 178 MemAccess mem; 179 Operand addr; 180 memset(&mem, 0, sizeof mem); 181 memset(&addr, 0, sizeof addr); 182 out = wasm_ir_value_op(e, in); 183 out.v.reg = wasm_ir_temp_reg(e); 184 addr.kind = OPK_LOCAL; 185 addr.type = in.type; 186 addr.v.frame_slot = slot; 187 mem.type = in.type; 188 mem.size = s->size; 189 mem.align = s->align; 190 wasm_load((CGTarget*)&e->target->base, out, addr, mem); 191 return out; 192 } 193 } 194 } 195 (void)loc; 196 return wasm_ir_value_op(e, in); 197 } 198 199 /* A value-producing instruction whose CG destination is an address-taken 200 * (stack-homed) local cannot write a wasm local directly: the variable lives 201 * in linear memory so its address is meaningful. Route the def through a temp 202 * reg and store it back to the slot once the underlying op has run. */ 203 typedef struct WasmIrDest { 204 int spill; 205 Operand tmp; /* OPK_REG temp the op writes into */ 206 Operand store_addr; /* OPK_LOCAL frame slot to store back to */ 207 MemAccess mem; 208 } WasmIrDest; 209 210 static Operand wasm_ir_dest_op(WasmIrEmitter* e, CgSemOperand in, 211 WasmIrDest* d) { 212 memset(d, 0, sizeof *d); 213 if (in.kind == OPK_LOCAL && in.v.local < e->local_slots_n) { 214 FrameSlot slot = e->local_slots[in.v.local]; 215 if (slot != FRAME_SLOT_NONE) { 216 WSlot* s = &e->target->slots[slot - 1u]; 217 if (s->kind != W_SLOT_LOCAL) { 218 Operand tmp = wasm_ir_value_op(e, in); 219 tmp.kind = OPK_REG; 220 tmp.v.reg = wasm_ir_temp_reg(e); 221 d->spill = 1; 222 d->tmp = tmp; 223 memset(&d->store_addr, 0, sizeof d->store_addr); 224 d->store_addr.kind = OPK_LOCAL; 225 d->store_addr.type = in.type; 226 d->store_addr.v.frame_slot = slot; 227 memset(&d->mem, 0, sizeof d->mem); 228 d->mem.type = in.type; 229 d->mem.size = s->size; 230 d->mem.align = s->align; 231 return tmp; 232 } 233 } 234 } 235 return wasm_ir_value_op(e, in); 236 } 237 238 static void wasm_ir_dest_finish(WasmIrEmitter* e, const WasmIrDest* d) { 239 if (d->spill) 240 wasm_store((CGTarget*)&e->target->base, d->store_addr, d->tmp, d->mem); 241 } 242 243 static Operand wasm_ir_addr_op(WasmIrEmitter* e, CgSemOperand in, SrcLoc loc) { 244 Operand out; 245 if (in.kind != OPK_LOCAL) return wasm_ir_value_op(e, in); 246 if (in.v.local == CG_LOCAL_NONE || in.v.local >= e->local_slots_n || 247 e->local_slots[in.v.local] == FRAME_SLOT_NONE) { 248 wasm_ir_fail(e, loc, "wasm IR emit: unknown local address"); 249 } 250 memset(&out, 0, sizeof out); 251 out.kind = OPK_LOCAL; 252 out.type = in.type; 253 out.v.frame_slot = e->local_slots[in.v.local]; 254 return out; 255 } 256 257 /* Lower an aggregate move to a memory.copy between two linear-memory homes. 258 * Both `dst` and `src` are lvalue operands (frame slots, indirect `[ptr]` 259 * addressing, or global symbols); copy_bytes wants each endpoint as a 260 * pointer-valued register, so materialize the effective address of each with 261 * addr_of first. `ty` names the aggregate being moved. */ 262 static void wasm_ir_emit_agg_move(WasmIrEmitter* e, CgSemOperand dst, 263 CgSemOperand src, KitCgTypeId ty, 264 SrcLoc loc) { 265 CGTarget* t = (CGTarget*)&e->target->base; 266 AggregateAccess agg; 267 KitCgTypeId pty = cg_type_ptr_to(e->target->c, ty); 268 Operand adst = wasm_ir_addr_op(e, dst, loc); 269 Operand asrc = wasm_ir_addr_op(e, src, loc); 270 Operand dreg, sreg; 271 memset(&dreg, 0, sizeof dreg); 272 memset(&sreg, 0, sizeof sreg); 273 dreg.kind = sreg.kind = OPK_REG; 274 dreg.type = sreg.type = pty; 275 dreg.cls = sreg.cls = (u8)RC_INT; 276 dreg.v.reg = wasm_ir_temp_reg(e); 277 sreg.v.reg = wasm_ir_temp_reg(e); 278 wasm_addr_of(t, dreg, adst); 279 wasm_addr_of(t, sreg, asrc); 280 memset(&agg, 0, sizeof agg); 281 agg.type = ty; 282 agg.size = (u32)abi_cg_sizeof(e->target->c->abi, ty); 283 agg.align = (u32)abi_cg_alignof(e->target->c->abi, ty); 284 wasm_copy_bytes(t, dreg, sreg, agg); 285 } 286 287 static CGScope wasm_ir_scope_lookup(WasmIrEmitter* e, CGScope recorded, 288 SrcLoc loc) { 289 if ((u32)recorded >= e->scope_map_n || !e->scope_map[recorded]) 290 wasm_ir_fail(e, loc, "wasm IR emit: unknown recorded scope"); 291 return e->scope_map[recorded]; 292 } 293 294 static void wasm_ir_bind_scope(WasmIrEmitter* e, CGScope recorded, 295 CGScope emitted, SrcLoc loc) { 296 if ((u32)recorded >= e->scope_map_n) 297 wasm_ir_fail(e, loc, "wasm IR emit: recorded scope out of range"); 298 e->scope_map[recorded] = emitted; 299 } 300 301 static const ABIArgInfo* wasm_ir_param_abi(const ABIFuncInfo* abi, u32 index) { 302 return abi && index < abi->nparams ? &abi->params[index] : NULL; 303 } 304 305 static CGABIValue wasm_ir_abi_value(WasmIrEmitter* e, CGLocal local, 306 KitCgTypeId type, const ABIArgInfo* abi, 307 int address, int source, SrcLoc loc) { 308 CGABIValue out; 309 CgSemOperand sem; 310 memset(&out, 0, sizeof out); 311 memset(&sem, 0, sizeof sem); 312 sem.kind = OPK_LOCAL; 313 sem.type = type; 314 sem.v.local = local; 315 out.type = type; 316 out.abi = abi; 317 out.storage = address ? wasm_ir_addr_op(e, sem, loc) 318 : source ? wasm_ir_source_op(e, sem, loc) 319 : wasm_ir_value_op(e, sem); 320 return out; 321 } 322 323 static void wasm_ir_emit_call(WasmIrEmitter* e, const CgIrInst* in) { 324 const CgIrCallAux* aux = (const CgIrCallAux*)in->extra.aux; 325 const CgSemCallDesc* src = &aux->desc; 326 Heap* h = e->target->c->ctx->heap; 327 const ABIFuncInfo* abi = abi_cg_func_info(e->target->c->abi, src->fn_type); 328 CGCallDesc d; 329 CGABIValue* args = NULL; 330 KitCgTypeId ret_type = cg_type_func_ret_id(e->target->c, src->fn_type); 331 memset(&d, 0, sizeof d); 332 d.fn_type = src->fn_type; 333 d.callee = wasm_ir_value_op(e, src->callee); 334 d.nargs = src->nargs; 335 d.flags = src->flags; 336 d.tail_policy = src->tail_policy; 337 d.inline_policy = src->inline_policy; 338 d.abi = abi; 339 if (src->nargs) { 340 args = (CGABIValue*)h->alloc(h, sizeof(*args) * src->nargs, 341 _Alignof(CGABIValue)); 342 if (!args) wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 343 for (u32 i = 0; i < src->nargs; ++i) { 344 const ABIArgInfo* ai = wasm_ir_param_abi(abi, i); 345 KitCgTypeId ty = KIT_CG_TYPE_NONE; 346 const CgType* fty = cg_type_get(e->target->c, src->fn_type); 347 if (fty && fty->kind == KIT_CG_TYPE_FUNC && i < fty->func.nparams) 348 ty = fty->func.params[i].type; 349 /* Variadic arguments have no signature slot; recover the type from the 350 * passed local so the call's vararg packing knows its wasm value type. */ 351 if (ty == KIT_CG_TYPE_NONE && src->args[i] != CG_LOCAL_NONE && 352 (u32)src->args[i] < e->local_slots_n) 353 ty = e->local_types[src->args[i]]; 354 args[i] = 355 wasm_ir_abi_value(e, src->args[i], ty, ai, 356 ai && ai->kind == ABI_ARG_INDIRECT, 1, in->loc); 357 } 358 } 359 d.args = args; 360 if (src->result != CG_LOCAL_NONE) { 361 d.ret = wasm_ir_abi_value(e, src->result, ret_type, abi ? &abi->ret : NULL, 362 abi && abi->has_sret, 0, in->loc); 363 } 364 wasm_call((CGTarget*)&e->target->base, &d); 365 if (args) h->free(h, args, sizeof(*args) * src->nargs); 366 } 367 368 static void wasm_ir_emit_ret(WasmIrEmitter* e, const CgIrFunc* f, 369 const CgIrInst* in) { 370 const CgIrRetAux* aux = (const CgIrRetAux*)in->extra.aux; 371 const ABIFuncInfo* abi = abi_cg_func_info(e->target->c->abi, f->desc.fn_type); 372 KitCgTypeId ret_type = cg_type_func_ret_id(e->target->c, f->desc.fn_type); 373 CGABIValue ret; 374 if (!aux || !aux->present) { 375 wasm_ret((CGTarget*)&e->target->base, NULL); 376 return; 377 } 378 ret = wasm_ir_abi_value(e, aux->value, ret_type, abi ? &abi->ret : NULL, 379 abi && abi->has_sret, 1, in->loc); 380 wasm_ret((CGTarget*)&e->target->base, &ret); 381 } 382 383 static void wasm_ir_emit_switch(WasmIrEmitter* e, const CgIrInst* in) { 384 const CgIrSwitchAux* aux = (const CgIrSwitchAux*)in->extra.aux; 385 CGSwitchDesc d; 386 memset(&d, 0, sizeof d); 387 d.selector = wasm_ir_source_op(e, in->opnds[0], in->loc); 388 d.selector_type = aux->selector_type; 389 d.default_label = aux->default_label; 390 d.cases = aux->cases; 391 d.ncases = aux->ncases; 392 d.hint = aux->hint; 393 d.opt_level = aux->opt_level; 394 wasm_switch((CGTarget*)&e->target->base, &d); 395 } 396 397 /* Bitfields have no native wasm insert/extract, so lower to load + shift/mask 398 * + store over the storage unit. All arithmetic runs in i64 regardless of 399 * storage width: the load zero-extends into i64 (i64.load{8,16,32}_u), the 400 * store truncates back (i64.store{8,16,32}), and a uniform 64-bit shift count 401 * keeps the field-extraction math width-agnostic. storage_offset is always 0 402 * here — the frontend folds it into record_addr. */ 403 #define WASM_BF_REG_BITS 64u 404 405 static Operand wasm_ir_temp_i64(WasmIrEmitter* e) { 406 Operand o; 407 memset(&o, 0, sizeof o); 408 o.kind = OPK_REG; 409 o.type = builtin_id(KIT_CG_BUILTIN_I64); 410 o.cls = (u8)RC_INT; 411 o.v.reg = wasm_ir_temp_reg(e); 412 return o; 413 } 414 415 static Operand wasm_ir_imm_i64(i64 v) { 416 Operand o; 417 memset(&o, 0, sizeof o); 418 o.kind = OPK_IMM; 419 o.type = builtin_id(KIT_CG_BUILTIN_I64); 420 o.cls = (u8)RC_INT; 421 o.v.imm = v; 422 return o; 423 } 424 425 /* Storage-unit access: i64 value, exactly storage_size bytes wide. */ 426 static MemAccess wasm_ir_bf_storage_mem(const BitFieldAccess* bf) { 427 MemAccess mem = bf->storage; 428 mem.type = builtin_id(KIT_CG_BUILTIN_I64); 429 mem.size = bf->storage.size ? bf->storage.size : 4u; 430 return mem; 431 } 432 433 static void wasm_ir_emit_bitfield_load(WasmIrEmitter* e, const CgIrInst* in) { 434 CGTarget* t = (CGTarget*)&e->target->base; 435 const CgIrBitFieldAux* aux = (const CgIrBitFieldAux*)in->extra.aux; 436 const BitFieldAccess* bf = &aux->access; 437 u32 width = bf->bit_width ? bf->bit_width : 1u; 438 u32 lsb = bf->bit_offset; 439 u32 left = WASM_BF_REG_BITS - lsb - width; /* bits above the field */ 440 u32 right = WASM_BF_REG_BITS - width; /* slide field back to bit 0 */ 441 Operand addr = wasm_ir_addr_op(e, in->opnds[1], in->loc); 442 Operand val = wasm_ir_temp_i64(e); 443 WasmIrDest d; 444 Operand dst; 445 446 /* Load the storage unit, slide the field to the top of the i64, then back 447 * down — arithmetic shift sign-extends a signed field, logical zero-extends 448 * an unsigned one. */ 449 wasm_load(t, val, addr, wasm_ir_bf_storage_mem(bf)); 450 if (left) wasm_binop(t, BO_SHL, val, val, wasm_ir_imm_i64((i64)left)); 451 if (right) 452 wasm_binop(t, bf->signed_ ? BO_SHR_S : BO_SHR_U, val, val, 453 wasm_ir_imm_i64((i64)right)); 454 dst = wasm_ir_dest_op(e, in->opnds[0], &d); 455 /* Narrow to the field's wasm value type; a no-op copy when dst is i64. */ 456 wasm_convert(t, CV_TRUNC, dst, val); 457 wasm_ir_dest_finish(e, &d); 458 } 459 460 static void wasm_ir_emit_bitfield_store(WasmIrEmitter* e, const CgIrInst* in) { 461 CGTarget* t = (CGTarget*)&e->target->base; 462 const CgIrBitFieldAux* aux = (const CgIrBitFieldAux*)in->extra.aux; 463 const BitFieldAccess* bf = &aux->access; 464 u32 width = bf->bit_width ? bf->bit_width : 1u; 465 u32 lsb = bf->bit_offset; 466 u64 ones = (width >= WASM_BF_REG_BITS) ? ~(u64)0 : (((u64)1 << width) - 1u); 467 u64 mask = ones << lsb; 468 MemAccess mem = wasm_ir_bf_storage_mem(bf); 469 Operand addr = wasm_ir_addr_op(e, in->opnds[0], in->loc); 470 Operand cur = wasm_ir_temp_i64(e); 471 472 /* Read-modify-write: clear the field bits, OR in the masked/shifted value. */ 473 wasm_load(t, cur, addr, mem); 474 wasm_binop(t, BO_AND, cur, cur, wasm_ir_imm_i64((i64)~mask)); 475 if (in->opnds[1].kind == OPK_IMM) { 476 u64 v = ((u64)in->opnds[1].v.imm & ones) << lsb; 477 wasm_binop(t, BO_OR, cur, cur, wasm_ir_imm_i64((i64)v)); 478 } else { 479 Operand src = wasm_ir_source_op(e, in->opnds[1], in->loc); 480 Operand staged = wasm_ir_temp_i64(e); 481 wasm_convert(t, CV_ZEXT, staged, src); /* widen field value to i64 */ 482 wasm_binop(t, BO_AND, staged, staged, wasm_ir_imm_i64((i64)ones)); 483 if (lsb) wasm_binop(t, BO_SHL, staged, staged, wasm_ir_imm_i64((i64)lsb)); 484 wasm_binop(t, BO_OR, cur, cur, staged); 485 } 486 wasm_store(t, addr, cur, mem); 487 } 488 489 static void wasm_ir_emit_inst(WasmIrEmitter* e, const CgIrFunc* f, 490 const CgIrInst* in) { 491 CGTarget* t = (CGTarget*)&e->target->base; 492 wasm_set_loc(t, in->loc); 493 switch ((CgIrOp)in->op) { 494 case CG_IR_NOP: 495 return; 496 case CG_IR_LABEL: 497 wasm_label_place(t, (Label)in->extra.imm); 498 return; 499 case CG_IR_LOAD_IMM: { 500 WasmIrDest d; 501 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 502 wasm_load_imm(t, dst, in->extra.imm); 503 wasm_ir_dest_finish(e, &d); 504 return; 505 } 506 case CG_IR_LOAD_CONST: { 507 WasmIrDest d; 508 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 509 wasm_load_const(t, dst, in->extra.cbytes); 510 wasm_ir_dest_finish(e, &d); 511 return; 512 } 513 case CG_IR_COPY: { 514 WasmIrDest d; 515 Operand src, dst; 516 if (wasm_ir_is_aggregate(e->target, in->opnds[0].type)) { 517 wasm_ir_emit_agg_move(e, in->opnds[0], in->opnds[1], in->opnds[0].type, 518 in->loc); 519 return; 520 } 521 src = wasm_ir_source_op(e, in->opnds[1], in->loc); 522 dst = wasm_ir_dest_op(e, in->opnds[0], &d); 523 wasm_copy(t, dst, src); 524 wasm_ir_dest_finish(e, &d); 525 return; 526 } 527 case CG_IR_LOAD: { 528 WasmIrDest d; 529 Operand addr, dst; 530 if (wasm_ir_is_aggregate(e->target, in->opnds[0].type)) { 531 /* Aggregate load: the source operand is the address of the aggregate 532 * (an indirect `[ptr]` or a global symbol), so its effective address 533 * is the source home. Lower to memory.copy into the destination's 534 * home rather than a scalar wasm load. */ 535 wasm_ir_emit_agg_move(e, in->opnds[0], in->opnds[1], in->opnds[0].type, 536 in->loc); 537 return; 538 } 539 addr = wasm_ir_addr_op(e, in->opnds[1], in->loc); 540 dst = wasm_ir_dest_op(e, in->opnds[0], &d); 541 wasm_load(t, dst, addr, in->extra.mem); 542 wasm_ir_dest_finish(e, &d); 543 return; 544 } 545 case CG_IR_STORE: { 546 Operand addr = wasm_ir_addr_op(e, in->opnds[0], in->loc); 547 Operand src = wasm_ir_source_op(e, in->opnds[1], in->loc); 548 wasm_store(t, addr, src, in->extra.mem); 549 return; 550 } 551 case CG_IR_ADDR_OF: { 552 WasmIrDest d; 553 Operand addr = wasm_ir_addr_op(e, in->opnds[1], in->loc); 554 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 555 wasm_addr_of(t, dst, addr); 556 wasm_ir_dest_finish(e, &d); 557 return; 558 } 559 case CG_IR_TLS_ADDR_OF: { 560 /* Wasm has no thread-local storage: a module instance owns one linear 561 * memory, so a thread-local resolves to a fixed data address. Lower to 562 * the symbol's (addend-adjusted) linear-memory address, exactly like a 563 * non-TLS addr_of of a global. */ 564 const CgIrTlsAux* aux = (const CgIrTlsAux*)in->extra.aux; 565 WasmIrDest d; 566 Operand src; 567 Operand dst; 568 memset(&src, 0, sizeof src); 569 src.kind = OPK_GLOBAL; 570 src.type = in->opnds[0].type; 571 src.v.global.sym = aux->sym; 572 src.v.global.addend = aux->addend; 573 dst = wasm_ir_dest_op(e, in->opnds[0], &d); 574 wasm_addr_of(t, dst, src); 575 wasm_ir_dest_finish(e, &d); 576 return; 577 } 578 case CG_IR_AGG_COPY: { 579 const CgIrAggAux* aux = (const CgIrAggAux*)in->extra.aux; 580 Operand dst = wasm_ir_source_op(e, in->opnds[0], in->loc); 581 Operand src = wasm_ir_source_op(e, in->opnds[1], in->loc); 582 wasm_copy_bytes(t, dst, src, aux->access); 583 return; 584 } 585 case CG_IR_AGG_SET: { 586 const CgIrAggAux* aux = (const CgIrAggAux*)in->extra.aux; 587 Operand dst = wasm_ir_source_op(e, in->opnds[0], in->loc); 588 Operand byte = wasm_ir_source_op(e, in->opnds[1], in->loc); 589 wasm_set_bytes(t, dst, byte, aux->access); 590 return; 591 } 592 case CG_IR_BITFIELD_LOAD: 593 wasm_ir_emit_bitfield_load(e, in); 594 return; 595 case CG_IR_BITFIELD_STORE: 596 wasm_ir_emit_bitfield_store(e, in); 597 return; 598 case CG_IR_BINOP: { 599 WasmIrDest d; 600 Operand a = wasm_ir_source_op(e, in->opnds[1], in->loc); 601 Operand b = wasm_ir_source_op(e, in->opnds[2], in->loc); 602 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 603 wasm_binop(t, (BinOp)in->extra.imm, dst, a, b); 604 wasm_ir_dest_finish(e, &d); 605 return; 606 } 607 case CG_IR_UNOP: { 608 WasmIrDest d; 609 Operand a = wasm_ir_source_op(e, in->opnds[1], in->loc); 610 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 611 wasm_unop(t, (UnOp)in->extra.imm, dst, a); 612 wasm_ir_dest_finish(e, &d); 613 return; 614 } 615 case CG_IR_CMP: { 616 WasmIrDest d; 617 Operand a = wasm_ir_source_op(e, in->opnds[1], in->loc); 618 Operand b = wasm_ir_source_op(e, in->opnds[2], in->loc); 619 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 620 wasm_cmp(t, (CmpOp)in->extra.imm, dst, a, b); 621 wasm_ir_dest_finish(e, &d); 622 return; 623 } 624 case CG_IR_CONVERT: { 625 WasmIrDest d; 626 Operand a = wasm_ir_source_op(e, in->opnds[1], in->loc); 627 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 628 wasm_convert(t, (ConvKind)in->extra.imm, dst, a); 629 wasm_ir_dest_finish(e, &d); 630 return; 631 } 632 case CG_IR_CALL: 633 wasm_ir_emit_call(e, in); 634 return; 635 case CG_IR_RET: 636 wasm_ir_emit_ret(e, f, in); 637 return; 638 case CG_IR_UNREACHABLE: 639 wasm_unreachable(t); 640 return; 641 case CG_IR_BR: 642 wasm_jump(t, (Label)in->extra.imm); 643 return; 644 case CG_IR_CMP_BRANCH: { 645 const CgIrCmpBranchAux* aux = (const CgIrCmpBranchAux*)in->extra.aux; 646 /* Use source_op, not value_op: a compared operand may be an 647 * address-taken local that lives in linear memory (e.g. the `expected` 648 * out-param of __atomic_compare_exchange), which must be loaded rather 649 * than read as a bare wasm local. */ 650 wasm_cmp_branch(t, aux->op, wasm_ir_source_op(e, in->opnds[0], in->loc), 651 wasm_ir_source_op(e, in->opnds[1], in->loc), aux->target); 652 return; 653 } 654 case CG_IR_SWITCH: 655 wasm_ir_emit_switch(e, in); 656 return; 657 case CG_IR_INDIRECT_BRANCH: 658 wasm_ir_fail(e, in->loc, 659 "wasm target: indirect_branch (computed goto) not yet " 660 "implemented"); 661 return; 662 case CG_IR_LOAD_LABEL_ADDR: 663 wasm_ir_fail(e, in->loc, 664 "wasm target: load_label_addr (&&label) not yet " 665 "implemented"); 666 return; 667 case CG_IR_LOCAL_STATIC_DATA_BEGIN: 668 case CG_IR_LOCAL_STATIC_DATA_WRITE: 669 case CG_IR_LOCAL_STATIC_DATA_LABEL_ADDR: 670 case CG_IR_LOCAL_STATIC_DATA_END: 671 wasm_ir_fail(e, in->loc, 672 "wasm target: function-local static data not yet " 673 "implemented"); 674 return; 675 case CG_IR_SCOPE_BEGIN: { 676 const CgIrScopeAux* aux = (const CgIrScopeAux*)in->extra.aux; 677 CGScopeDesc d; 678 memset(&d, 0, sizeof d); 679 d.kind = aux->desc.kind; 680 d.break_label = aux->desc.break_label; 681 d.continue_label = aux->desc.continue_label; 682 d.result_type = aux->desc.result_type; 683 wasm_ir_bind_scope(e, aux->scope, wasm_scope_begin(t, &d), in->loc); 684 return; 685 } 686 case CG_IR_SCOPE_END: 687 wasm_scope_end(t, 688 wasm_ir_scope_lookup(e, (CGScope)in->extra.imm, in->loc)); 689 return; 690 case CG_IR_BREAK_TO: 691 wasm_break_to(t, 692 wasm_ir_scope_lookup(e, (CGScope)in->extra.imm, in->loc)); 693 return; 694 case CG_IR_CONTINUE_TO: 695 wasm_continue_to( 696 t, wasm_ir_scope_lookup(e, (CGScope)in->extra.imm, in->loc)); 697 return; 698 case CG_IR_ALLOCA: { 699 WasmIrDest d; 700 Operand size = wasm_ir_source_op(e, in->opnds[1], in->loc); 701 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 702 wasm_alloca(t, dst, size, (u32)in->extra.imm); 703 wasm_ir_dest_finish(e, &d); 704 return; 705 } 706 case CG_IR_VA_START: 707 wasm_va_start(t, wasm_ir_source_op(e, in->opnds[0], in->loc)); 708 return; 709 case CG_IR_VA_ARG: { 710 WasmIrDest d; 711 Operand ap = wasm_ir_source_op(e, in->opnds[1], in->loc); 712 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 713 wasm_va_arg(t, dst, ap, (KitCgTypeId)in->extra.imm); 714 wasm_ir_dest_finish(e, &d); 715 return; 716 } 717 case CG_IR_VA_END: 718 wasm_va_end(t, wasm_ir_source_op(e, in->opnds[0], in->loc)); 719 return; 720 case CG_IR_VA_COPY: { 721 Operand src = wasm_ir_source_op(e, in->opnds[1], in->loc); 722 Operand dst = wasm_ir_source_op(e, in->opnds[0], in->loc); 723 wasm_va_copy(t, dst, src); 724 return; 725 } 726 case CG_IR_ATOMIC_LOAD: { 727 const CgIrAtomicAux* aux = (const CgIrAtomicAux*)in->extra.aux; 728 WasmIrDest d; 729 Operand addr = wasm_ir_source_op(e, in->opnds[1], in->loc); 730 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 731 wasm_atomic_load(t, dst, addr, aux->mem, aux->order); 732 wasm_ir_dest_finish(e, &d); 733 return; 734 } 735 case CG_IR_ATOMIC_STORE: { 736 const CgIrAtomicAux* aux = (const CgIrAtomicAux*)in->extra.aux; 737 Operand addr = wasm_ir_source_op(e, in->opnds[0], in->loc); 738 Operand val = wasm_ir_source_op(e, in->opnds[1], in->loc); 739 wasm_atomic_store(t, addr, val, aux->mem, aux->order); 740 return; 741 } 742 case CG_IR_ATOMIC_RMW: { 743 const CgIrAtomicAux* aux = (const CgIrAtomicAux*)in->extra.aux; 744 WasmIrDest d; 745 Operand addr = wasm_ir_source_op(e, in->opnds[1], in->loc); 746 Operand val = wasm_ir_source_op(e, in->opnds[2], in->loc); 747 Operand dst = wasm_ir_dest_op(e, in->opnds[0], &d); 748 wasm_atomic_rmw(t, aux->op, dst, addr, val, aux->mem, aux->order); 749 wasm_ir_dest_finish(e, &d); 750 return; 751 } 752 case CG_IR_ATOMIC_CAS: { 753 const CgIrAtomicAux* aux = (const CgIrAtomicAux*)in->extra.aux; 754 WasmIrDest dprior, dok; 755 Operand addr = wasm_ir_source_op(e, in->opnds[2], in->loc); 756 Operand expected = wasm_ir_source_op(e, in->opnds[3], in->loc); 757 Operand desired = wasm_ir_source_op(e, in->opnds[4], in->loc); 758 Operand prior = wasm_ir_dest_op(e, in->opnds[0], &dprior); 759 Operand ok = wasm_ir_dest_op(e, in->opnds[1], &dok); 760 wasm_atomic_cas(t, prior, ok, addr, expected, desired, aux->mem, 761 aux->order, aux->failure); 762 wasm_ir_dest_finish(e, &dprior); 763 wasm_ir_dest_finish(e, &dok); 764 return; 765 } 766 case CG_IR_FENCE: 767 wasm_fence(t, (KitCgMemOrder)in->extra.imm); 768 return; 769 case CG_IR_INTRINSIC: { 770 const CgIrIntrinsicAux* aux = (const CgIrIntrinsicAux*)in->extra.aux; 771 Heap* h = e->target->c->ctx->heap; 772 Operand* dsts = NULL; 773 Operand* args = NULL; 774 WasmIrDest* dd = NULL; 775 if (aux->ndst) { 776 dsts = 777 (Operand*)h->alloc(h, sizeof(*dsts) * aux->ndst, _Alignof(Operand)); 778 dd = (WasmIrDest*)h->alloc(h, sizeof(*dd) * aux->ndst, 779 _Alignof(WasmIrDest)); 780 if (!dsts || !dd) 781 wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 782 } 783 if (aux->narg) { 784 args = 785 (Operand*)h->alloc(h, sizeof(*args) * aux->narg, _Alignof(Operand)); 786 if (!args) wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 787 for (u32 i = 0; i < aux->narg; ++i) 788 args[i] = wasm_ir_source_op(e, aux->args[i], in->loc); 789 } 790 for (u32 i = 0; i < aux->ndst; ++i) 791 dsts[i] = wasm_ir_dest_op(e, aux->dsts[i], &dd[i]); 792 wasm_intrinsic(t, aux->kind, dsts, aux->ndst, args, aux->narg); 793 for (u32 i = 0; i < aux->ndst; ++i) wasm_ir_dest_finish(e, &dd[i]); 794 if (dsts) h->free(h, dsts, sizeof(*dsts) * aux->ndst); 795 if (dd) h->free(h, dd, sizeof(*dd) * aux->ndst); 796 if (args) h->free(h, args, sizeof(*args) * aux->narg); 797 return; 798 } 799 case CG_IR_ASM_BLOCK: { 800 const CgIrAsmAux* aux = (const CgIrAsmAux*)in->extra.aux; 801 Heap* h = e->target->c->ctx->heap; 802 Operand* outs = NULL; 803 Operand* ins = NULL; 804 WasmIrDest* od = NULL; 805 if (aux->nout) { 806 outs = 807 (Operand*)h->alloc(h, sizeof(*outs) * aux->nout, _Alignof(Operand)); 808 od = (WasmIrDest*)h->alloc(h, sizeof(*od) * aux->nout, 809 _Alignof(WasmIrDest)); 810 if (!outs || !od) 811 wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 812 } 813 if (aux->nin) { 814 ins = (Operand*)h->alloc(h, sizeof(*ins) * aux->nin, _Alignof(Operand)); 815 if (!ins) wasm_ir_fail(e, in->loc, "wasm IR emit: out of memory"); 816 for (u32 i = 0; i < aux->nin; ++i) 817 ins[i] = wasm_ir_source_op(e, aux->in_ops[i], in->loc); 818 } 819 for (u32 i = 0; i < aux->nout; ++i) 820 outs[i] = wasm_ir_dest_op(e, aux->out_ops[i], &od[i]); 821 wasm_asm_block(t, aux->tmpl, aux->outs, aux->nout, outs, aux->ins, 822 aux->nin, ins, aux->clobbers, aux->nclob); 823 for (u32 i = 0; i < aux->nout; ++i) wasm_ir_dest_finish(e, &od[i]); 824 if (outs) h->free(h, outs, sizeof(*outs) * aux->nout); 825 if (od) h->free(h, od, sizeof(*od) * aux->nout); 826 if (ins) h->free(h, ins, sizeof(*ins) * aux->nin); 827 return; 828 } 829 } 830 wasm_ir_fail(e, in->loc, "wasm IR emit: unknown op"); 831 } 832 833 static void wasm_ir_emit_func(WTarget* t, const CgIrFunc* f) { 834 Heap* h = t->c->ctx->heap; 835 WasmIrEmitter e; 836 CGFuncDesc fd; 837 CGParamDesc* params = NULL; 838 memset(&e, 0, sizeof e); 839 e.target = t; 840 e.next_temp_reg = (Reg)(f->nlocals + 1u); 841 e.local_slots_n = f->nlocals + 1u; 842 e.local_slots = (FrameSlot*)h->alloc(h, sizeof(FrameSlot) * e.local_slots_n, 843 _Alignof(FrameSlot)); 844 if (!e.local_slots) 845 compiler_panic(t->c, f->desc.loc, "wasm IR emit: out of memory"); 846 for (u32 i = 0; i < e.local_slots_n; ++i) e.local_slots[i] = FRAME_SLOT_NONE; 847 e.local_types = (KitCgTypeId*)h->alloc( 848 h, sizeof(KitCgTypeId) * e.local_slots_n, _Alignof(KitCgTypeId)); 849 if (!e.local_types) 850 compiler_panic(t->c, f->desc.loc, "wasm IR emit: out of memory"); 851 for (u32 i = 0; i < e.local_slots_n; ++i) e.local_types[i] = KIT_CG_TYPE_NONE; 852 e.scope_map_n = f->nscopes + 1u; 853 if (e.scope_map_n) { 854 e.scope_map = (CGScope*)h->alloc(h, sizeof(CGScope) * e.scope_map_n, 855 _Alignof(CGScope)); 856 if (!e.scope_map) 857 compiler_panic(t->c, f->desc.loc, "wasm IR emit: out of memory"); 858 memset(e.scope_map, 0, sizeof(CGScope) * e.scope_map_n); 859 } 860 861 memset(&fd, 0, sizeof fd); 862 fd.sym = f->desc.sym; 863 fd.text_section_id = f->desc.text_section_id; 864 fd.group_id = f->desc.group_id; 865 fd.fn_type = f->desc.fn_type; 866 fd.result_type = f->desc.result_type; 867 fd.nparams = f->desc.nparams; 868 fd.loc = f->desc.loc; 869 fd.flags = f->desc.flags; 870 fd.inline_policy = f->desc.inline_policy; 871 fd.atomize = f->desc.atomize; 872 fd.abi = abi_cg_func_info(t->c->abi, f->desc.fn_type); 873 if (f->nparams) { 874 params = (CGParamDesc*)h->alloc(h, sizeof(*params) * f->nparams, 875 _Alignof(CGParamDesc)); 876 if (!params) 877 compiler_panic(t->c, f->desc.loc, "wasm IR emit: out of memory"); 878 for (u32 i = 0; i < f->nparams; ++i) { 879 const CgIrParam* src = &f->params[i]; 880 memset(¶ms[i], 0, sizeof params[i]); 881 params[i].index = src->desc.index; 882 params[i].name = src->desc.name; 883 params[i].type = src->desc.type; 884 params[i].size = src->desc.size; 885 params[i].align = src->desc.align; 886 params[i].flags = src->desc.flags; 887 params[i].loc = src->desc.loc; 888 params[i].abi = wasm_ir_param_abi(fd.abi, src->desc.index); 889 } 890 fd.params = params; 891 } 892 893 wasm_func_begin((CGTarget*)&t->base, &fd); 894 for (u32 i = 0; i < f->nlabels; ++i) 895 (void)wasm_label_new((CGTarget*)&t->base); 896 for (u32 i = 0; i < f->nparams; ++i) { 897 const CgIrParam* p = &f->params[i]; 898 CGLocalStorage st = wasm_param((CGTarget*)&t->base, ¶ms[i]); 899 if (p->local < e.local_slots_n) { 900 e.local_slots[p->local] = st.v.frame_slot; 901 e.local_types[p->local] = p->desc.type; 902 } 903 wasm_ir_bind_value_local(&e, p->local, p->desc.type, st.v.frame_slot); 904 } 905 for (u32 i = 0; i < f->nlocals; ++i) { 906 const CgIrLocal* l = &f->locals[i]; 907 if (l->is_param) continue; 908 CGLocalStorage st = wasm_local((CGTarget*)&t->base, &l->desc); 909 if (l->id < e.local_slots_n) { 910 e.local_slots[l->id] = st.v.frame_slot; 911 e.local_types[l->id] = l->desc.type; 912 } 913 wasm_ir_bind_value_local(&e, l->id, l->desc.type, st.v.frame_slot); 914 } 915 for (u32 i = 0; i < f->ninsts; ++i) wasm_ir_emit_inst(&e, f, &f->insts[i]); 916 wasm_func_end((CGTarget*)&t->base); 917 918 if (params) h->free(h, params, sizeof(*params) * f->nparams); 919 if (e.scope_map) h->free(h, e.scope_map, sizeof(CGScope) * e.scope_map_n); 920 h->free(h, e.local_types, sizeof(KitCgTypeId) * e.local_slots_n); 921 h->free(h, e.local_slots, sizeof(FrameSlot) * e.local_slots_n); 922 } 923 924 void wasm_emit_ir_module(WTarget* t, const CgIrModule* module) { 925 if (!t || !module) return; 926 /* Process aliases before emitting function bodies. Function linearization is 927 * eager (wasm_func_end), so a call through a function alias resolves its 928 * wasm func index at emit time; if the alias->target mapping isn't installed 929 * first, the call allocates a fresh empty func for the alias (bad body -> 930 * "function result type mismatch") instead of dispatching to the target. 931 * Data aliases only need the target's (section_id,value), shared earlier at 932 * the ObjBuilder layer, so they are equally safe here. */ 933 for (u32 i = 0; i < module->naliases; ++i) { 934 const CgIrAlias* a = &module->aliases[i]; 935 wasm_alias((CGTarget*)&t->base, a->alias_sym, a->target_sym, a->type); 936 } 937 for (u32 i = 0; i < module->nfuncs; ++i) { 938 wasm_ir_emit_func(t, module->funcs[i]); 939 } 940 }