asm.c (13224B)
1 #include "cg/internal.h" 2 3 const char* api_sym_cstr(KitCg* g, KitSym sym) { 4 const char* s; 5 if (!sym) return ""; 6 s = pool_slice(g->c->global, (Sym)sym).s; 7 return s ? s : ""; 8 } 9 10 int api_asm_parse_match_index(const char* s) { 11 return kit_cg_asm_constraint_match_index(s); 12 } 13 14 const char* api_asm_constraint_body(const char* s) { 15 return kit_cg_asm_constraint_body(s); 16 } 17 18 int api_asm_is_early_clobber(const char* s) { 19 return kit_cg_asm_constraint_early(s); 20 } 21 22 /* Does this constraint body name a register operand (one that binds to a temp 23 * local, as opposed to 'i' immediate or 'm' memory)? 'r' is the architecture- 24 * neutral general-register class; 'f' (riscv), 'x' (x86 SSE) and 'w' (aarch64 25 * SIMD/FP) are the per-target FP/vector register classes. The temp local's type 26 * selects the actual NativeAllocClass downstream, and the target's asm hook 27 * rejects a letter that does not apply to it, so listing all three here is safe 28 * across backends. A hard-register pin (AsmConstraint.reg, from a GNU local 29 * register variable) rides alongside such a register operand and does not 30 * change this classification — the constraint letter stays "r". */ 31 int api_asm_is_reg_constraint(char c) { 32 return c == 'r' || c == 'f' || c == 'x' || c == 'w'; 33 } 34 35 static int api_asm_constraint_is_reg(KitCg* g, const char* constraint) { 36 const char* body = api_asm_constraint_body(constraint); 37 if (api_asm_is_reg_constraint(body[0])) return 1; 38 if (g && g->target && g->target->asm_is_reg_constraint) 39 return g->target->asm_is_reg_constraint(g->target, constraint); 40 return 0; 41 } 42 43 /* A register ('r'/'f'/'x'/'w') asm operand must live in a single hardware 44 * register. A 64-bit scalar on a 32-bit target does not fit one: it would need 45 * a register pair, which this inline-asm lowering does not model, so binding it 46 * to a single register would silently truncate to the low word. Reject it up 47 * front; the source can use a memory ("m") constraint (the value is already 48 * memory-resident) or split it into two 32-bit operands instead. Wider scalars 49 * on a 64-bit target (i128/f128) take a different lowering and are not this 50 * helper's concern. */ 51 static void api_asm_reject_wide_reg(KitCg* g, KitCgTypeId ty) { 52 if (api_is_wide8_scalar_type(g->c, ty)) { 53 compiler_panic(g->c, g->cur_loc, 54 "KitCg: 64-bit value in a register asm constraint is not " 55 "supported on a 32-bit target; use a memory (\"m\") " 56 "constraint or split into two 32-bit operands"); 57 } 58 } 59 60 void api_asm_memory_clobber_sv(KitCg* g, ApiSValue* sv, CGLocal local_id) { 61 (void)g; 62 (void)sv; 63 (void)local_id; 64 } 65 66 void kit_cg_inline_asm(KitCg* g, KitCgInlineAsm asm_block) { 67 static const char* const match_strs[10] = {"0", "1", "2", "3", "4", 68 "5", "6", "7", "8", "9"}; 69 CgTarget* T; 70 Heap* h; 71 KitCgTypeId fallback_ty; 72 AsmConstraint* outs; 73 AsmConstraint* ins; 74 Sym* clobs; 75 ApiSValue* in_svs; 76 Operand* in_ops; 77 Operand* out_ops; 78 u8* out_local_owned; 79 const char* tmpl_str; 80 Sym sym_memory; 81 int has_memory_clobber; 82 uint32_t ninout; 83 uint32_t total_inputs; 84 KitSym tmpl = asm_block.tmpl; 85 const KitCgAsmOperand* outputs = asm_block.outputs; 86 uint32_t noutputs = asm_block.noutputs; 87 const KitCgAsmOperand* inputs = asm_block.inputs; 88 uint32_t ninputs = asm_block.ninputs; 89 const KitSym* clobbers = asm_block.clobbers; 90 uint32_t nclobbers = asm_block.nclobbers; 91 uint32_t clobber_abi_sets = asm_block.clobber_abi_sets; 92 uint32_t flags = asm_block.flags; 93 if (!g) return; 94 if (flags & ~KIT_CG_ASM_VOLATILE) { 95 compiler_panic(g->c, g->cur_loc, 96 "KitCg: unsupported inline asm flags"); 97 } 98 if (api_unevaluated(g)) { 99 uint32_t ninout_u = 0; 100 for (u32 i = 0; i < noutputs; ++i) 101 if (outputs[i].dir == KIT_CG_ASM_INOUT) ninout_u++; 102 for (u32 i = 0; i < ninputs + ninout_u; ++i) { 103 ApiSValue sv = api_pop(g); 104 api_release(g, &sv); 105 } 106 for (u32 i = 0; i < noutputs; ++i) { 107 KitCgTypeId oty = resolve_type(g->c, outputs[i].type); 108 if (!oty) oty = builtin_id(KIT_CG_BUILTIN_I64); 109 api_push(g, api_uneval_value(g, oty)); 110 api_const_set_top(g, api_const_unknown(oty)); 111 } 112 return; 113 } 114 api_local_const_memory_boundary(g); 115 T = g->target; 116 h = g->c->ctx->heap; 117 fallback_ty = builtin_id(KIT_CG_BUILTIN_I64); 118 tmpl_str = api_sym_cstr(g, tmpl); 119 ninout = 0; 120 121 outs = NULL; 122 ins = NULL; 123 clobs = NULL; 124 in_svs = NULL; 125 in_ops = NULL; 126 out_ops = NULL; 127 out_local_owned = NULL; 128 129 if (noutputs) { 130 outs = (AsmConstraint*)h->alloc(h, sizeof(*outs) * noutputs, 131 _Alignof(AsmConstraint)); 132 memset(outs, 0, sizeof(*outs) * noutputs); 133 for (u32 i = 0; i < noutputs; ++i) { 134 outs[i].str = api_sym_cstr(g, outputs[i].constraint); 135 outs[i].name = (Sym)outputs[i].name; 136 outs[i].type = resolve_type(g->c, outputs[i].type); 137 outs[i].reg = (Sym)outputs[i].reg; 138 outs[i].dir = (u8)outputs[i].dir; 139 if (!outs[i].type) outs[i].type = fallback_ty; 140 if (outs[i].reg && !api_asm_constraint_is_reg(g, outs[i].str)) { 141 compiler_panic(g->c, g->cur_loc, 142 "KitCg: asm hard-register output requires a register " 143 "constraint"); 144 } 145 if (outs[i].dir == KIT_CG_ASM_INOUT) { 146 if (i >= 10) { 147 compiler_panic(g->c, g->cur_loc, 148 "KitCg: asm inout output index exceeds matching " 149 "constraint range"); 150 } 151 ninout++; 152 } 153 } 154 out_ops = 155 (Operand*)h->alloc(h, sizeof(*out_ops) * noutputs, _Alignof(Operand)); 156 memset(out_ops, 0, sizeof(*out_ops) * noutputs); 157 out_local_owned = (u8*)h->alloc(h, noutputs, 1); 158 memset(out_local_owned, 0, noutputs); 159 } 160 161 total_inputs = ninputs + ninout; 162 if (total_inputs) { 163 uint32_t inout_index; 164 ins = (AsmConstraint*)h->alloc(h, sizeof(*ins) * total_inputs, 165 _Alignof(AsmConstraint)); 166 memset(ins, 0, sizeof(*ins) * total_inputs); 167 in_svs = (ApiSValue*)h->alloc(h, sizeof(*in_svs) * total_inputs, 168 _Alignof(ApiSValue)); 169 in_ops = (Operand*)h->alloc(h, sizeof(*in_ops) * total_inputs, 170 _Alignof(Operand)); 171 memset(in_ops, 0, sizeof(*in_ops) * total_inputs); 172 for (u32 i = 0; i < ninputs; ++i) { 173 ins[i].str = api_sym_cstr(g, inputs[i].constraint); 174 ins[i].name = (Sym)inputs[i].name; 175 ins[i].type = resolve_type(g->c, inputs[i].type); 176 ins[i].reg = (Sym)inputs[i].reg; 177 ins[i].dir = (u8)inputs[i].dir; 178 if (!ins[i].type) ins[i].type = fallback_ty; 179 if (ins[i].reg && !api_asm_constraint_is_reg(g, ins[i].str)) { 180 compiler_panic(g->c, g->cur_loc, 181 "KitCg: asm hard-register input requires a register " 182 "constraint"); 183 } 184 } 185 inout_index = ninputs; 186 for (u32 i = 0; i < noutputs; ++i) { 187 if (outs[i].dir != KIT_CG_ASM_INOUT) continue; 188 ins[inout_index].str = match_strs[i]; 189 ins[inout_index].type = outs[i].type ? outs[i].type : fallback_ty; 190 ins[inout_index].dir = KIT_CG_ASM_IN; 191 inout_index++; 192 } 193 for (u32 i = 0; i < total_inputs; ++i) { 194 u32 idx = total_inputs - 1u - i; 195 in_svs[idx] = api_pop(g); 196 api_ensure_local(g, &in_svs[idx]); 197 } 198 } 199 200 if (nclobbers) { 201 clobs = (Sym*)h->alloc(h, sizeof(*clobs) * nclobbers, _Alignof(Sym)); 202 for (u32 i = 0; i < nclobbers; ++i) clobs[i] = (Sym)clobbers[i]; 203 } 204 205 for (u32 i = 0; i < noutputs; ++i) { 206 if (api_asm_is_early_clobber(outs[i].str)) continue; 207 /* A register constraint binds to a temp local; the local's type selects the 208 * register class (integer vs FP), so the backend hook places an FP-class 209 * output (riscv 'f', x86 'x', aarch64 'w') in an FP register. */ 210 if (api_asm_constraint_is_reg(g, outs[i].str)) { 211 KitCgTypeId oty = outs[i].type ? outs[i].type : fallback_ty; 212 CGLocal r; 213 api_asm_reject_wide_reg(g, oty); 214 r = api_alloc_temp_local(g, oty); 215 out_ops[i] = api_op_local(r, oty); 216 out_local_owned[i] = 1; 217 } else { 218 compiler_panic(g->c, g->cur_loc, 219 "KitCg: unsupported asm output constraint"); 220 } 221 } 222 223 for (u32 i = 0; i < total_inputs; ++i) { 224 const char* s = ins[i].str ? ins[i].str : ""; 225 int matched = api_asm_parse_match_index(s); 226 KitCgTypeId ity = api_sv_type(&in_svs[i]); 227 if (matched >= 0) { 228 Operand bound; 229 if ((u32)matched >= noutputs) { 230 compiler_panic(g->c, g->cur_loc, 231 "KitCg: asm matching constraint out of range"); 232 continue; 233 } 234 if (api_asm_is_early_clobber(outs[matched].str)) { 235 compiler_panic(g->c, g->cur_loc, 236 "KitCg: asm matching input uses early-clobber output"); 237 continue; 238 } 239 bound = out_ops[matched]; 240 if (api_sv_op_is(&in_svs[i], OPK_LOCAL) && 241 in_svs[i].op.v.local == bound.v.local) { 242 } else if (api_sv_op_is(&in_svs[i], OPK_IMM)) { 243 T->load_imm(T, bound, in_svs[i].op.v.imm); 244 } else { 245 Operand src = api_force_local(g, &in_svs[i], ity); 246 T->copy(T, bound, src); 247 } 248 in_ops[i] = bound; 249 } else if (api_asm_constraint_is_reg(g, s)) { 250 api_asm_reject_wide_reg(g, ity); 251 in_ops[i] = api_force_local(g, &in_svs[i], ity); 252 } else if (s[0] == 'i') { 253 if (!api_sv_op_is(&in_svs[i], OPK_IMM)) { 254 compiler_panic(g->c, g->cur_loc, 255 "KitCg: asm 'i' constraint requires an immediate"); 256 } 257 in_ops[i] = in_svs[i].op; 258 } else if (s[0] == 'm') { 259 if (api_sv_op_is(&in_svs[i], OPK_INDIRECT)) { 260 in_ops[i] = in_svs[i].op; 261 } else if (api_is_lvalue_sv(&in_svs[i])) { 262 KitCgTypeId pty = 263 cg_type_ptr_to(g->c, ity ? ity : builtin_id(KIT_CG_BUILTIN_VOID)); 264 Operand dst = api_lvalue_addr(g, &in_svs[i], pty); 265 in_svs[i].op = api_op_indirect(dst.v.local, 0, ity); 266 api_sv_set_res(&in_svs[i], RES_LOCAL); 267 in_ops[i] = in_svs[i].op; 268 } else { 269 compiler_panic(g->c, g->cur_loc, 270 "KitCg: asm 'm' constraint requires an lvalue"); 271 } 272 } else { 273 compiler_panic(g->c, g->cur_loc, 274 "KitCg: unsupported asm input constraint"); 275 } 276 } 277 278 for (u32 i = 0; i < noutputs; ++i) { 279 KitCgTypeId oty; 280 CGLocal r; 281 if (!api_asm_is_early_clobber(outs[i].str)) continue; 282 if (!api_asm_constraint_is_reg(g, outs[i].str)) { 283 compiler_panic(g->c, g->cur_loc, 284 "KitCg: unsupported early-clobber asm output"); 285 continue; 286 } 287 oty = outs[i].type ? outs[i].type : fallback_ty; 288 api_asm_reject_wide_reg(g, oty); 289 r = api_alloc_temp_local(g, oty); 290 for (u32 k = 0; k < total_inputs; ++k) { 291 if ((in_ops[k].kind == OPK_LOCAL && in_ops[k].v.local == r) || 292 (in_ops[k].kind == OPK_INDIRECT && in_ops[k].v.ind.base == r)) { 293 compiler_panic(g->c, g->cur_loc, 294 "KitCg: asm early-clobber local collision"); 295 } 296 } 297 out_ops[i] = api_op_local(r, oty); 298 out_local_owned[i] = 1; 299 } 300 301 sym_memory = pool_intern_slice(g->c->global, SLICE_LIT("memory")); 302 has_memory_clobber = 0; 303 for (u32 i = 0; i < nclobbers; ++i) { 304 if (clobs[i] == sym_memory) has_memory_clobber = 1; 305 } 306 if (has_memory_clobber) { 307 for (u32 i = 0; i < g->sp; ++i) { 308 ApiSValue* sv = &g->stack[i]; 309 CGLocal local_id; 310 if (api_sv_res(sv) != RES_LOCAL) continue; 311 local_id = api_local_of_sv(sv); 312 api_asm_memory_clobber_sv(g, sv, local_id); 313 } 314 } 315 316 T->asm_block(T, tmpl_str, outs, noutputs, out_ops, ins, total_inputs, in_ops, 317 clobs, nclobbers, clobber_abi_sets); 318 319 for (u32 i = 0; i < total_inputs; ++i) api_release(g, &in_svs[i]); 320 for (u32 i = 0; i < noutputs; ++i) { 321 KitCgTypeId oty = outs[i].type ? outs[i].type : fallback_ty; 322 ApiSValue sv = api_make_sv(out_ops[i], oty); 323 if (!out_local_owned[i] && api_sv_res(&sv) == RES_LOCAL) 324 api_sv_set_res(&sv, RES_INHERENT); 325 api_push(g, sv); 326 } 327 328 if (outs) h->free(h, outs, sizeof(*outs) * noutputs); 329 if (ins) h->free(h, ins, sizeof(*ins) * total_inputs); 330 if (clobs) h->free(h, clobs, sizeof(*clobs) * nclobbers); 331 if (in_svs) h->free(h, in_svs, sizeof(*in_svs) * total_inputs); 332 if (in_ops) h->free(h, in_ops, sizeof(*in_ops) * total_inputs); 333 if (out_ops) h->free(h, out_ops, sizeof(*out_ops) * noutputs); 334 if (out_local_owned) h->free(h, out_local_owned, noutputs); 335 } 336 337 void kit_cg_file_scope_asm(KitCg* g, KitSlice asm_source) { 338 if (!g || !asm_source.s) return; 339 if (api_unevaluated(g)) return; 340 if (g->check_only) return; 341 if (g->target && g->target->file_scope_asm) { 342 g->target->file_scope_asm(g->target, asm_source.s, asm_source.len); 343 return; 344 } 345 compiler_panic(g->c, api_no_loc(), 346 "KitCg: file-scope asm requires target support"); 347 } 348 349 /* ============================================================ 350 * Labels / branches 351 * ============================================================ */