call.c (13520B)
1 #include "cg/internal.h" 2 3 static u32 api_func_nparams(KitCg* g, KitCgTypeId fty) { 4 const CgType* ty = cg_type_get(g->c, fty); 5 if (!ty || ty->kind != KIT_CG_TYPE_FUNC) return 0; 6 return ty->func.nparams; 7 } 8 9 CGLocal* api_alloc_call_args(KitCg* g, u32 nargs) { 10 if (!nargs) return NULL; 11 CGLocal* args = arena_array(g->c->tu, CGLocal, nargs); 12 memset(args, 0, sizeof(CGLocal) * nargs); 13 return args; 14 } 15 16 static CGLocal api_materialize_call_local(KitCg* g, ApiSValue* arg, 17 KitCgTypeId ty) { 18 if (cg_type_is_aggregate(g->c, ty)) { 19 if (api_is_lvalue_sv(arg) && arg->op.kind == OPK_LOCAL) { 20 if (arg->op.type == ty) return arg->op.v.local; 21 } 22 CGLocal r = api_alloc_temp_local(g, ty); 23 Operand dst = api_op_local(r, ty); 24 MemAccess ma; 25 memset(&ma, 0, sizeof ma); 26 ma.type = ty; 27 ma.size = abi_cg_sizeof(g->c->abi, ty); 28 ma.align = abi_cg_alignof(g->c->abi, ty); 29 if (api_is_lvalue_sv(arg) && api_operand_can_address(&arg->op)) { 30 g->target->load(g->target, dst, arg->op, ma); 31 } else if (arg->op.kind == OPK_GLOBAL || arg->op.kind == OPK_INDIRECT) { 32 g->target->load(g->target, dst, arg->op, ma); 33 } else if (arg->op.kind == OPK_LOCAL) { 34 g->target->load(g->target, dst, api_op_indirect(arg->op.v.local, 0, ty), 35 ma); 36 } else { 37 CG_BUG(g, "KitCg: aggregate call argument is not addressable"); 38 } 39 return r; 40 } 41 KitCgTypeId src_ty = api_sv_type(arg); 42 Operand op; 43 /* A 16-byte scalar immediate (an i128 small constant) only carries 64 bits in 44 * op.v.imm; materialize it into both sign-extended lanes so it flows as an 45 * ordinary 16-byte value rather than load_imm'ing only the low lane. */ 46 if (api_sv_op_is(arg, OPK_IMM) && api_is_wide16_scalar_type(g->c, ty)) { 47 *arg = api_make_wide16_int_const(g, arg->op.v.imm, ty); 48 } 49 /* Same for a split-lane 8-byte immediate argument: materialize it as a 50 * 2-lane memory value so the multi-part ABI path marshals both words. */ 51 if (api_sv_op_is(arg, OPK_IMM) && api_is_wide8_scalar_type(g->c, ty)) { 52 *arg = api_make_wide8_int_const(g, arg->op.v.imm, ty); 53 } 54 op = api_force_local_unless_imm(g, arg, src_ty); 55 if (op.kind == OPK_LOCAL && op.type == ty) { 56 return op.v.local; 57 } 58 59 CGLocal r = api_alloc_temp_local(g, ty); 60 Operand dst = api_op_local(r, ty); 61 if (op.kind == OPK_IMM) { 62 g->target->load_imm(g->target, dst, op.v.imm); 63 } else if (op.kind == OPK_LOCAL) { 64 g->target->copy(g->target, dst, op); 65 } else { 66 CG_BUG(g, "KitCg: scalar call argument is not materialized"); 67 } 68 api_release(g, arg); 69 return r; 70 } 71 72 void api_pack_call_arg(KitCg* g, CGLocal* out, KitCgTypeId fty, u32 idx) { 73 ApiSValue arg = api_pop(g); 74 u32 nfixed = api_func_nparams(g, fty); 75 KitCgTypeId aty = 76 idx >= nfixed ? api_sv_type(&arg) : cg_type_func_param_id(g->c, fty, idx); 77 if (!aty) aty = api_sv_type(&arg); 78 *out = api_materialize_call_local(g, &arg, aty); 79 } 80 81 /* Pack the NARGS top-of-stack call arguments into ARGS[0..nargs) IN ARG ORDER, 82 * materializing each (emitting any pending producer) arg0-first. The arguments 83 * sit on the value stack as `arg0 .. arg(n-1)` (arg(n-1) on top); reverse that 84 * sub-array so the LIFO pop yields arg0 first, then pop+materialize forward. 85 * 86 * Why forward: an argument materialized into a register lands in the first free 87 * -O0 cache register, and the target's `ndt_allocable` order front-loads the 88 * ABI argument registers (x0,x1,.. on aa64). Producing arg0 first therefore 89 * lands it in x0, arg1 in x1, .. so the per-call parallel-copy collapses to 90 * no-ops, instead 91 * of the reverse order leaving each value in the *wrong* arg register and 92 * forcing a permutation shuffle. Matches tcc's eager left-to-right argument 93 * emission. 94 * 95 * Reversing the on-stack order (rather than popping all up front) preserves the 96 * api_temp_dead semantics exactly: the not-yet-materialized arguments stay on 97 * the stack, so a temp shared between arguments (e.g. f(t*2, t*3)) is scanned 98 * as live until its last (highest-index) use and is coalesced there, never 99 * killed prematurely. */ 100 static void api_pack_call_args_in_order(KitCg* g, CGLocal* args, u32 nargs, 101 KitCgTypeId fty) { 102 u32 base = g->sp - nargs; /* arg0's stack slot (callee, if any, sits below) */ 103 for (u32 i = 0; i < nargs / 2u; ++i) { 104 ApiSValue tmp = g->stack[base + i]; 105 ApiConstValue ctmp = g->const_stack[base + i]; 106 g->stack[base + i] = g->stack[base + nargs - 1u - i]; 107 g->const_stack[base + i] = g->const_stack[base + nargs - 1u - i]; 108 g->stack[base + nargs - 1u - i] = tmp; 109 g->const_stack[base + nargs - 1u - i] = ctmp; 110 } 111 for (u32 i = 0; i < nargs; ++i) api_pack_call_arg(g, &args[i], fty, i); 112 } 113 114 CGLocal api_alloc_call_result(KitCg* g, KitCgTypeId ret_ty) { 115 return api_alloc_temp_local(g, ret_ty); 116 } 117 118 static int api_type_has_value(KitCg* g, KitCgTypeId ty) { 119 return ty != KIT_CG_TYPE_NONE && !cg_type_is_void(g->c, ty); 120 } 121 122 void api_push_call_result(KitCg* g, CGLocal result, KitCgTypeId ret_ty) { 123 Operand op = api_op_local(result, ret_ty); 124 /* An aggregate result is a PLACE (it is addressed/copied, never a scalar 125 * VALUE); i128/f128 are scalar VALUEs and flow like any other result. */ 126 if (cg_type_is_aggregate(g->c, ret_ty)) { 127 api_push(g, api_make_lv(op, ret_ty)); 128 } else { 129 api_push(g, api_make_sv(op, ret_ty)); 130 } 131 } 132 133 /* Push the placeholder for a call result in an unevaluated context (the call is 134 * type-checked but emits no code). Mirrors api_push_call_result's place-vs-value 135 * rule: an aggregate result must be a PLACE even here, since api_push rejects an 136 * aggregate VALUE (e.g. a struct-returning call in an initializer's type-compat 137 * probe). */ 138 static void api_push_uneval_call_result(KitCg* g, KitCgTypeId result_type) { 139 if (!api_type_has_value(g, result_type)) return; 140 api_push(g, cg_type_is_aggregate(g->c, result_type) 141 ? api_uneval_place(g, result_type) 142 : api_uneval_value(g, result_type)); 143 api_const_set_top(g, api_const_unknown(result_type)); 144 } 145 146 static void api_call_clobber_boundary(KitCg* g, const CGCallDesc* d) { 147 (void)g; 148 (void)d; 149 } 150 151 static int api_tail_ret_compatible(KitCg* g, KitCgTypeId callee_fn_type) { 152 KitCgTypeId cr = cg_type_func_result_id(g->c, callee_fn_type); 153 return g->fn_desc.result_type == cr; 154 } 155 156 static int api_tail_decide(KitCg* g, const CGCallDesc* desc, 157 KitCgTailPolicy policy) { 158 CgTarget* T = g->target; 159 const char* reason; 160 if (!api_tail_ret_compatible(g, desc->fn_type)) { 161 compiler_panic(g->c, g->cur_loc, 162 "tail call: callee return type is incompatible with the " 163 "enclosing function's return type"); 164 return 0; 165 } 166 if (g->stack_protected) { 167 if (policy == KIT_CG_TAIL_MUST) { 168 compiler_panic(g->c, g->cur_loc, 169 "musttail call not realizable: stack protector requires " 170 "a checked return epilogue"); 171 } 172 return 0; 173 } 174 reason = T->tail_call_unrealizable_reason 175 ? T->tail_call_unrealizable_reason(T, desc) 176 : "target does not support tail calls"; 177 if (!reason) return 1; 178 if (policy == KIT_CG_TAIL_MUST) { 179 compiler_panic(g->c, g->cur_loc, "musttail call not realizable: %s", 180 reason); 181 return 0; 182 } 183 return 0; 184 } 185 186 static void api_finish_call(KitCg* g, CGCallDesc* desc, int want_tail, 187 int emit_tail) { 188 if (!emit_tail) api_call_clobber_boundary(g, desc); 189 /* Flag arguments that are dead after the call (no live value-stack reference) 190 * so the -O0 backend can keep them register-resident across the call instead 191 * of round-tripping through their homes. api_temp_dead is sound — true only 192 * when the transient is provably dead — so a live or shared arg stays clear 193 * and is spilled like any other live-across value. */ 194 desc->arg_dead_mask = 0; 195 for (u32 i = 0; i < desc->nargs && i < 64u; ++i) 196 if (api_temp_dead(g, desc->args[i])) desc->arg_dead_mask |= (u64)1u << i; 197 g->target->call(g->target, desc); 198 199 /* Push the single result (if any) onto the stack. */ 200 if (desc->result != CG_LOCAL_NONE) { 201 KitCgTypeId rty = cg_type_func_result_id(g->c, desc->fn_type); 202 api_push_call_result(g, desc->result, rty); 203 } 204 /* ALLOWED tail call that degraded to an ordinary call: synthesize the 205 * caller's return of the just-pushed result. */ 206 if (want_tail && !emit_tail) kit_cg_ret(g); 207 } 208 209 void kit_cg_call(KitCg* g, uint32_t nargs, KitCgTypeId fn_type, 210 KitCgCallAttrs attrs) { 211 CgTarget* T; 212 KitCgTypeId fty; 213 KitCgTypeId result_type; 214 CGLocal* args; 215 CGCallDesc desc; 216 ApiSValue callee; 217 Operand callee_op; 218 int want_tail; 219 int emit_tail; 220 if (!g) return; 221 api_local_const_memory_boundary(g); 222 want_tail = 223 attrs.tail == KIT_CG_TAIL_ALLOWED || attrs.tail == KIT_CG_TAIL_MUST; 224 T = g->target; 225 fty = resolve_type(g->c, fn_type); 226 if (!fty) return; 227 228 CG_REQUIRE(g, g->sp >= (u32)nargs + 1u, "KitCg: call stack underflow"); 229 if (api_unevaluated(g)) { 230 u32 i; 231 result_type = cg_type_func_result_id(g->c, fty); 232 for (i = 0; i < nargs; ++i) { 233 ApiSValue arg = api_pop(g); 234 api_release(g, &arg); 235 } 236 callee = api_pop(g); 237 api_release(g, &callee); 238 api_push_uneval_call_result(g, result_type); 239 return; 240 } 241 242 args = api_alloc_call_args(g, nargs); 243 api_pack_call_args_in_order(g, args, nargs, fty); 244 245 callee = api_pop(g); 246 api_ensure_local(g, &callee); 247 callee_op = (callee.op.kind == OPK_GLOBAL) 248 ? callee.op 249 : api_force_local(g, &callee, api_sv_type(&callee)); 250 KitCgInlinePolicy inline_policy = attrs.inline_policy; 251 if (inline_policy == KIT_CG_INLINE_DEFAULT && callee_op.kind == OPK_GLOBAL && 252 callee_op.v.global.addend == 0) { 253 KitCgDecl callee_attrs = api_sym_attrs(g, (KitCgSym)callee_op.v.global.sym); 254 if (callee_attrs.kind == KIT_CG_DECL_FUNC) 255 inline_policy = callee_attrs.as.func.inline_policy; 256 } 257 258 memset(&desc, 0, sizeof desc); 259 desc.fn_type = fty; 260 desc.callee = callee_op; 261 desc.args = args; 262 desc.nargs = nargs; 263 desc.tail_policy = (u8)attrs.tail; 264 desc.inline_policy = inline_policy; 265 266 emit_tail = want_tail ? api_tail_decide(g, &desc, attrs.tail) : 0; 267 desc.flags = emit_tail ? CG_CALL_TAIL : CG_CALL_NONE; 268 result_type = 269 emit_tail ? KIT_CG_TYPE_NONE : cg_type_func_result_id(g->c, fty); 270 if (api_type_has_value(g, result_type)) 271 desc.result = api_alloc_call_result(g, result_type); 272 273 (void)T; 274 api_finish_call(g, &desc, want_tail, emit_tail); 275 } 276 277 void api_call_symbol_common(KitCg* g, KitCgSym sym, uint32_t nargs, 278 KitCgCallAttrs attrs) { 279 KitCgTypeId fty; 280 KitCgTypeId result_type; 281 CGLocal* args; 282 CGCallDesc desc; 283 Operand callee_op; 284 KitCgInlinePolicy inline_policy; 285 int want_tail; 286 int emit_tail; 287 if (!g) return; 288 api_local_const_memory_boundary(g); 289 want_tail = 290 attrs.tail == KIT_CG_TAIL_ALLOWED || attrs.tail == KIT_CG_TAIL_MUST; 291 fty = api_sym_type(g, sym); 292 if (!fty) return; 293 CG_REQUIRE(g, g->sp >= nargs, "KitCg: call stack underflow"); 294 if (api_unevaluated(g)) { 295 u32 i; 296 result_type = cg_type_func_result_id(g->c, fty); 297 for (i = 0; i < nargs; ++i) { 298 ApiSValue arg = api_pop(g); 299 api_release(g, &arg); 300 } 301 api_push_uneval_call_result(g, result_type); 302 return; 303 } 304 args = api_alloc_call_args(g, nargs); 305 api_pack_call_args_in_order(g, args, nargs, fty); 306 callee_op = api_op_global((ObjSymId)sym, 0, cg_type_ptr_to(g->c, fty)); 307 inline_policy = attrs.inline_policy; 308 if (inline_policy == KIT_CG_INLINE_DEFAULT) { 309 KitCgDecl callee_attrs = api_sym_attrs(g, sym); 310 if (callee_attrs.kind == KIT_CG_DECL_FUNC) 311 inline_policy = callee_attrs.as.func.inline_policy; 312 } 313 memset(&desc, 0, sizeof desc); 314 desc.fn_type = fty; 315 desc.callee = callee_op; 316 desc.args = args; 317 desc.nargs = nargs; 318 desc.tail_policy = (u8)attrs.tail; 319 desc.inline_policy = inline_policy; 320 321 emit_tail = want_tail ? api_tail_decide(g, &desc, attrs.tail) : 0; 322 desc.flags = emit_tail ? CG_CALL_TAIL : CG_CALL_NONE; 323 result_type = 324 emit_tail ? KIT_CG_TYPE_NONE : cg_type_func_result_id(g->c, fty); 325 if (api_type_has_value(g, result_type)) 326 desc.result = api_alloc_call_result(g, result_type); 327 api_finish_call(g, &desc, want_tail, emit_tail); 328 } 329 330 void kit_cg_call_symbol(KitCg* g, KitCgSym sym, uint32_t nargs, 331 KitCgCallAttrs attrs) { 332 api_call_symbol_common(g, sym, nargs, attrs); 333 } 334 335 void kit_cg_ret(KitCg* g) { 336 KitCgTypeId rty; 337 ApiSValue v; 338 CGLocal value; 339 if (!g) return; 340 rty = g->fn_desc.result_type; 341 if (api_unevaluated(g)) { 342 if (api_type_has_value(g, rty)) { 343 v = api_pop(g); 344 api_release(g, &v); 345 } 346 return; 347 } 348 if (g->stack_protected) api_stack_protector_check(g); 349 if (!api_type_has_value(g, rty)) { 350 g->target->ret(g->target, CG_LOCAL_NONE); 351 return; 352 } 353 /* TOS holds the single result value. */ 354 v = api_pop(g); 355 if (cg_type_is_aggregate(g->c, rty)) { 356 value = api_materialize_call_local(g, &v, rty); 357 } else { 358 Operand ret_op = api_force_local(g, &v, rty); 359 value = ret_op.v.local; 360 } 361 g->target->ret(g->target, value); 362 api_release(g, &v); 363 } 364 365 /* ============================================================ 366 * Data definitions (stubs) 367 * ============================================================ */