commit e4e0e56721a23e85259656d3677a4386f6963ee8
parent ee05a9bd27505aa0728952971e0a686ccecbfd75
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sun, 19 Jul 2026 12:40:24 -0700
refactor(native): centralize scratch and clobber ownership
Make physical register roles and shared policy lists the single source of truth across O0 and optimized native codegen. Centralize inline-asm allocation, callee-save merging, call marshalling names, and exhaustive machine effects while keeping emitter reservations policy-owned.
Fix AAPCS32 aggregate passing/returns, harden exact-width x64 scratch handling, include ARM/RISC-V backend coverage, and isolate cross-test target state.
Diffstat:
37 files changed, 1705 insertions(+), 955 deletions(-)
diff --git a/doc/ARCH.md b/doc/ARCH.md
@@ -130,7 +130,7 @@ It is driven from two directions:
is the `CgTarget`. It owns semantic local homes, a small register cache, and
conservative flushes, and lowers each semantic op directly into
`NativeTarget` hook calls. The arch supplies a `NativeTarget` plus a small
- semantic adapter, `NativeOps` (bind_param, plan_call, emit_call/ret, va_*,
+ semantic adapter, `NativeOps` (bind_param, marshal_call, emit_call/ret, va_*,
asm_block, barriers, legality predicates) — the parts that need a foot in the
semantic world. Everything else (frame slots, `class_for_type`, `addr_legal`)
the direct target calls straight through to `NativeTarget`.
@@ -146,6 +146,16 @@ It is driven from two directions:
`NativeLoc`s to the backend. `-O2` currently selects this same path; the
maintained SSA schedule is not publicly reachable yet.
+ Both paths consume the same validated `NativeRegInfo`. Physical-register
+ role flags are the single authority for static allocability, reservation,
+ argument/return use, and caller/callee preservation; mask consumers derive
+ their view from those rows. A target may override only caller/callee masks
+ when the live OS ABI changes preservation (Win64 x64). Ordered policy lists
+ separately describe O0 allocation, backend scratch, optimized emit/asm temp
+ banks, and direct inline-asm allocation. Shared validation runs at both the
+ direct and optimized adapter boundaries, while the shared asm binder owns
+ selection from the target-declared inline-asm pool.
+
A physical address may retain a frame-resident base or index as
`NATIVE_ADDR_BASE_FRAME_VALUE` / `NATIVE_ADDR_INDEX_FRAME_VALUE`. Its
`base_type` / `index_type` is the exact scalar type to load, not merely the
@@ -171,11 +181,12 @@ generators byte-compatible per arch. The hook families:
`store`, `tls_addr_of`, `copy_bytes`/`set_bytes` (aggregate memcpy/memset),
`bitfield_load`/`bitfield_store`, `spill`/`reload`.
- **Arithmetic.** `binop`, `unop`, `cmp`, `convert`, `alloca_`.
-- **Calls & returns.** A marshalling/emission split: `plan_call` is deliberately
- mutating. It fills a `NativeCallPlan` and performs target-owned pre-call work
+- **Calls & returns.** A marshalling/emission split: `marshal_call` is
+ deliberately mutating. It fills a `NativeCallPhase` and performs target-owned
+ pre-call work
such as indirect-callee staging, outgoing stack stores, and register
- parallel copies; the shared caller realizes any remaining plan argument
- moves. `emit_call` emits the actual control transfer. `plan_ret` describes
+ parallel copies; the shared caller realizes any remaining phase argument
+ moves. `emit_call` emits the actual control transfer. `marshal_ret` describes
and may perform target-owned return marshalling, then returns any remaining
ordered moves for the caller to realize before `ret`. `call_stack_bytes` and
`signature_stack_bytes` are the pure pre-pass queries used to size the
@@ -309,8 +320,8 @@ Each native backend declares its register file as static `NativeRegInfo` data in
its `native.c` (e.g. `aa_reg_info`, wired into the `NativeTarget` at
construction; the DWARF-index ↔ assembler-name tables that the `ArchImpl`
exposes for objdump/asm live separately in `regs.c`). A `NativeRegInfo` is a set
-of `NativeAllocClassInfo` (one per `NativeAllocClass`: INT, FP, VEC), each
-carrying:
+of `NativeAllocClassInfo` (one per `NativeAllocClass`: INT, FP, VEC), keyed by
+its `cls` field (table position is not semantic), with each class carrying:
- an ordered O0 **ndt_allocable** list — the `NativeDirectTarget` value-cache
bank and preference order; it is never an O1 allocator input;
@@ -321,6 +332,8 @@ carrying:
locations, not backend-private implementation registers;
- an **asm_temps** list — a disjoint backend-internal bank the optimized emitter
may borrow only while staging a compatible inline-asm block;
+- a **direct_asm_allocable** list — the independent ordered register policy for
+ operands self-allocated by the O0 inline-asm binder;
- an **emit_cache_mask** — the subset of `emit_temps` whose clean spill values
an emitter may retain across ordinary hooks within one block without an ABI
save; a dead O1 allocation register may also become a cache entry when its
@@ -328,7 +341,9 @@ carrying:
- a `NativePhysRegInfo` row per physical register (class, ABI arg/ret index,
caller/callee-saved flags, the independent `NATIVE_REG_ALLOCABLE` O1 flag,
and spill/copy costs);
-- precomputed caller/callee/arg/ret/reserved bitmasks.
+- caller/callee/arg/ret/reserved role masks derived from those rows (with a
+ live-ABI override where one architecture supports multiple preservation
+ conventions).
This one declaration describes **both** code paths without making either path's
register bank an input to the other:
@@ -392,18 +407,21 @@ therefore enumerates the cross-product cells that both sides enable:
| aa64 | `aapcs64` | `apple_arm64` | `aapcs64_windows` |
| x64 | `sysv_x64` | `apple_x64` | `win64_x64` |
| rv64 | `rv64` | — | — |
+| rv32 | `rv32` | — | — |
+| arm32 | `aapcs32` | — | — |
| wasm | — (`wasm32`, via the wasm object format) |
Each per-ABI TU (`abi_aapcs64.c`, `abi_sysv_x64.c`, `abi_apple_arm64.c`,
-`abi_apple_x64.c`, `abi_win64_x64.c`, `abi_aapcs64_windows.c`, `abi_rv64.c`)
-implements its `compute_func_info` and `va_list` facts; the Apple/Windows
-variants encode their divergences (e.g. Apple ARM64 routes the variadic tail
-exclusively to the stack, recorded as `vararg_on_stack` in `ABIFuncInfo`). The
-classification is the *only* authority — the `NativeTarget` plan/bind hooks and
-the optimizer both consume `ABIFuncInfo`; they never re-derive argument
-placement. Frame-relevant ABI facts (the vararg register-save-area size) are
-funneled through `src/cg/native_frame.c` so the per-arch magic numbers all trace
-back to one `va_list`-layout query.
+`abi_apple_x64.c`, `abi_win64_x64.c`, `abi_aapcs64_windows.c`, `abi_rv64.c`,
+and `abi_aapcs32.c`) implements its `compute_func_info` and `va_list` facts;
+the RISC-V TU supplies both XLEN variants, and the Apple/Windows variants encode
+their divergences (e.g. Apple ARM64 routes the variadic tail exclusively to the
+stack, recorded as `vararg_on_stack` in `ABIFuncInfo`). The classification is
+the *only* authority — the `NativeTarget` plan/bind hooks and the optimizer both
+consume `ABIFuncInfo`; they never re-derive argument placement. Frame-relevant
+ABI facts (the vararg register-save-area size) are funneled through
+`src/cg/native_frame.c` so the per-arch magic numbers all trace back to one
+`va_list`-layout query.
## 8. Per-call cost model (aa64 -O1)
diff --git a/doc/BACKEND_REFACTOR_CROSS_TEST.md b/doc/BACKEND_REFACTOR_CROSS_TEST.md
@@ -68,3 +68,164 @@ Targeted validation after the fixes:
green at O0/O1 on x86-64 and AArch64.
This was a targeted RV64 follow-up, not a repeat of the full 19-target matrix.
+
+## RV32 follow-up
+
+RV32 uses the same RISC-V NativeTarget implementation as RV64, with `xlen=32`,
+so the follow-up emphasized the refactor-sensitive shared paths: temporary and
+clobber ownership, intrinsic operand locations, call marshalling, small
+aggregate arguments, TLS, and both RV32 ABIs used by the test harness.
+
+The original full cross log was already clean for RV32: the Toy corpus passed
+427 cases with two intentional AArch64-only skips, and the parser corpus passed
+970 cases with 64 intentional skips. Current-tree validation added:
+
+- Targeted parser cases for signed-overflow intrinsics, initialized TLS,
+ instruction-temporary boundaries, machine-clobbered returns, and small
+ aggregate argument spills pass at O0 and O1.
+- The Toy bit/overflow intrinsic case passes at O0 and O1.
+- `test-smoke-rv32` passes all 20 checks. This covers hard-float `ilp32f` and
+ soft-float `ilp32` at O0/O1, automatic runtime linking, concurrent runtime
+ cache lanes, ELF flags, mixed-ABI rejection, and the negative control.
+- The RV32 assembler corpus passes all 16 cases and the decoder unit passes all
+ 96 checks. The JIT image/lookup test builds and validates its bytes; native
+ execution is intentionally skipped on the non-RV32 host.
+- The current full Toy run again passes 427 cases with the same two intentional
+ skips. The full parser run recorded 969 passes and one QEMU timeout, plus the
+ same 64 intentional skips. The timed-out O0 `6_5_45_ptr_eq_null` artifact
+ disassembles correctly, its saved ELF returns 3 when run directly, and an
+ isolated rerun passes at both O0 and O1. This is a transient QEMU/harness
+ timeout rather than a reproducible backend failure; the heavy corpus was not
+ rerun merely to replace that result.
+- `test-cross TARGET=freestanding-rv32 DEPTH=smoke KIT_VM=1` passes all three
+ compile, link, and execution lane checks.
+
+No RV32-specific or shared production-code change was required. The shared
+immediate-location correction made during the RV64 follow-up remains the
+appropriate architectural fix and is covered by the RV32 overflow probes.
+
+## ARM32 follow-up
+
+The clean `8d7927db` revision reproduced both ARM32 failures from the original
+matrix exactly: `130_record_sret_return` in the Toy corpus and
+`6_8_6_4_05_struct_return_mixed_fp_int` in the C corpus passed at O0, while O1
+aborted after the first three ABI result registers made a fourth temporary
+unavailable. The shared native emitter must load a frame/stack return part
+directly into its final ABI register; routing it through a temporary creates a
+false fifth-register requirement. The current emitter does that direct load.
+The shared regression now fills an entire four-register result bank, matching
+the ARM32 failure rather than testing only two parts.
+
+The audit also found two independent contract bugs that the original corpus
+could not expose reliably because Kit was both caller and callee:
+
+- The AAPCS32 classifier treated arguments and results symmetrically at a
+ 16-byte cutoff. [AAPCS32](https://github.com/ARM-software/abi-aa/blob/main/aapcs32/aapcs32.rst)
+ instead returns composites larger than one word through caller-provided
+ storage, while every statically sized composite argument remains a by-value
+ memory image split across r0-r3 and the stack. Classification now represents
+ that asymmetry directly. `ABIArgInfo.nparts` is consequently 32-bit: a large
+ fixed-size argument cannot silently truncate at 65,535 word lanes.
+- Known-frame prologues did not include callee-saved registers named by ARM32
+ O1 inline-asm clobbers. Parsing the raw target clobbers, expanding the shared
+ ABI clobber sets, and merging the result with allocator-used callee saves now
+ lives in `cg/native_asm.c`; AA64, x64, RISC-V, and ARM32 provide only their
+ register-name parser and target preservation predicate. An ARM32 O1 test
+ verifies that a named r4 clobber produces a matching save/restore, and r7,
+ sp, and pc are rejected as structural clobbers.
+
+The shared direct-asm binder also used an I64 location as the address carrier
+for every memory operand. It now selects I32 or I64 from the target pointer
+width, keeping this target-neutral path valid on ARM32 and RV32.
+
+Validation on the corrected tree:
+
+- The two historical cases pass at O0 and O1. Focused AAPCS cases covering
+ 3-byte tails, 4/8/16/32-byte records, 8-byte alignment, and a 65,537-word
+ argument pass; the ABI classifier unit is 495/0.
+- Two-way Clang interoperability passes at O0 and O1: a Clang caller consumes
+ Kit's 8-byte composite return and 32-byte by-value argument, and a Kit caller
+ consumes the same interfaces implemented by Clang.
+- Shared native-boundary units pass: register effects 102/0, direct target 55,
+ location MIR 57/0, full-bank frame-result emission 335/0, and frame-value
+ backends 316/0. All four native inline-asm units pass. ARM32 ISA decoding is
+ 583/0 and target-feature coverage is 32/0.
+- `test-cross TARGET=freestanding-arm32 DEPTH=full KIT_VM=1 RUN=1` passed all
+ eight compile/link/smoke/amalgam/Toy lane checks; Toy was 427 pass, 0 fail,
+ 2 intentional skips. The parser batch recorded 966 pass, 2 QEMU timeouts,
+ and 66 intentional skips. Both timed-out O0 artifacts
+ (`6_5_9_01_suppressed_null_ptr_cmp`, `6_6_01_enum_const`) and their O1 forms
+ passed immediately in isolated serial runs (4/0, 0.28-0.92 seconds), so the
+ only non-green full-lane verdict is a transient harness timeout rather than a
+ reproducible backend failure. The heavy corpus was not rerun merely to erase
+ that result.
+
+Artifacts:
+
+- `build/test-logs/test-cross-freestanding-arm32-full.log`
+- `build/test-logs/arm32-timeout-isolated.log`
+
+## X64 follow-up
+
+The hosted x64 failures exposed a broader ownership problem than the original
+R11 aggregate-copy corruption. Exact-width 3/5/6/7-byte loads and stores chose
+address and data temporaries through backend-global `part_scratch` state while
+the generic x64 memory emitter independently borrowed the same private bank.
+Those operations now stabilize the complete effective address in an explicitly
+selected private register for their local phase and use the other private
+register for chunk data. The backend no longer carries `part_scratch`.
+
+Call setup had the same hidden-state shape. Stack bias, parallel-copy scratch,
+tail-call state, and indirect-callee parking now live in a local
+`X64CallMarshalCtx`; the old backend-global `call_callee_parked` flag is gone.
+The mutating interface is named `marshal_call`/`marshal_ret` and fills a
+`NativeCallPhase`, making its ownership and lifetime distinct from the pure
+`call_stack_bytes` frame-sizing query. Dead call-plan clobber/return mask fields
+were removed.
+
+The cross-architecture audit then consolidated the shared scratch/clobber
+model:
+
+- `NativePhysRegInfo.flags` is the single static authority for allocability,
+ reservation, argument/return roles, and caller/callee preservation. Role
+ masks are derived centrally. The only live-ABI override is x64
+ caller/callee preservation, where SysV/Apple and Win64 differ.
+- A shared `native_reg_info_validate()` checks physical rows, ordered policy
+ lists, role contradictions, temp-bank separation, asm-temp reservation,
+ emit-temp non-allocability, and emitter cache containment at both the O0
+ direct-target and O1 machinization boundaries. Machinization separately adds
+ the policy-owned emitter bank to each function's reserved set.
+- O0 inline-asm register selection and ABI-clobber expansion live in
+ `cg/native_asm.c`; targets declare only their ordered operand pool and retain
+ target-specific constraint parsing/emission. Known-frame asm/callee-save
+ merging is shared there as well.
+- Parallel argument-copy scratch is class-tagged explicitly, so physical
+ register zero cannot double as “unconfigured”; cycle breaking rejects a
+ missing, invalid, or live-aliased scratch register.
+- Machine-operation effects are exhaustive through `NATIVE_MOP_COUNT`. Focused
+ x64 coverage includes division/remainder, variable shifts, bitfields,
+ atomics, variadics, intrinsics, syscalls, and TLS. Exact role-mask tests cover
+ SysV x64, dynamic Win64 preservation, AA64, RV64, and ARM32; they found and
+ corrected missing argument-role flags on x64 XMM6/XMM7.
+
+Two cross-harness ownership bugs were also corrected while validating x64:
+FreeBSD/Windows full selectors now pass only the requested VM architecture,
+and parse startup objects are cached per architecture rather than sharing a
+mutable `parse_start.o` across concurrent workers. The x64 execution lane now
+classifies the IEEE-binary128 semantic fixtures as inapplicable (while retaining
+the layout/macro probe), matching the actual SysV/Apple x87 and Win64
+double-aliased `long double` ABIs.
+
+Final validation, in the requested platform order:
+
+- macOS x64: Mach-O build plus direct and queued Rosetta execution, 3/3.
+- Linux x64, glibc + musl, full depth: 42/42 cross lanes. Each target passed
+ 427 Toy cases and 1,004 parser executions; glibc passed 9 libc cases and musl
+ passed 18. Only the documented AA64-only and inapplicable x64 binary128 cases
+ skipped.
+- FreeBSD x64, full depth: 13/13 cross lanes and 668/668 applicable Toy cases.
+- Windows x64, full depth: 13/13 cross lanes and 334/334 applicable Toy cases,
+ including FP callee saves, typed-asm ABI clobbers, indirect packed-stack
+ musttail, aggregate returns, variadics, and O0/O1 paths.
+- Shared focused units: 865/865 checks plus all x64/AA64/RV64/ARM32 inline-asm
+ units. Tier-1 optimizer and architecture suites both pass.
diff --git a/doc/INTERFACES.md b/doc/INTERFACES.md
@@ -155,10 +155,10 @@ its raw word/halfword emit helpers for the assembler path:
**Backend-tier notes:**
- `NativeTarget` is the physical-emission contract: frame setup and prologue
policy, control flow, data movement, arithmetic/compare/convert, calls
- (`plan_call`/`emit_call`/`plan_ret`/`ret`), atomics, variadics, intrinsics, and
- inline/file-scope asm — roughly three dozen hooks. The caller (NDT or the
- optimizer) owns program-value placement and supplies legal physical
- locations. A backend may materialize exact-typed `FRAME_VALUE` address
+ (`marshal_call`/`emit_call`/`marshal_ret`/`ret`), atomics, variadics,
+ intrinsics, and inline/file-scope asm — roughly three dozen hooks. The caller
+ (NDT or the optimizer) owns program-value placement and supplies legal
+ physical locations. A backend may materialize exact-typed `FRAME_VALUE` address
components using the explicit destination or fixed backend-private
implementation temps, but it must never allocate a persistent program value
or borrow a caller-owned register outside that hook. All four native backends
@@ -167,9 +167,10 @@ its raw word/halfword emit helpers for the assembler path:
semantic shims the -O0 adapter calls). Keep the split clean: semantic decisions
and semantic-to-physical adaptation live in `NativeOps`; physical
marshalling, validation, and instruction emission live in `NativeTarget`.
- `plan_call` is an intentionally mutating marshalling phase: it fills the plan
- while performing target-owned callee/argument staging and stack stores.
- `plan_ret` is the corresponding mutating return-marshalling phase;
+ `marshal_call` is an intentionally mutating marshalling phase: it fills the
+ `NativeCallPhase` while performing target-owned callee/argument staging and
+ stack stores.
+ `marshal_ret` is the corresponding mutating return-marshalling phase;
`call_stack_bytes` is the pure frame-sizing query.
- A handful of `NativeTarget` hooks are explicitly optional and exist for archs
whose ISA needs them — `machine_op_clobbers` (including x64 division, shifts,
diff --git a/doc/OPT.md b/doc/OPT.md
@@ -600,9 +600,10 @@ operand materialization; after that bank is exhausted it may scavenge a
caller-saved O1 allocation register only when physical liveness proves it dead
for the instruction. Multiple references to the same location within the
instruction share a scoped materialization. Calls have a mutating marshalling
-phase: `plan_call` may stage the callee, store stack arguments, and perform
-target-owned argument shuffles while it fills the plan. Each completed ABI
-register destination remains phase-owned through the full multi-move phase.
+phase: `marshal_call` may stage the callee, store stack arguments, and perform
+target-owned argument shuffles while it fills the `NativeCallPhase`. Each
+completed ABI register destination remains phase-owned through the full
+multi-move phase.
After `emit_call`, a phase barrier discards argument/callee staging before
return writeback begins. Inline asm may additionally borrow `asm_temps` only
under the target-declared constraints; its staging remains local to that asm
diff --git a/doc/plan/ARM32.md b/doc/plan/ARM32.md
@@ -11,6 +11,23 @@ emit `.ARM.attributes`, ARM32 `.thumb_func` symbols carry the ABI Thumb bit, and
C inline asm templates now route real Thumb-2 mnemonics through
`src/arch/arm32/asm.c`.
+### Backend-refactor audit — 2026-07-19
+
+The post-`8d7927db` audit is complete. The two O1 aggregate-return failures were
+reproduced on that revision and are covered by a shared full-four-register
+return regression; frame/stack parts load directly into their final ABI result
+registers, without consuming an extra instruction-local temporary.
+
+The audit corrected the soft-float AAPCS32 aggregate contract: composites larger
+than one word return indirectly through r0, while fixed-size composite
+parameters of any size remain by-value across r0-r3 and the stack. Independent
+Clang↔Kit tests cover both directions for an 8-byte return and a 32-byte
+argument at O0/O1. Known-frame inline-asm clobber preservation is now shared
+across native targets and covers ARM32 r4; structural r7/sp/pc clobbers are
+rejected. See [BACKEND_REFACTOR_CROSS_TEST.md](../BACKEND_REFACTOR_CROSS_TEST.md#arm32-follow-up)
+for the full evidence and the two isolated-green QEMU timeout artifacts from the
+full ARM32 cross run.
+
### Follow-on landed — 2026-06-17 — DSP, flag/macro parsing, code density, disasm
Four follow-on items landed (parallel worktree agents, serial cherry-pick behind
@@ -951,7 +968,7 @@ the internal seams — `NativeTarget` (`src/arch/native_target.h`), `ArchImpl`
(`src/arch/arch.h`), the ABI vtable (`src/abi/abi_internal.h`), and
`reloc_apply_insn`. Each later track lives behind exactly one of them:
-- frame/ABI work behind `func_begin` / `plan_call` / `spill` / `frame_slot`
+- frame/ABI work behind `func_begin` / `marshal_call` / `spill` / `frame_slot`
- op-group work behind `binop` / `emit` + the isa table
- reloc work behind `arm32_reloc_apply_insn` + the descriptor table
- runtime entirely in `rt/`
@@ -1013,7 +1030,7 @@ were built by parallel agents.
### Phase 2 — remaining tracks
-Same seams as before (`func_begin`/`plan_call`/`spill`; `binop`/`emit` + the isa
+Same seams as before (`func_begin`/`marshal_call`/`spill`; `binop`/`emit` + the isa
table; `arm32_reloc_apply_insn`; `rt/`). Most of `native.c`/`abi`/`reloc` is in
place; the unimplemented ops `compiler_panic` with a clear message. Gate stays
**run-correctness under qemu**, reds left red. Op-groups contend on `native.c` /
diff --git a/scripts/cross_test.sh b/scripts/cross_test.sh
@@ -308,7 +308,7 @@ _full_lane() {
}
full_token() {
- local t="$1" os arch libc
+ local t="$1" os arch libc vm_arch
os="$(tok_os "$t")"; arch="$(tok_arch "$t")"; libc="$(tok_libc "$t")"
case "$os" in
linux)
@@ -331,10 +331,23 @@ full_token() {
esac
;;
freebsd)
- _full_lane "$t:toy-vm" env KIT="$KIT" bash "$ROOT/test/toy/vm.sh" freebsd
+ case "$arch" in
+ x64) vm_arch=amd64 ;;
+ aa64) vm_arch=aarch64 ;;
+ rv64) vm_arch=riscv64 ;;
+ esac
+ _full_lane "$t:toy-vm" \
+ env KIT="$KIT" KIT_TOY_FREEBSD_ARCHES="$vm_arch" \
+ bash "$ROOT/test/toy/vm.sh" freebsd
;;
windows)
- _full_lane "$t:toy-vm" env KIT="$KIT" bash "$ROOT/test/toy/vm.sh" windows
+ case "$arch" in
+ x64) vm_arch=x64 ;;
+ aa64) vm_arch=aarch64 ;;
+ esac
+ _full_lane "$t:toy-vm" \
+ env KIT="$KIT" KIT_TOY_WINDOWS_ARCHES="$vm_arch" \
+ bash "$ROOT/test/toy/vm.sh" windows
;;
freestanding)
case "$arch" in
diff --git a/src/abi/abi.h b/src/abi/abi.h
@@ -108,7 +108,12 @@ typedef struct ABIArgPart {
typedef struct ABIArgInfo {
u8 kind; /* ABIArgKind */
u8 flags; /* ABIArgFlag applying to the whole argument */
- u16 nparts;
+ u16 reserved;
+ /* A fixed-size by-value aggregate is not bounded by the argument-register
+ * bank: its remaining word parts continue on the stack. Keep this count
+ * wide enough to describe the full object rather than truncating at 64K
+ * carrier lanes. */
+ u32 nparts;
u32 indirect_align; /* required alignment for ABI_ARG_INDIRECT/byval copy */
const ABIArgPart* parts;
} ABIArgInfo;
diff --git a/src/abi/abi_aapcs32.c b/src/abi/abi_aapcs32.c
@@ -7,8 +7,10 @@
* The 8-byte even-register / 8-byte-stack alignment that
* AAPCS requires is surfaced to the native backend via
* ABIArgPart.align; this layer just declares two lanes.
- * aggregate <= 16B -> DIRECT, up to four word parts (r0..r3)
- * aggregate > 16B -> INDIRECT (sret pointer in r0 / byval)
+ * aggregate return <= 4B -> DIRECT, one INT part in r0
+ * aggregate return > 4B -> INDIRECT (sret pointer in r0)
+ * fixed-size aggregate argument -> DIRECT word parts, filled through
+ * r0..r3 and then the stack
*
* The sret pointer is passed in r0 (the first integer arg register), consuming
* that slot, so sret_consumes_int_arg = 1 (like RISC-V, unlike AArch64's x8).
@@ -21,8 +23,7 @@
#include "core/arena.h"
#include "core/core.h"
-#define ARM32_GPR_BYTES 4u /* r0..r3 are 32-bit */
-#define ARM32_AGG_GPR_BYTES 16u /* aggregates <= 4 words pass in registers */
+#define ARM32_GPR_BYTES 4u /* r0..r3 are 32-bit */
static void classify_scalar(TargetABI* a, KitCgTypeId t, ABIArgInfo* out) {
ABITypeInfo ti = abi_cg_type_info(a, t);
@@ -72,7 +73,17 @@ static void classify_aggregate(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
abi_classify_void(out);
return;
}
- if (ti.size <= ARM32_AGG_GPR_BYTES) {
+ /* AAPCS32 treats results and parameters asymmetrically. Composite results
+ * larger than one word use caller-provided result storage, while every
+ * statically sized composite parameter remains a by-value memory image that
+ * may occupy any number of consecutive core-register/stack words. */
+ if (is_return && ti.size > ARM32_GPR_BYTES) {
+ out->kind = ABI_ARG_INDIRECT;
+ out->flags = ABI_AF_SRET;
+ out->indirect_align = ti.align ? ti.align : ARM32_GPR_BYTES;
+ out->parts = NULL;
+ out->nparts = 0;
+ } else {
u32 nparts = (ti.size + ARM32_GPR_BYTES - 1u) / ARM32_GPR_BYTES;
ABIArgPart* parts = arena_array(a->c->tu, ABIArgPart, nparts);
u32 off = 0;
@@ -99,14 +110,8 @@ static void classify_aggregate(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
out->kind = ABI_ARG_DIRECT;
out->flags = ABI_AF_NONE;
out->parts = parts;
- out->nparts = (u16)nparts;
+ out->nparts = nparts;
out->indirect_align = 0;
- } else {
- out->kind = ABI_ARG_INDIRECT;
- out->flags = is_return ? ABI_AF_SRET : ABI_AF_BYVAL;
- out->indirect_align = ti.align ? ti.align : ARM32_GPR_BYTES;
- out->parts = NULL;
- out->nparts = 0;
}
}
diff --git a/src/abi/abi_aapcs64.c b/src/abi/abi_aapcs64.c
@@ -54,7 +54,7 @@ static void classify_aggregate(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
out->kind = ABI_ARG_DIRECT;
out->flags = ABI_AF_NONE;
out->parts = parts;
- out->nparts = (u16)nparts;
+ out->nparts = nparts;
out->indirect_align = 0;
} else {
out->kind = ABI_ARG_INDIRECT;
diff --git a/src/abi/abi_aapcs64_windows.c b/src/abi/abi_aapcs64_windows.c
@@ -27,7 +27,7 @@ extern ABIFuncInfo* aapcs64_compute_func_info(TargetABI*, KitCgTypeId);
static void remap_fp_parts_to_int(TargetABI* a, ABIArgInfo* ai) {
if (!ai || ai->kind != ABI_ARG_DIRECT || ai->nparts == 0) return;
int needs_copy = 0;
- for (u16 i = 0; i < ai->nparts; ++i) {
+ for (u32 i = 0; i < ai->nparts; ++i) {
if (ai->parts[i].cls == ABI_CLASS_FP) {
needs_copy = 1;
break;
@@ -37,7 +37,7 @@ static void remap_fp_parts_to_int(TargetABI* a, ABIArgInfo* ai) {
ABIArgPart* parts = arena_array(a->c->tu, ABIArgPart, ai->nparts);
memcpy(parts, ai->parts, sizeof(ABIArgPart) * ai->nparts);
- for (u16 i = 0; i < ai->nparts; ++i) {
+ for (u32 i = 0; i < ai->nparts; ++i) {
if (parts[i].cls == ABI_CLASS_FP) parts[i].cls = ABI_CLASS_INT;
}
ai->parts = parts;
diff --git a/src/abi/abi_rv64.c b/src/abi/abi_rv64.c
@@ -213,7 +213,7 @@ static int riscv_classify_fp_aggregate(TargetABI* a, KitCgTypeId t,
out->kind = ABI_ARG_DIRECT;
out->flags = ABI_AF_NONE;
out->parts = parts;
- out->nparts = (u16)n;
+ out->nparts = n;
out->indirect_align = 0;
return 1;
}
@@ -245,7 +245,7 @@ static void classify_aggregate(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
out->kind = ABI_ARG_DIRECT;
out->flags = ABI_AF_NONE;
out->parts = parts;
- out->nparts = (u16)nparts;
+ out->nparts = nparts;
out->indirect_align = 0;
} else {
out->kind = ABI_ARG_INDIRECT;
diff --git a/src/abi/abi_sysv_x64.c b/src/abi/abi_sysv_x64.c
@@ -154,7 +154,7 @@ static void classify_aggregate(TargetABI* a, KitCgTypeId t, ABIArgInfo* out,
out->kind = ABI_ARG_DIRECT;
out->flags = nparts > 1 ? ABI_AF_SPLIT : ABI_AF_NONE;
out->parts = parts;
- out->nparts = (u16)nparts;
+ out->nparts = nparts;
out->indirect_align = 0;
} else {
out->kind = ABI_ARG_INDIRECT;
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -60,7 +60,7 @@ enum {
/* Tail epilogues restore x30 and may use x16/x17 while rebuilding the frame,
* so an indirect tail target is moved here only after argument marshalling.
* x9 is an instruction-scoped emitter temp and no later operand phase runs
- * between plan_call and the tail branch. */
+ * between marshal_call and the tail branch. */
AA_TAIL_TARGET = 9u,
AA_TMP0 = 16u,
AA_TMP1 = 17u,
@@ -1193,8 +1193,8 @@ static u32 aa_part_transfer_tmp(AANativeTarget* a, NativeLoc reg,
/* A register carrier is 8 bytes, but the final part of a small aggregate may
* carry any byte count from 1 through 8. Keep the one-instruction native-width
* path; decompose 3/5/6/7-byte memory transfers into exact low-to-high chunks.
- * This is deliberately below the call planner so incoming binds, outgoing
- * args, and generic NativeCallPlanRet write phases all share the same rule. */
+ * This is deliberately below call marshalling so incoming binds, outgoing
+ * args, and generic NativeCallPhaseRet write phases all share the same rule. */
static void aa_emit_mem(AANativeTarget* a, int load, NativeLoc reg,
NativeAddr addr, MemAccess mem) {
u32 bytes = mem.size
@@ -2378,31 +2378,21 @@ static void aa_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
* bits of v8..v15 (x29/x30 are the frame pointer and link register, handled by
* the prologue head, not as ordinary callee-saves). This is the same register
* selection the per-block spill used, hoisted into the prologue. */
+static int aa_asm_reg_is_callee_saved(NativeTarget* t, NativeAllocClass cls,
+ Reg r) {
+ (void)t;
+ return (cls == NATIVE_REG_INT && r >= 19u && r <= 28u) ||
+ (cls == NATIVE_REG_FP && r >= 8u && r <= 15u);
+}
+
static u32 aa_known_callee_saves(NativeTarget* t,
const NativeKnownFrameDesc* frame, u32* out,
u32 cap) {
- u32 ncls = frame->ncallee_classes;
- u32 clob_int = 0, clob_fp = 0, abi_int, abi_fp;
- if (ncls > cap) ncls = cap;
- for (u32 c = 0; c < ncls; ++c)
- out[c] = frame->callee_saved_used ? frame->callee_saved_used[c] : 0u;
- if (frame->asm_clobbers && frame->nasm_clobbers) {
- AANativeTarget* a = aa_of(t);
- SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
- aa_asm_clobber_masks(t->c, loc, frame->asm_clobbers, frame->nasm_clobbers,
- &clob_int, &clob_fp);
- }
- native_asm_abi_clobber_masks(t, frame->asm_clobber_abi_sets, &abi_int,
- &abi_fp);
- clob_int |= abi_int;
- clob_fp |= abi_fp;
- for (Reg r = 0; r < 32u; ++r) {
- if (NATIVE_REG_INT < ncls && (clob_int & (1u << r)) && r >= 19u && r <= 28u)
- out[NATIVE_REG_INT] |= 1u << r;
- if (NATIVE_REG_FP < ncls && (clob_fp & (1u << r)) && r >= 8u && r <= 15u)
- out[NATIVE_REG_FP] |= 1u << r;
- }
- return ncls;
+ AANativeTarget* a = aa_of(t);
+ SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
+ return native_asm_known_callee_saves(
+ t, loc, frame, out, cap, aa_asm_clobber_masks,
+ aa_asm_reg_is_callee_saved);
}
static u32 aa_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
@@ -2425,8 +2415,8 @@ static void aa_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
aa_func_begin_common(t, fd);
a->frame.known_frame = 1;
if (frame) {
- u32 cs[NATIVE_CALL_PLAN_CLASSES];
- u32 ncs = aa_known_callee_saves(t, frame, cs, NATIVE_CALL_PLAN_CLASSES);
+ u32 cs[NATIVE_REG_CLASS_COUNT];
+ u32 ncs = aa_known_callee_saves(t, frame, cs, NATIVE_REG_CLASS_COUNT);
a->frame.has_alloca = frame->has_alloca;
if (ncs) aa_reserve_callee_saves(t, cs, ncs);
for (u32 i = 0; i < frame->nslots; ++i) {
@@ -3715,12 +3705,13 @@ static void aa_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves,
s.reg_move = aa_move;
s.scratch[NATIVE_REG_INT] = AA_TMP1;
s.scratch[NATIVE_REG_FP] = 16u;
+ s.scratch_class_mask = (1u << NATIVE_REG_INT) | (1u << NATIVE_REG_FP);
native_arg_shuffle(&s, moves, n);
}
-static void aa_plan_call(NativeTarget* t, const NativeCallDesc* desc,
- NativeCallPlan* plan) {
- NativeCallPlanRet* rets;
+static void aa_marshal_call(NativeTarget* t, const NativeCallDesc* desc,
+ NativeCallPhase* plan) {
+ NativeCallPhaseRet* rets;
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
NativeLoc saved_indirect_callee;
int preserve_indirect_callee = 0;
@@ -3732,7 +3723,7 @@ static void aa_plan_call(NativeTarget* t, const NativeCallDesc* desc,
: ((!abi && desc->nresults) ? 1u : 0u);
memset(plan, 0, sizeof *plan);
rets =
- nrets_cap ? arena_zarray(t->c->tu, NativeCallPlanRet, nrets_cap) : NULL;
+ nrets_cap ? arena_zarray(t->c->tu, NativeCallPhaseRet, nrets_cap) : NULL;
plan->callee = desc->callee;
plan->rets = rets;
plan->flags = desc->flags;
@@ -3950,7 +3941,7 @@ static void aa_emit_tail_site(NativeTarget* t, NativeLoc callee) {
}
}
-static void aa_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
+static void aa_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
int is_tail = (plan->flags & CG_CALL_TAIL) != 0;
if (is_tail) {
if (plan->callee.kind != NATIVE_LOC_GLOBAL &&
@@ -3973,11 +3964,11 @@ static void aa_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
aa_panic(aa_of(t), "unsupported call target");
}
-static void aa_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
- const NativeLoc* value, NativeCallPlanRet** out_rets,
- u32* out_nrets) {
+static void aa_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
+ const NativeLoc* value,
+ NativeCallPhaseRet** out_rets, u32* out_nrets) {
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
- NativeCallPlanRet* rets = NULL;
+ NativeCallPhaseRet* rets = NULL;
u32 nr = 0;
if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
AANativeTarget* a = aa_of(t);
@@ -4020,7 +4011,7 @@ static void aa_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
/* The generic return write phase materializes a memory source using the
* scalar carrier type, so an I64 carrier for a 3/5/6/7-byte aggregate part
- * becomes an eight-byte load before NativeCallPlanRet.mem can narrow the
+ * becomes an eight-byte load before NativeCallPhaseRet.mem can narrow the
* final move. Once one exact-width part is present, marshal every return
* part here in ABI order. aa_load_part retains the exact byte count and
* uses only backend-private temporaries, so already-filled x0/x1 or v0..v3
@@ -4040,7 +4031,7 @@ static void aa_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
return;
}
- rets = arena_zarray(t->c->tu, NativeCallPlanRet, abi->ret.nparts);
+ rets = arena_zarray(t->c->tu, NativeCallPhaseRet, abi->ret.nparts);
for (u32 p = 0; p < abi->ret.nparts; ++p) {
const ABIArgPart* part = &abi->ret.parts[p];
NativeAllocClass cls =
@@ -4063,7 +4054,7 @@ static void aa_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
nr++;
}
} else if (value) {
- rets = arena_zarray(t->c->tu, NativeCallPlanRet, 1);
+ rets = arena_zarray(t->c->tu, NativeCallPhaseRet, 1);
rets[0].src = *value;
rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, 0);
rets[0].mem = aa_mem_for_type(t, value->type, 0);
@@ -4672,25 +4663,34 @@ static void aa_trap(NativeTarget* t) { aa_emit32(t->mc, aa64_brk(0)); }
/* file_scope_asm + finalize are shared (cg/native_asm.h). */
static int aa_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
- u32 mask[NATIVE_CALL_PLAN_CLASSES]) {
+ u32 mask[NATIVE_REG_CLASS_COUNT]) {
mask[0] = mask[1] = mask[2] = 0;
- if ((NativeMachineOpKind)op->kind == NATIVE_MOP_TLS_ADDR) {
- /* ELF Local-Exec materializes the address using only the destination
- * register (mrs tpidr_el0 + add/add into rd) — no extra clobbers. The
- * Mach-O TLV sequence loads the descriptor into x0 and calls the resolver
- * thunk through backend-private x16, also clobbering private x17/LR. Only
- * x0 can hold an optimizer-owned value, so it is the sole effect at this
- * boundary. */
- if (!obj_format_tls_via_descriptor(t->c)) return 0;
- mask[NATIVE_REG_INT] = 1u << 0;
- return 1;
+ switch ((NativeMachineOpKind)op->kind) {
+ case NATIVE_MOP_TLS_ADDR:
+ /* ELF Local-Exec materializes the address using only the destination
+ * register. Mach-O TLV calls its resolver through private x16/x17/LR;
+ * only x0 can hold an optimizer-owned value at this boundary. */
+ if (!obj_format_tls_via_descriptor(t->c)) return 0;
+ mask[NATIVE_REG_INT] = 1u << 0;
+ return 1;
+ case NATIVE_MOP_INTRINSIC:
+ if ((IntrinKind)op->intrin != INTRIN_SYSCALL) return 0;
+ mask[NATIVE_REG_INT] = (1u << 0) | (1u << 1) | (1u << 2) | (1u << 3) |
+ (1u << 4) | (1u << 5) | (1u << AA_X8);
+ return 1;
+ case NATIVE_MOP_BINOP:
+ case NATIVE_MOP_BITFIELD_LOAD:
+ case NATIVE_MOP_BITFIELD_STORE:
+ case NATIVE_MOP_VA_START:
+ case NATIVE_MOP_VA_ARG:
+ case NATIVE_MOP_ATOMIC_CAS:
+ case NATIVE_MOP_ATOMIC_RMW:
+ return 0;
+ case NATIVE_MOP_COUNT:
+ break;
}
- if ((NativeMachineOpKind)op->kind != NATIVE_MOP_INTRINSIC ||
- (IntrinKind)op->intrin != INTRIN_SYSCALL)
- return 0;
- mask[NATIVE_REG_INT] = (1u << 0) | (1u << 1) | (1u << 2) | (1u << 3) |
- (1u << 4) | (1u << 5) | (1u << AA_X8);
- return 1;
+ aa_panic(aa_of(t), "invalid machine-effect operation");
+ return 0;
}
static void aa_set_loc(NativeTarget* t, SrcLoc loc) {
@@ -4725,6 +4725,8 @@ static const Reg aa_ndt_int_allocable[] = {
* use the same bank through its independent `scratch` policy. */
static const Reg aa_int_scratch[] = {9u, 10u, 11u};
static const Reg aa_int_asm_temps[] = {AA_TMP0, AA_TMP1};
+static const Reg aa_direct_asm_int[] = {0u, 1u, 2u, 3u, 4u, 5u, 6u,
+ 7u, 8u, 11u, 12u, 13u, 14u, 15u};
/* ABI fp argument/return registers (v0..v7) come FIRST (same rationale as the
* int pool: fp args land in place, the shuffle is a no-op). Then the caller-
* saved fp temps v18/v19, v22..v31 — the rest of the -O0 value cache under
@@ -4736,6 +4738,9 @@ static const Reg aa_ndt_fp_allocable[] = {
0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 18u, 19u, 22u, 23u, 24u, 25u,
26u, 27u, 28u, 29u, 30u, 31u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, 15u};
static const Reg aa_fp_scratch[] = {20u, 21u};
+static const Reg aa_direct_asm_fp[] = {
+ 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 16u, 17u, 18u,
+ 19u, 22u, 23u, 24u, 25u, 26u, 27u, 28u, 29u, 30u, 31u};
#define AA_PHYS_INT_ALLOC(r) \
{.reg = (r), \
@@ -4766,25 +4771,29 @@ static const Reg aa_fp_scratch[] = {20u, 21u};
.flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
.spill_cost = 4u, \
.copy_cost = 1u}
-#define AA_PHYS_INT_RESERVED(r) \
- {.reg = (r), \
- .cls = NATIVE_REG_INT, \
- .abi_index = 0xffu, \
- .flags = NATIVE_REG_RESERVED, \
- .spill_cost = 0u, \
+#define AA_PHYS_INT_RESERVED_ROLE(r, role) \
+ {.reg = (r), \
+ .cls = NATIVE_REG_INT, \
+ .abi_index = 0xffu, \
+ .flags = NATIVE_REG_RESERVED | (role), \
+ .spill_cost = 0u, \
.copy_cost = 0u}
+#define AA_PHYS_INT_RESERVED(r) \
+ AA_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_NONE)
+#define AA_PHYS_INT_RESERVED_CALLER(r) \
+ AA_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_CALLER_SAVED)
static const NativePhysRegInfo aa_int_phys[] = {
AA_PHYS_INT_ARG(0u), AA_PHYS_INT_ARG(1u),
AA_PHYS_INT_ARG(2u), AA_PHYS_INT_ARG(3u),
AA_PHYS_INT_ARG(4u), AA_PHYS_INT_ARG(5u),
AA_PHYS_INT_ARG(6u), AA_PHYS_INT_ARG(7u),
- AA_PHYS_INT_ALLOC(8u), AA_PHYS_INT_RESERVED(9u),
- AA_PHYS_INT_RESERVED(10u), AA_PHYS_INT_RESERVED(11u),
+ AA_PHYS_INT_ALLOC(8u), AA_PHYS_INT_RESERVED_CALLER(9u),
+ AA_PHYS_INT_RESERVED_CALLER(10u), AA_PHYS_INT_RESERVED_CALLER(11u),
AA_PHYS_INT_ALLOC(12u), AA_PHYS_INT_ALLOC(13u),
AA_PHYS_INT_ALLOC(14u), AA_PHYS_INT_ALLOC(15u),
- AA_PHYS_INT_RESERVED(16u), AA_PHYS_INT_RESERVED(17u),
- AA_PHYS_INT_RESERVED(18u), AA_PHYS_INT_CALLEE(19u),
+ AA_PHYS_INT_RESERVED_CALLER(16u), AA_PHYS_INT_RESERVED_CALLER(17u),
+ AA_PHYS_INT_RESERVED_CALLER(18u), AA_PHYS_INT_CALLEE(19u),
AA_PHYS_INT_CALLEE(20u), AA_PHYS_INT_CALLEE(21u),
AA_PHYS_INT_CALLEE(22u), AA_PHYS_INT_CALLEE(23u),
AA_PHYS_INT_CALLEE(24u), AA_PHYS_INT_CALLEE(25u),
@@ -4822,12 +4831,13 @@ static const NativePhysRegInfo aa_int_phys[] = {
.flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
.spill_cost = 4u, \
.copy_cost = 1u}
-#define AA_PHYS_FP_RESERVED(r) \
- {.reg = (r), \
- .cls = NATIVE_REG_FP, \
- .abi_index = 0xffu, \
- .flags = NATIVE_REG_RESERVED, \
- .spill_cost = 0u, \
+#define AA_PHYS_FP_RESERVED(r) \
+ {.reg = (r), \
+ .cls = NATIVE_REG_FP, \
+ .abi_index = 0xffu, \
+ .flags = NATIVE_REG_RESERVED | \
+ NATIVE_REG_CALLER_SAVED, \
+ .spill_cost = 0u, \
.copy_cost = 0u}
static const NativePhysRegInfo aa_fp_phys[] = {
@@ -4855,15 +4865,12 @@ static const NativeAllocClassInfo aa_classes[] = {
.nemit_temps = sizeof aa_int_scratch / sizeof aa_int_scratch[0],
.asm_temps = aa_int_asm_temps,
.nasm_temps = sizeof aa_int_asm_temps / sizeof aa_int_asm_temps[0],
+ .direct_asm_allocable = aa_direct_asm_int,
+ .ndirect_asm_allocable =
+ sizeof aa_direct_asm_int / sizeof aa_direct_asm_int[0],
.emit_cache_mask = (1u << 9u) | (1u << 10u) | (1u << 11u),
.phys = aa_int_phys,
- .nphys = sizeof aa_int_phys / sizeof aa_int_phys[0],
- .caller_saved_mask = 0x0007ffffu,
- .callee_saved_mask = 0x1ff80000u,
- .arg_mask = 0x000000ffu,
- .ret_mask = 0x00000003u,
- .reserved_mask =
- (1u << AA_TMP0) | (1u << AA_TMP1) | (1u << AA_FP) | (1u << AA_LR)},
+ .nphys = sizeof aa_int_phys / sizeof aa_int_phys[0]},
{.cls = NATIVE_REG_FP,
.ndt_allocable = aa_ndt_fp_allocable,
.ndt_allocable_count =
@@ -4872,15 +4879,12 @@ static const NativeAllocClassInfo aa_classes[] = {
.nscratch = sizeof aa_fp_scratch / sizeof aa_fp_scratch[0],
.emit_temps = aa_fp_scratch,
.nemit_temps = sizeof aa_fp_scratch / sizeof aa_fp_scratch[0],
+ .direct_asm_allocable = aa_direct_asm_fp,
+ .ndirect_asm_allocable =
+ sizeof aa_direct_asm_fp / sizeof aa_direct_asm_fp[0],
.emit_cache_mask = (1u << 20u) | (1u << 21u),
.phys = aa_fp_phys,
- .nphys = sizeof aa_fp_phys / sizeof aa_fp_phys[0],
- /* v8..v15 are callee-saved (low 64 bits per AAPCS64); the rest are
- * caller-saved. */
- .caller_saved_mask = 0xffff00ffu,
- .callee_saved_mask = 0x0000ff00u,
- .arg_mask = 0x000000ffu,
- .ret_mask = 0x0000000fu},
+ .nphys = sizeof aa_fp_phys / sizeof aa_fp_phys[0]},
};
/* Resolve a register name ("x8", "v3", ...) to its (class, Reg). Powers the
@@ -5042,9 +5046,9 @@ NativeTarget* aa64_native_target_new(Compiler* c, ObjBuilder* obj,
t->alloca_ = aa_alloca;
t->spill = aa_spill;
t->reload = aa_reload;
- t->plan_call = aa_plan_call;
+ t->marshal_call = aa_marshal_call;
t->emit_call = aa_emit_call;
- t->plan_ret = aa_plan_ret;
+ t->marshal_ret = aa_marshal_ret;
t->ret = aa_ret;
t->atomic_load = aa_atomic_load;
t->atomic_store = aa_atomic_store;
@@ -5588,29 +5592,6 @@ static void aa_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
}
}
-AA_UNUSED_FN static Reg aa_asm_alloc_reg(NativeDirectTarget* d,
- NativeAllocClass cls, u32 allowed_mask,
- u32* used_int, u32* used_fp) {
- static const Reg int_pool[] = {0u, 1u, 2u, 3u, 4u, 5u, 6u,
- 7u, 8u, 11u, 12u, 13u, 14u, 15u};
- static const Reg fp_pool[] = {0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u,
- 16u, 17u, 18u, 19u, 22u, 23u, 24u, 25u,
- 26u, 27u, 28u, 29u, 30u, 31u};
- const Reg* pool = cls == NATIVE_REG_FP ? fp_pool : int_pool;
- u32 n = cls == NATIVE_REG_FP ? (u32)(sizeof fp_pool / sizeof fp_pool[0])
- : (u32)(sizeof int_pool / sizeof int_pool[0]);
- u32* used = cls == NATIVE_REG_FP ? used_fp : used_int;
- for (u32 i = 0; i < n; ++i) {
- Reg r = pool[i];
- if (allowed_mask && (allowed_mask & (1u << r)) == 0) continue;
- if ((*used & (1u << r)) != 0) continue;
- *used |= 1u << r;
- return r;
- }
- aa_asm_panic(d, "out of registers for asm operands");
- return REG_NONE;
-}
-
/* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
AA_UNUSED_FN static void aa_direct_load_operand_to_reg(NativeDirectTarget* d,
@@ -5764,16 +5745,11 @@ static void aa_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
const Sym* clobbers, u32 nclob,
u32 clobber_abi_sets) {
static const NativeAsmDirectHooks hooks = {
- /* Reserve backend-private x16/x17, x18 (platform reg), fp/lr/sp. */
- .reserved_int_mask = (1u << AA_TMP0) | (1u << AA_TMP1) | (1u << 18u) |
- (1u << AA_FP) | (1u << AA_LR) | (1u << AA_SP),
- .reserved_fp_mask = (1u << 20u) | (1u << 21u),
.opk_reg = AA64_INLINE_OPK_REG,
.opcls_fp = AA64_INLINE_OPCLS_FP,
.panic = aa_asm_panic,
.bound_reg = aa_asm_bound_reg,
.bound_mem = aa_asm_bound_mem,
- .alloc_reg = aa_asm_alloc_reg,
.clobber_masks = aa_asm_clobber_masks,
.save_callee_clobbers = aa_asm_hook_save_callee_clobbers,
.restore_one = aa_asm_hook_restore_one,
diff --git a/src/arch/arm32/native.c b/src/arch/arm32/native.c
@@ -200,13 +200,15 @@ static void arm_emit_pop(MCEmitter* mc, u32 reglist) {
.flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
.spill_cost = 4u, \
.copy_cost = 1u}
-#define ARM_PHYS_RESERVED(r) \
- {.reg = (r), \
- .cls = NATIVE_REG_INT, \
- .abi_index = 0xffu, \
- .flags = NATIVE_REG_RESERVED, \
- .spill_cost = 0u, \
+#define ARM_PHYS_RESERVED_ROLE(r, role) \
+ {.reg = (r), \
+ .cls = NATIVE_REG_INT, \
+ .abi_index = 0xffu, \
+ .flags = NATIVE_REG_RESERVED | (role), \
+ .spill_cost = 0u, \
.copy_cost = 0u}
+#define ARM_PHYS_RESERVED(r) \
+ ARM_PHYS_RESERVED_ROLE((r), NATIVE_REG_NONE)
/* NDT (-O0) value-cache pool: the caller-saved arg registers r0..r3, fronted so
* a producer materializes a call's args directly into their ABI registers
@@ -221,15 +223,18 @@ static const Reg arm_int_scratch[] = {ARM_SCRATCH};
* exposing either as an operand location would couple independent lifetimes. */
static const Reg arm_int_emit_temps[] = {0u, 1u, 2u, 3u};
static const Reg arm_int_asm_temps[] = {ARM_TMP};
+static const Reg arm_direct_asm_int[] = {0u, 1u, 2u, 3u, 4u, 5u,
+ 6u, 8u, 9u, 10u, 11u};
static const NativePhysRegInfo arm_int_phys[] = {
ARM_PHYS_ARG(0u, 0u), ARM_PHYS_ARG(1u, 1u),
ARM_PHYS_ARG(2u, 2u), ARM_PHYS_ARG(3u, 3u),
ARM_PHYS_CALLEE(4u), ARM_PHYS_CALLEE(5u),
- ARM_PHYS_CALLEE(6u), ARM_PHYS_RESERVED(7u), /* fp */
+ ARM_PHYS_CALLEE(6u),
+ ARM_PHYS_RESERVED_ROLE(7u, NATIVE_REG_CALLEE_SAVED), /* fp */
ARM_PHYS_CALLEE(8u), ARM_PHYS_CALLEE(9u),
ARM_PHYS_CALLEE(10u), ARM_PHYS_CALLEE(11u),
- ARM_PHYS_RESERVED(12u), /* ip = O0/backend temp */
+ ARM_PHYS_RESERVED_ROLE(12u, NATIVE_REG_CALLER_SAVED), /* ip/temp */
ARM_PHYS_RESERVED(13u), /* sp */
ARM_PHYS_RESERVED(14u), /* lr */
ARM_PHYS_RESERVED(15u), /* pc */
@@ -288,17 +293,12 @@ static const NativeAllocClassInfo arm_classes[] = {
.nemit_temps = sizeof arm_int_emit_temps / sizeof arm_int_emit_temps[0],
.asm_temps = arm_int_asm_temps,
.nasm_temps = sizeof arm_int_asm_temps / sizeof arm_int_asm_temps[0],
+ .direct_asm_allocable = arm_direct_asm_int,
+ .ndirect_asm_allocable =
+ sizeof arm_direct_asm_int / sizeof arm_direct_asm_int[0],
.emit_cache_mask = 0x0000000fu, /* r0..r3; hooks use private ip/lr */
.phys = arm_int_phys,
- .nphys = sizeof arm_int_phys / sizeof arm_int_phys[0],
- /* r0..r3 + r12 (lr/ip caller-saved; lr reserved) */
- .caller_saved_mask = 0x0000100fu,
- /* r4..r11 */
- .callee_saved_mask = 0x00000ff0u,
- .arg_mask = 0x0000000fu,
- .ret_mask = 0x00000003u,
- /* r7 (fp), r12 (ip), r13 (sp), r14 (lr), r15 (pc) */
- .reserved_mask = 0x0000f080u},
+ .nphys = sizeof arm_int_phys / sizeof arm_int_phys[0]},
};
static const NativeRegInfo arm_reg_info = {
@@ -1298,18 +1298,30 @@ static int arm_reg_is_callee_int(Reg r) {
return r >= 4u && r <= 11u && r != ARM_FP;
}
+static void arm_asm_clobber_masks(Compiler* c, SrcLoc loc,
+ const Sym* clobbers, u32 nclob,
+ u32* int_mask, u32* fp_mask);
+
/* Build the callee-saved set the prologue must preserve, from the allocator's
- * per-class masks. Only the integer class exists in this soft-float backend, so
- * the FP/vec classes are ignored; inline asm is unsupported (asm_block is not
- * installed), so there are no opaque asm clobbers to fold in. Restricts to the
- * register file's actual callee-saved set as a guard. */
-static u32 arm_known_callee_saves(const NativeKnownFrameDesc* frame) {
- u32 mask = 0;
- if (frame && frame->callee_saved_used && frame->ncallee_classes > NATIVE_REG_INT)
- mask = frame->callee_saved_used[NATIVE_REG_INT];
- for (Reg r = 0; r < 16u; ++r)
- if ((mask & (1u << r)) && !arm_reg_is_callee_int(r)) mask &= ~(1u << r);
- return mask;
+ * per-class masks plus opaque inline-asm clobbers. Only the integer class
+ * exists in this soft-float backend. r7 is structural frame state and is
+ * excluded here (and rejected when named explicitly by the clobber parser). */
+static int arm_asm_reg_is_callee_saved(NativeTarget* t, NativeAllocClass cls,
+ Reg r) {
+ (void)t;
+ return cls == NATIVE_REG_INT && arm_reg_is_callee_int(r);
+}
+
+static u32 arm_known_callee_saves(NativeTarget* t,
+ const NativeKnownFrameDesc* frame) {
+ Arm32NativeTarget* a = arm_of(t);
+ u32 masks[NATIVE_REG_CLASS_COUNT] = {0, 0, 0};
+ SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
+ native_asm_known_callee_saves(t, loc, frame, masks,
+ NATIVE_REG_CLASS_COUNT,
+ arm_asm_clobber_masks,
+ arm_asm_reg_is_callee_saved);
+ return masks[NATIVE_REG_INT];
}
/* True (and sets *imm7) when `n` fits the 16-bit `SUB sp,sp,#imm7*4` form:
@@ -1482,11 +1494,12 @@ static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
Arm32NativeTarget* a = arm_of(t);
MCEmitter* mc = t->mc;
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
- u32 cs_mask = arm_known_callee_saves(kf);
+ u32 cs_mask;
u32 reglist, nsaved, frame, i;
a->func = fd;
a->loc = fd->loc;
+ cs_mask = arm_known_callee_saves(t, kf);
native_frame_reset(&a->frame);
a->next_param_int = 0;
a->next_param_stack = 0;
@@ -1511,9 +1524,9 @@ static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
/* Record the callee-save set (for parity / debug) and build the body's slots
* and outgoing-area reservation, then settle the exact frame. */
if (cs_mask) {
- u32 used[NATIVE_CALL_PLAN_CLASSES] = {0, 0, 0};
+ u32 used[NATIVE_REG_CLASS_COUNT] = {0, 0, 0};
used[NATIVE_REG_INT] = cs_mask;
- arm_reserve_callee_saves(t, used, NATIVE_CALL_PLAN_CLASSES);
+ arm_reserve_callee_saves(t, used, NATIVE_REG_CLASS_COUNT);
}
if (kf) {
for (i = 0; i < kf->nslots; ++i) {
@@ -1680,8 +1693,8 @@ static void arm_func_end(NativeTarget* t) {
#define ARM32_GPR_BYTES 4u
/* Lane scalar type for an ABI part: an 8-byte i64/double DIRECT argument is two
- * 4-byte INT lanes, an aggregate up to four word/sub-word lanes. The NDT sizes
- * every load/store/move it emits for a plan-return / plan-call / param-bind move
+ * 4-byte INT lanes, and an aggregate uses word/sub-word lanes. The NDT sizes
+ * every load/store/move it emits for return/call marshalling or a param bind
* from the lane location's TYPE (nd_type_mem -> cg_type_size), so each lane must
* carry a type whose natural size equals the lane's byte width — NOT the full
* 8-byte scalar type (which would drive an 8-byte access the 32-bit backend has
@@ -1724,9 +1737,9 @@ static void arm_bind_native_param(NativeTarget* t, const CGParamDesc* p,
u32 i;
if (!ai || ai->kind == ABI_ARG_IGNORE) return;
if (ai->kind == ABI_ARG_INDIRECT) {
- /* Large aggregate passed by reference: the incoming arg is a pointer to the
- * caller's copy. Copy it into the parameter's frame home so the body owns a
- * private copy (matching the by-value semantics). */
+ /* An ABI-classified indirect argument is a pointer to caller-owned storage.
+ * Copy it into the parameter's frame home so the body owns a private copy
+ * (matching the by-value semantics). */
KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
NativeLoc src = native_loc_reg(i32t, NATIVE_REG_INT,
a->next_param_int < 4u
@@ -1868,7 +1881,7 @@ static const ABIArgInfo* arm_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
* alignment so the AAPCS even/8-align walk treats a synthesized variadic
* i64/double (or 8-aligned aggregate) the same as a named one. */
parts[0].align = align;
- scratch->nparts = (u16)nparts;
+ scratch->nparts = nparts;
scratch->parts = parts;
return scratch;
}
@@ -1986,7 +1999,7 @@ static void arm_store_outgoing(NativeTarget* t, u32 stack_off, NativeLoc src,
}
/* Materialize the address of a frame/stack-resident NativeLoc into `dst` (used
- * to pass a large INDIRECT aggregate argument by reference). */
+ * for arguments that the ABI explicitly classifies as indirect). */
static void arm_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
NativeAddr addr;
if (!native_loc_storage_addr(src, 0, &addr))
@@ -2011,21 +2024,22 @@ static void arm_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves,
s.emit_one = arm_emit_one_arg_move;
s.reg_move = arm_move;
s.scratch[NATIVE_REG_INT] = ARM_SCRATCH;
+ s.scratch_class_mask = 1u << NATIVE_REG_INT;
native_arg_shuffle(&s, moves, n);
}
-static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
- NativeCallPlan* plan) {
+static void arm_marshal_call(NativeTarget* t, const NativeCallDesc* desc,
+ NativeCallPhase* plan) {
Arm32NativeTarget* a = arm_of(t);
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
- NativeCallPlanRet* rets;
+ NativeCallPhaseRet* rets;
u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
? abi->ret.nparts
: ((!abi && desc->nresults) ? 1u : 0u);
int tail = (desc->flags & CG_CALL_TAIL) != 0;
int tail_callee_saved = 0;
memset(plan, 0, sizeof *plan);
- rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPlanRet, nrets_cap) : NULL;
+ rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPhaseRet, nrets_cap) : NULL;
plan->callee = desc->callee;
plan->rets = rets;
plan->flags = desc->flags;
@@ -2107,9 +2121,9 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
int even = arm_arg_needs_even(ai);
if (ai->kind == ABI_ARG_IGNORE) continue;
if (ai->kind == ABI_ARG_INDIRECT) {
- /* Pass a large aggregate by reference: a pointer to the caller's copy
- * (the callee makes its own copy in bind_param). The pointer rides a
- * core reg when one is free, else the outgoing stack. */
+ /* Pass an ABI-classified indirect argument as a pointer to the caller's
+ * copy (the callee makes its own copy in bind_param). The pointer rides
+ * a core reg when one is free, else the outgoing stack. */
if (next_int < 4u) {
NativeArgMove* m = &moves[nmoves++];
m->dst = native_loc_reg(i32t, NATIVE_REG_INT, (Reg)(next_int++));
@@ -2202,7 +2216,7 @@ static void arm_plan_call(NativeTarget* t, const NativeCallDesc* desc,
}
}
-static void arm_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
+static void arm_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
Arm32NativeTarget* a = arm_of(t);
MCEmitter* mc = t->mc;
if (plan->flags & CG_CALL_TAIL) {
@@ -2241,7 +2255,7 @@ static void arm_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
}
if (plan->callee.kind == NATIVE_LOC_STACK) {
/* -O0 indirect call with stack args: the callee was spilled to a frame home
- * (arm_plan_call) to free the backend staging registers. Arg staging is
+ * (arm_marshal_call) to free the backend staging registers. Arg staging is
* done now, so IP is free; reload the target and BLX it. */
NativeLoc tmp = native_loc_reg(plan->callee.type, NATIVE_REG_INT, ARM_SCRATCH);
NativeAddr home;
@@ -2256,14 +2270,14 @@ static void arm_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
arm_panic(a, "unsupported call target");
}
-static void arm_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
- const NativeLoc* value, NativeCallPlanRet** out_rets,
- u32* out_nrets) {
+static void arm_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
+ const NativeLoc* value,
+ NativeCallPhaseRet** out_rets, u32* out_nrets) {
Arm32NativeTarget* a = arm_of(t);
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
- NativeCallPlanRet* rets = NULL;
+ NativeCallPhaseRet* rets = NULL;
u32 nr = 0;
- if (value) rets = arena_zarray(t->c->tu, NativeCallPlanRet, 4);
+ if (value) rets = arena_zarray(t->c->tu, NativeCallPhaseRet, 4);
if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
/* Large aggregate return: copy the value into the caller-provided
* destination via the hidden sret pointer spilled at entry. The body's
@@ -3402,8 +3416,9 @@ static void arm_asm_bound_mem(Operand* out, KitCgTypeId type, Reg base) {
out->v.ind.index = CG_LOCAL_NONE;
}
-/* Parse one clobber name ("r0".."r12", "lr"/"sp", "cc", "memory") into a
- * register mask bit; cc/memory contribute no register. */
+/* Parse one clobber name into a core-register mask bit. r7 is the live frame
+ * anchor and sp/pc are structural state, so an inline block may not declare
+ * them clobbered; cc/memory contribute no register. */
static void arm_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
u32 nclob, u32* int_mask, u32* fp_mask) {
*int_mask = 0;
@@ -3416,28 +3431,13 @@ static void arm_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
memcpy(buf, s.s, s.len);
buf[s.len] = '\0';
if (!strcmp(buf, "cc") || !strcmp(buf, "memory")) continue;
- if (arm32_register_index(buf, &dwarf) == 0 && dwarf <= 15u)
+ if (arm32_register_index(buf, &dwarf) != 0 || dwarf > 15u)
+ arm_asm_panic_at(c, loc, "unsupported clobber register");
+ if (dwarf == ARM_FP || dwarf == 13u || dwarf == 15u)
+ arm_asm_panic_at(c, loc, "structural register cannot be clobbered");
+ if (dwarf <= 15u)
*int_mask |= 1u << dwarf;
}
- (void)loc;
-}
-
-/* Allocate a free operand register from the core pool, honoring allowed_mask and
- * the running used set. r7 (fp), r12 (ip), r13 (sp), r14 (lr), r15 (pc) are
- * never handed out. */
-static Reg arm_asm_alloc_reg(NativeDirectTarget* d, NativeAllocClass cls,
- u32 allowed_mask, u32* used_int, u32* used_fp) {
- static const Reg int_pool[] = {0u, 1u, 2u, 3u, 4u, 5u, 6u, 8u, 9u, 10u, 11u};
- (void)used_fp;
- if (cls != NATIVE_REG_INT) arm_asm_panic(d, "no FP registers (soft-float)");
- for (u32 i = 0; i < sizeof int_pool / sizeof int_pool[0]; ++i) {
- Reg r = int_pool[i];
- if (allowed_mask && (allowed_mask & (1u << r)) == 0) continue;
- if ((*used_int & (1u << r)) != 0) continue;
- *used_int |= 1u << r;
- return r;
- }
- arm_asm_panic(d, "out of registers for asm operands");
}
/* Operand-address resolution for the direct (-O0) path (mirrors aa64). */
@@ -3612,17 +3612,11 @@ static void arm_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
const Sym* clobbers, u32 nclob,
u32 clobber_abi_sets) {
static const NativeAsmDirectHooks hooks = {
- /* Reserve backend/O0 IP plus FP, SP, LR, and PC from this direct-path
- * operand allocator. */
- .reserved_int_mask = (1u << ARM_SCRATCH) | (1u << ARM_FP) |
- (1u << 13u) | (1u << 14u) | (1u << 15u),
- .reserved_fp_mask = 0u,
.opk_reg = ARM32_INLINE_OPK_REG,
.opcls_fp = ARM32_INLINE_OPCLS_FP,
.panic = arm_asm_panic,
.bound_reg = arm_asm_bound_reg,
.bound_mem = arm_asm_bound_mem,
- .alloc_reg = arm_asm_alloc_reg,
.clobber_masks = arm_asm_clobber_masks,
.save_callee_clobbers = arm_asm_hook_save_callee_clobbers,
.restore_one = arm_asm_hook_restore_one,
@@ -3772,9 +3766,9 @@ NativeTarget* arm32_native_target_new(Compiler* c, ObjBuilder* obj,
t->alloca_ = arm_alloca;
t->spill = arm_spill;
t->reload = arm_reload;
- t->plan_call = arm_plan_call;
+ t->marshal_call = arm_marshal_call;
t->emit_call = arm_emit_call;
- t->plan_ret = arm_plan_ret;
+ t->marshal_ret = arm_marshal_ret;
t->ret = arm_ret;
t->atomic_load = arm_atomic_load;
t->atomic_store = arm_atomic_store;
diff --git a/src/arch/native_target.c b/src/arch/native_target.c
@@ -0,0 +1,158 @@
+#include "arch/native_target.h"
+
+static const NativePhysRegInfo* native_class_phys_reg(
+ const NativeAllocClassInfo* ci, Reg reg) {
+ for (u32 i = 0; ci && i < ci->nphys; ++i)
+ if (ci->phys[i].reg == reg) return &ci->phys[i];
+ return NULL;
+}
+
+const NativeAllocClassInfo* native_reg_info_class_info(
+ const NativeRegInfo* ri, NativeAllocClass cls) {
+ if (!ri || cls >= NATIVE_REG_CLASS_COUNT) return NULL;
+ for (u32 i = 0; i < ri->nclasses; ++i) {
+ const NativeAllocClassInfo* ci = &ri->classes[i];
+ if ((NativeAllocClass)ci->cls == cls) return ci;
+ }
+ return NULL;
+}
+
+u32 native_reg_info_flag_mask(const NativeRegInfo* ri, NativeAllocClass cls,
+ u16 flag) {
+ const NativeAllocClassInfo* ci = native_reg_info_class_info(ri, cls);
+ u32 mask = 0;
+ for (u32 i = 0; ci && i < ci->nphys; ++i)
+ if ((ci->phys[i].flags & flag) != 0 &&
+ ci->phys[i].reg < NATIVE_MAX_HARD_REGS)
+ mask |= 1u << ci->phys[i].reg;
+ return mask;
+}
+
+static u32 native_validate_reg_list(Compiler* c,
+ const NativeAllocClassInfo* ci,
+ const Reg* regs, u32 nregs,
+ const char* list_name) {
+ u32 mask = 0;
+ if ((nregs != 0u) != (regs != NULL))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u has inconsistent %s list",
+ (unsigned)ci->cls, list_name);
+ if (nregs > NATIVE_MAX_HARD_REGS)
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u has too many %s registers "
+ "(%u)",
+ (unsigned)ci->cls, list_name, (unsigned)nregs);
+ for (u32 i = 0; i < nregs; ++i) {
+ Reg reg = regs[i];
+ const NativePhysRegInfo* phys;
+ if (reg >= NATIVE_MAX_HARD_REGS || (mask & (1u << reg)))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u has invalid/duplicate "
+ "%s register %u",
+ (unsigned)ci->cls, list_name, (unsigned)reg);
+ phys = native_class_phys_reg(ci, reg);
+ if (!phys || phys->cls != ci->cls)
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u %s register %u is not a "
+ "matching physical register",
+ (unsigned)ci->cls, list_name, (unsigned)reg);
+ mask |= 1u << reg;
+ }
+ return mask;
+}
+
+void native_reg_info_validate(Compiler* c, const NativeRegInfo* ri) {
+ u32 seen = 0;
+ if (!ri) return;
+ if (ri->nclasses > NATIVE_REG_CLASS_COUNT ||
+ (ri->nclasses != 0u) != (ri->classes != NULL))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: invalid register class table");
+ for (u32 n = 0; n < ri->nclasses; ++n) {
+ const NativeAllocClassInfo* ci = &ri->classes[n];
+ u32 cls = ci->cls;
+ u32 known_mask = 0;
+ u32 emit_mask;
+ u32 asm_mask;
+ if (cls >= NATIVE_REG_CLASS_COUNT || (seen & (1u << cls)))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: duplicate/invalid register class "
+ "%u",
+ (unsigned)cls);
+ seen |= 1u << cls;
+ if ((ci->nphys != 0u) != (ci->phys != NULL) ||
+ ci->nphys > NATIVE_MAX_HARD_REGS)
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u has invalid physical "
+ "table size %u",
+ (unsigned)cls, (unsigned)ci->nphys);
+ for (u32 i = 0; i < ci->nphys; ++i) {
+ const NativePhysRegInfo* phys = &ci->phys[i];
+ if (phys->reg >= NATIVE_MAX_HARD_REGS || phys->cls != cls ||
+ (known_mask & (1u << phys->reg)))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u has invalid physical "
+ "register %u",
+ (unsigned)cls, (unsigned)phys->reg);
+ if ((phys->flags & NATIVE_REG_CALLER_SAVED) &&
+ (phys->flags & NATIVE_REG_CALLEE_SAVED))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u register %u is both "
+ "caller- and callee-saved",
+ (unsigned)cls, (unsigned)phys->reg);
+ if ((phys->flags & NATIVE_REG_ALLOCABLE) &&
+ (phys->flags & NATIVE_REG_RESERVED))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u register %u is both "
+ "allocable and reserved",
+ (unsigned)cls, (unsigned)phys->reg);
+ known_mask |= 1u << phys->reg;
+ }
+ (void)native_validate_reg_list(c, ci, ci->ndt_allocable,
+ ci->ndt_allocable_count, "O0 allocable");
+ (void)native_validate_reg_list(c, ci, ci->scratch, ci->nscratch,
+ "O0 scratch");
+ emit_mask = native_validate_reg_list(c, ci, ci->emit_temps,
+ ci->nemit_temps, "emit temp");
+ asm_mask = native_validate_reg_list(c, ci, ci->asm_temps, ci->nasm_temps,
+ "asm-internal temp");
+ (void)native_validate_reg_list(c, ci, ci->direct_asm_allocable,
+ ci->ndirect_asm_allocable,
+ "direct-asm allocable");
+ if (emit_mask & asm_mask)
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u emit and asm temp banks "
+ "overlap",
+ (unsigned)cls);
+ if (asm_mask &
+ ~native_reg_info_flag_mask(ri, (NativeAllocClass)cls,
+ NATIVE_REG_RESERVED))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u asm temp bank is not "
+ "reserved",
+ (unsigned)cls);
+ if (ci->emit_cache_mask & ~emit_mask)
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u cache mask escapes emit "
+ "temp bank",
+ (unsigned)cls);
+ for (u32 i = 0; i < ci->nemit_temps; ++i) {
+ const NativePhysRegInfo* phys =
+ native_class_phys_reg(ci, ci->emit_temps[i]);
+ if (phys && (phys->flags & NATIVE_REG_ALLOCABLE))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u emit temp %u is "
+ "allocable",
+ (unsigned)cls, (unsigned)ci->emit_temps[i]);
+ }
+ for (u32 i = 0; i < ci->nasm_temps; ++i) {
+ const NativePhysRegInfo* phys =
+ native_class_phys_reg(ci, ci->asm_temps[i]);
+ if (phys && (phys->flags & NATIVE_REG_ALLOCABLE))
+ compiler_panic(c, (SrcLoc){0, 0, 0},
+ "native register model: class %u asm temp %u is "
+ "allocable",
+ (unsigned)cls, (unsigned)ci->asm_temps[i]);
+ }
+ }
+}
diff --git a/src/arch/native_target.h b/src/arch/native_target.h
@@ -121,8 +121,13 @@ typedef enum NativeAllocClass {
NATIVE_REG_INT,
NATIVE_REG_FP,
NATIVE_REG_VEC,
+ NATIVE_REG_CLASS_COUNT,
} NativeAllocClass;
+#define NATIVE_MAX_HARD_REGS 32u
+
+typedef u32 NativeRegMaskSet[NATIVE_REG_CLASS_COUNT];
+
typedef enum NativeRegFlag {
NATIVE_REG_NONE = 0,
NATIVE_REG_ALLOCABLE = 1u << 0,
@@ -174,6 +179,14 @@ typedef struct NativeAllocClassInfo {
const Reg* asm_temps;
u32 nasm_temps;
+ /* Ordered register policy for operands self-allocated by the O0 direct
+ * inline-asm binder. This is distinct from ndt_allocable and scratch: values
+ * live for the complete asm block, and the order is an architecture policy.
+ * Explicit hard-register pins are validated separately and need not appear
+ * here. Selection itself is shared in cg/native_asm.c. */
+ const Reg* direct_asm_allocable;
+ u32 ndirect_asm_allocable;
+
/* Subset of reserved optimizer emitter temps whose contents survive ordinary
* backend hooks when they are not passed as operands or named by a clobber
* effect. The native emitter may retain clean FS_SPILL values in these and
@@ -185,12 +198,6 @@ typedef struct NativeAllocClassInfo {
const NativePhysRegInfo* phys;
u32 nphys;
-
- u32 caller_saved_mask;
- u32 callee_saved_mask;
- u32 arg_mask;
- u32 ret_mask;
- u32 reserved_mask;
} NativeAllocClassInfo;
typedef struct NativeRegInfo NativeRegInfo;
@@ -253,6 +260,22 @@ struct NativeRegInfo {
u32 (*dwarf_reg)(const NativeRegInfo*, NativeAllocClass, Reg);
};
+/* Validate the target-independent NativeRegInfo contract. Both the direct and
+ * optimized adapters call this at their boundary, so malformed physical rows,
+ * role flags, or temporary banks fail identically at O0 and O1. */
+void native_reg_info_validate(Compiler* c, const NativeRegInfo* ri);
+
+/* Classes are identified by NativeAllocClass, not by their position in the
+ * target's table. Keep the lookup policy shared by every register consumer. */
+const NativeAllocClassInfo* native_reg_info_class_info(
+ const NativeRegInfo* ri, NativeAllocClass cls);
+
+/* Static register roles have one authority: NativePhysRegInfo.flags. Targets
+ * with an OS-dependent preservation ABI override caller/callee masks through
+ * NativeTarget below; all other role consumers derive their masks here. */
+u32 native_reg_info_flag_mask(const NativeRegInfo* ri, NativeAllocClass cls,
+ u16 flag);
+
typedef enum NativeLocKind {
NATIVE_LOC_NONE,
NATIVE_LOC_REG,
@@ -416,8 +439,6 @@ typedef struct NativeFramePatchState {
u32 max_align;
} NativeFramePatchState;
-#define NATIVE_CALL_PLAN_CLASSES 3u
-
/* A semantic machine operation, enough for the target to report the physical
* registers its encoding clobbers as a side effect (e.g. x86 idiv writes
* rax/rdx, variable shifts use cl, and a bitfield read-modify-write borrows
@@ -439,6 +460,7 @@ typedef enum NativeMachineOpKind {
* whose TLS sequence touches only the destination register (ELF Local-Exec)
* report no clobbers. */
NATIVE_MOP_TLS_ADDR,
+ NATIVE_MOP_COUNT,
} NativeMachineOpKind;
typedef struct NativeMachineOp {
@@ -469,40 +491,38 @@ typedef struct NativeCallDesc {
KitCgInlinePolicy inline_policy;
} NativeCallDesc;
-typedef enum NativeCallPlanMoveKind {
+typedef enum NativeCallPhaseMoveKind {
NATIVE_CALL_MOVE_NONE,
NATIVE_CALL_MOVE_VALUE,
NATIVE_CALL_MOVE_ADDR,
-} NativeCallPlanMoveKind;
+} NativeCallPhaseMoveKind;
-typedef struct NativeCallPlanMove {
+typedef struct NativeCallPhaseMove {
NativeLoc src;
NativeLoc dst;
MemAccess mem;
- u8 src_kind; /* NativeCallPlanMoveKind */
+ u8 src_kind; /* NativeCallPhaseMoveKind */
u8 dst_kind; /* NativeLocKind */
u8 pad[2];
-} NativeCallPlanMove;
+} NativeCallPhaseMove;
-typedef struct NativeCallPlanRet {
+typedef struct NativeCallPhaseRet {
NativeLoc src;
NativeLoc dst;
MemAccess mem;
-} NativeCallPlanRet;
+} NativeCallPhaseRet;
-typedef struct NativeCallPlan {
+typedef struct NativeCallPhase {
NativeLoc callee;
- NativeCallPlanMove* args;
- NativeCallPlanRet* rets;
+ NativeCallPhaseMove* args;
+ NativeCallPhaseRet* rets;
u32 nargs;
u32 nrets;
u32 stack_arg_size;
- u32 clobber_mask[NATIVE_CALL_PLAN_CLASSES];
- u32 return_mask[NATIVE_CALL_PLAN_CLASSES];
u16 flags; /* CGCallFlag */
u8 has_sret;
u8 is_variadic;
-} NativeCallPlan;
+} NativeCallPhase;
typedef struct NativeTarget NativeTarget;
struct NativeTarget {
@@ -545,7 +565,7 @@ struct NativeTarget {
* clobbered, 0 otherwise. NULL promises no optimizer-visible fixed
* clobbers. */
int (*machine_op_clobbers)(NativeTarget*, const NativeMachineOp* op,
- u32 clobber_mask[NATIVE_CALL_PLAN_CLASSES]);
+ NativeRegMaskSet clobber_mask);
void (*func_begin)(NativeTarget*, const CGFuncDesc*);
void (*func_begin_known_frame)(NativeTarget*, const CGFuncDesc*,
@@ -581,7 +601,7 @@ struct NativeTarget {
* flags, nargs, and each args[i].type — never argument *locations* — so the
* optimizer can call it in a frame-planning pre-pass, before any argument
* marshalling is emitted, to size the outgoing area. Must equal the
- * stack_arg_size plan_call computes for the same descriptor. May be NULL. */
+ * stack_arg_size marshal_call computes for the same descriptor. May be NULL. */
u32 (*call_stack_bytes)(NativeTarget*, const NativeCallDesc*);
/* Integer hardware zero register, if the ISA has one (aa64 wzr/xzr, rv64
* x0). When `has_store_zero_reg` is set, the emit path stores a constant 0
@@ -686,18 +706,18 @@ struct NativeTarget {
void (*spill)(NativeTarget*, NativeLoc src_reg, NativeFrameSlot, MemAccess);
void (*reload)(NativeTarget*, NativeLoc dst_reg, NativeFrameSlot, MemAccess);
- /* Mutating ABI-marshalling phase, despite the historical `plan_` name. It
- * may stage the callee, emit stack arguments and register shuffles, and fill
- * any remaining generic moves/results in NativeCallPlan. On return, every
- * populated ABI argument destination stays live until emit_call. */
- void (*plan_call)(NativeTarget*, const NativeCallDesc*, NativeCallPlan*);
- void (*emit_call)(NativeTarget*, const NativeCallPlan*);
+ /* Mutating ABI-marshalling phase. It may stage the callee, emit stack
+ * arguments and register shuffles, and fill any remaining generic
+ * moves/results in NativeCallPhase. The phase owns every populated ABI
+ * argument destination and any staged callee until emit_call consumes it. */
+ void (*marshal_call)(NativeTarget*, const NativeCallDesc*, NativeCallPhase*);
+ void (*emit_call)(NativeTarget*, const NativeCallPhase*);
/* The corresponding mutating return-marshalling phase. `value` is the
* single returned local's location, or NULL for void; the hook may emit an
* indirect-result copy and returns the ordered register moves in
* out_rets/out_nrets. Completed destinations remain live through the phase. */
- void (*plan_ret)(NativeTarget*, const CGFuncDesc*, const NativeLoc* value,
- NativeCallPlanRet** out_rets, u32* out_nrets);
+ void (*marshal_ret)(NativeTarget*, const CGFuncDesc*, const NativeLoc* value,
+ NativeCallPhaseRet** out_rets, u32* out_nrets);
void (*ret)(NativeTarget*);
/* Like load(), a one-register atomic load must consume a plain BASE_REG
@@ -752,28 +772,38 @@ struct NativeTarget {
static inline const NativeAllocClassInfo* native_target_class_info(
const NativeTarget* t, NativeAllocClass cls) {
- if (!t || !t->regs) return NULL;
- for (u32 i = 0; i < t->regs->nclasses; ++i) {
- const NativeAllocClassInfo* ci = &t->regs->classes[i];
- if ((NativeAllocClass)ci->cls == cls) return ci;
- }
- return NULL;
+ return t ? native_reg_info_class_info(t->regs, cls) : NULL;
}
static inline u32 native_target_caller_saved_mask(NativeTarget* t,
NativeAllocClass cls) {
- const NativeAllocClassInfo* ci;
if (t && t->caller_saved_mask) return t->caller_saved_mask(t, cls);
- ci = native_target_class_info(t, cls);
- return ci ? ci->caller_saved_mask : 0u;
+ return t ? native_reg_info_flag_mask(t->regs, cls,
+ NATIVE_REG_CALLER_SAVED)
+ : 0u;
}
static inline u32 native_target_callee_saved_mask(NativeTarget* t,
NativeAllocClass cls) {
- const NativeAllocClassInfo* ci;
if (t && t->callee_saved_mask) return t->callee_saved_mask(t, cls);
- ci = native_target_class_info(t, cls);
- return ci ? ci->callee_saved_mask : 0u;
+ return t ? native_reg_info_flag_mask(t->regs, cls,
+ NATIVE_REG_CALLEE_SAVED)
+ : 0u;
+}
+
+static inline u32 native_target_arg_mask(const NativeTarget* t,
+ NativeAllocClass cls) {
+ return t ? native_reg_info_flag_mask(t->regs, cls, NATIVE_REG_ARG) : 0u;
+}
+
+static inline u32 native_target_ret_mask(const NativeTarget* t,
+ NativeAllocClass cls) {
+ return t ? native_reg_info_flag_mask(t->regs, cls, NATIVE_REG_RET) : 0u;
+}
+
+static inline u32 native_target_reserved_mask(const NativeTarget* t,
+ NativeAllocClass cls) {
+ return t ? native_reg_info_flag_mask(t->regs, cls, NATIVE_REG_RESERVED) : 0u;
}
/* Intrinsic operands whose immediate form is shared across every NativeTarget.
diff --git a/src/arch/riscv/native.c b/src/arch/riscv/native.c
@@ -438,13 +438,17 @@ static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0x1fu; }
.flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
.spill_cost = 4u, \
.copy_cost = 1u}
-#define RV_PHYS_INT_RESERVED(r) \
- {.reg = (r), \
- .cls = NATIVE_REG_INT, \
- .abi_index = 0xffu, \
- .flags = NATIVE_REG_RESERVED, \
- .spill_cost = 0u, \
+#define RV_PHYS_INT_RESERVED_ROLE(r, role) \
+ {.reg = (r), \
+ .cls = NATIVE_REG_INT, \
+ .abi_index = 0xffu, \
+ .flags = NATIVE_REG_RESERVED | (role), \
+ .spill_cost = 0u, \
.copy_cost = 0u}
+#define RV_PHYS_INT_RESERVED(r) \
+ RV_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_NONE)
+#define RV_PHYS_INT_RESERVED_CALLER(r) \
+ RV_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_CALLER_SAVED)
/* t0..t3 (x5,x6,x7,x28) are backend-private hook/asm temporaries
* (RV_TMP0..RV_TMP3), reserved and never exposed as optimizer operand
@@ -471,6 +475,8 @@ static const Reg rv_ndt_int_allocable[] = {10u, 11u, 12u, 13u, 14u,
15u, 16u, 17u}; /* a0..a7 */
static const Reg rv_int_scratch[] = {29u, 30u, 31u}; /* t4, t5, t6 */
static const Reg rv_int_asm_temps[] = {RV_TMP2, RV_TMP3};
+static const Reg rv_direct_asm_int[] = {10u, 11u, 12u, 13u, 14u, 15u,
+ 16u, 17u, 29u, 30u, 31u};
static const NativePhysRegInfo rv_int_phys[] = {
RV_PHYS_INT_RESERVED(0u), /* zero */
@@ -478,10 +484,10 @@ static const NativePhysRegInfo rv_int_phys[] = {
RV_PHYS_INT_RESERVED(2u), /* sp */
RV_PHYS_INT_RESERVED(3u), /* gp */
RV_PHYS_INT_RESERVED(4u), /* tp */
- RV_PHYS_INT_RESERVED(5u), /* t0 = TMP0 */
- RV_PHYS_INT_RESERVED(6u), /* t1 = TMP1 */
- RV_PHYS_INT_RESERVED(7u), /* t2 = TMP2 (backend/asm) */
- RV_PHYS_INT_RESERVED(8u), /* s0/fp */
+ RV_PHYS_INT_RESERVED_CALLER(5u), /* t0 = TMP0 */
+ RV_PHYS_INT_RESERVED_CALLER(6u), /* t1 = TMP1 */
+ RV_PHYS_INT_RESERVED_CALLER(7u), /* t2 = TMP2 (backend/asm) */
+ RV_PHYS_INT_RESERVED_ROLE(8u, NATIVE_REG_CALLEE_SAVED), /* s0/fp */
RV_PHYS_INT_CALLEE(9u), /* s1 */
RV_PHYS_INT_ARG(10u, 0u), RV_PHYS_INT_ARG(11u, 1u),
RV_PHYS_INT_ARG(12u, 2u), RV_PHYS_INT_ARG(13u, 3u),
@@ -492,10 +498,10 @@ static const NativePhysRegInfo rv_int_phys[] = {
RV_PHYS_INT_CALLEE(22u), RV_PHYS_INT_CALLEE(23u),
RV_PHYS_INT_CALLEE(24u), RV_PHYS_INT_CALLEE(25u),
RV_PHYS_INT_CALLEE(26u), RV_PHYS_INT_CALLEE(27u),
- RV_PHYS_INT_RESERVED(28u), /* t3 = TMP3 (backend/asm) */
- RV_PHYS_INT_RESERVED(29u), /* t4 = O1 operand temp / O0 scratch */
- RV_PHYS_INT_RESERVED(30u), /* t5 = O1 operand temp / O0 scratch */
- RV_PHYS_INT_RESERVED(31u), /* t6 = O1 operand temp / O0 scratch */
+ RV_PHYS_INT_RESERVED_CALLER(28u), /* t3 = TMP3 (backend/asm) */
+ RV_PHYS_INT_RESERVED_CALLER(29u), /* t4 = O1 operand temp / O0 scratch */
+ RV_PHYS_INT_RESERVED_CALLER(30u), /* t5 = O1 operand temp / O0 scratch */
+ RV_PHYS_INT_RESERVED_CALLER(31u), /* t6 = O1 operand temp / O0 scratch */
};
#define RV_PHYS_FP_ARG(r, idx) \
@@ -520,12 +526,13 @@ static const NativePhysRegInfo rv_int_phys[] = {
.flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
.spill_cost = 4u, \
.copy_cost = 1u}
-#define RV_PHYS_FP_RESERVED(r) \
- {.reg = (r), \
- .cls = NATIVE_REG_FP, \
- .abi_index = 0xffu, \
- .flags = NATIVE_REG_RESERVED, \
- .spill_cost = 0u, \
+#define RV_PHYS_FP_RESERVED(r) \
+ {.reg = (r), \
+ .cls = NATIVE_REG_FP, \
+ .abi_index = 0xffu, \
+ .flags = NATIVE_REG_RESERVED | \
+ NATIVE_REG_CALLER_SAVED, \
+ .spill_cost = 0u, \
.copy_cost = 0u}
/* NDT (-O0) fp value-cache / scratch pool (read only by NativeDirectTarget; the
@@ -544,6 +551,9 @@ static const Reg rv_ndt_fp_allocable[] = {
28u, 29u, 30u, 31u}; /* fa0-7,ft4-7,ft8-11 */
static const Reg rv_fp_scratch[] = {2u, 3u}; /* ft2, ft3 */
static const Reg rv_fp_asm_temps[] = {RV_FTMP0, RV_FTMP1};
+static const Reg rv_direct_asm_fp[] = {10u, 11u, 12u, 13u, 14u, 15u,
+ 16u, 17u, 4u, 5u, 6u, 7u,
+ 28u, 29u, 30u, 31u};
static const NativePhysRegInfo rv_fp_phys[] = {
RV_PHYS_FP_RESERVED(0u), /* ft0 = FTMP0 */
@@ -573,18 +583,12 @@ static const NativeAllocClassInfo rv_classes[] = {
.nemit_temps = sizeof rv_int_scratch / sizeof rv_int_scratch[0],
.asm_temps = rv_int_asm_temps,
.nasm_temps = sizeof rv_int_asm_temps / sizeof rv_int_asm_temps[0],
+ .direct_asm_allocable = rv_direct_asm_int,
+ .ndirect_asm_allocable =
+ sizeof rv_direct_asm_int / sizeof rv_direct_asm_int[0],
.emit_cache_mask = (1u << 29u) | (1u << 30u) | (1u << 31u),
.phys = rv_int_phys,
- .nphys = sizeof rv_int_phys / sizeof rv_int_phys[0],
- /* t0-t6 (5-7,28-31) + a0-a7 (10-17) */
- .caller_saved_mask = 0xf00000e0u | 0x0003fc00u,
- /* s0-s11 (8,9,18-27) */
- .callee_saved_mask = 0x0ffc0300u,
- .arg_mask = 0x0003fc00u,
- .ret_mask = 0x00000c00u,
- /* zero,ra,sp,gp,tp,t0,t1,t2,s0 (bits 0-8) + t3 (bit 28). The validated
- * emit_temps bank adds t4/t5/t6 to the optimizer's reserved mask. */
- .reserved_mask = 0x000001ffu | (1u << 28)},
+ .nphys = sizeof rv_int_phys / sizeof rv_int_phys[0]},
{.cls = NATIVE_REG_FP,
.ndt_allocable = rv_ndt_fp_allocable,
.ndt_allocable_count =
@@ -595,16 +599,12 @@ static const NativeAllocClassInfo rv_classes[] = {
.nemit_temps = sizeof rv_fp_scratch / sizeof rv_fp_scratch[0],
.asm_temps = rv_fp_asm_temps,
.nasm_temps = sizeof rv_fp_asm_temps / sizeof rv_fp_asm_temps[0],
+ .direct_asm_allocable = rv_direct_asm_fp,
+ .ndirect_asm_allocable =
+ sizeof rv_direct_asm_fp / sizeof rv_direct_asm_fp[0],
.emit_cache_mask = (1u << 2u) | (1u << 3u),
.phys = rv_fp_phys,
- .nphys = sizeof rv_fp_phys / sizeof rv_fp_phys[0],
- /* ft0-ft7 (0-7), fa0-fa7 (10-17), ft8-ft11 (28-31) */
- .caller_saved_mask = 0xf00000ffu | 0x0003fc00u,
- /* fs0-fs11 (8,9,18-27) */
- .callee_saved_mask = 0x0ffc0300u,
- .arg_mask = 0x0003fc00u,
- .ret_mask = 0x00000c00u,
- .reserved_mask = 0x0000000fu /* ft0-ft3 */},
+ .nphys = sizeof rv_fp_phys / sizeof rv_fp_phys[0]},
};
/* Resolve a register name ("a7", "fa0", ...) to its (class, Reg). Powers the
@@ -1115,7 +1115,7 @@ static void rv_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
/* copy_bytes: resolve dst and src to dedicated pointer regs (RV_TMP3 / RV_TMP0)
* once, then copy granule-by-granule advancing both pointers. dst is resolved
* first because its base may itself live in RV_TMP1 (the transfer reg, e.g. the
- * sret pointer from plan_ret); capturing it into RV_TMP3 before src resolution
+ * sret pointer from marshal_ret); capturing it into RV_TMP3 before src resolution
* (which may clobber RV_TMP1 for far offsets) keeps it live. Advancing the
* pointers keeps every load/store at offset 0, so no offset ever exceeds imm12
* and the transfer reg never aliases a base. */
@@ -2138,32 +2138,21 @@ static void rv_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
* callee-save). This is the same register selection the per-block spill used,
* hoisted into the prologue. Writes up to `cap` per-class masks into `out` and
* returns the class count to reserve. */
+static int rv_asm_reg_is_callee_saved(NativeTarget* t, NativeAllocClass cls,
+ Reg r) {
+ (void)t;
+ return (cls == NATIVE_REG_INT && rv_reg_is_callee_int(r)) ||
+ (cls == NATIVE_REG_FP && rv_reg_is_callee_fp(r));
+}
+
static u32 rv_known_callee_saves(NativeTarget* t,
const NativeKnownFrameDesc* frame, u32* out,
u32 cap) {
- u32 ncls = frame->ncallee_classes;
- u32 clob_int = 0, clob_fp = 0, abi_int, abi_fp;
- if (ncls > cap) ncls = cap;
- for (u32 c = 0; c < ncls; ++c)
- out[c] = frame->callee_saved_used ? frame->callee_saved_used[c] : 0u;
- if (frame->asm_clobbers && frame->nasm_clobbers) {
- RvNativeTarget* a = rv_of(t);
- SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
- rv_asm_clobber_masks(t->c, loc, frame->asm_clobbers, frame->nasm_clobbers,
- &clob_int, &clob_fp);
- }
- native_asm_abi_clobber_masks(t, frame->asm_clobber_abi_sets, &abi_int,
- &abi_fp);
- clob_int |= abi_int;
- clob_fp |= abi_fp;
- for (Reg r = 0; r < 32u; ++r) {
- if (NATIVE_REG_INT < ncls && (clob_int & (1u << r)) &&
- rv_reg_is_callee_int(r))
- out[NATIVE_REG_INT] |= 1u << r;
- if (NATIVE_REG_FP < ncls && (clob_fp & (1u << r)) && rv_reg_is_callee_fp(r))
- out[NATIVE_REG_FP] |= 1u << r;
- }
- return ncls;
+ RvNativeTarget* a = rv_of(t);
+ SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
+ return native_asm_known_callee_saves(
+ t, loc, frame, out, cap, rv_asm_clobber_masks,
+ rv_asm_reg_is_callee_saved);
}
static u32 rv_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
@@ -2187,8 +2176,8 @@ static void rv_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
rv_func_begin_common(t, fd);
a->frame.known_frame = 1;
if (frame) {
- u32 cs[NATIVE_CALL_PLAN_CLASSES];
- u32 ncs = rv_known_callee_saves(t, frame, cs, NATIVE_CALL_PLAN_CLASSES);
+ u32 cs[NATIVE_REG_CLASS_COUNT];
+ u32 ncs = rv_known_callee_saves(t, frame, cs, NATIVE_REG_CLASS_COUNT);
a->frame.has_alloca = frame->has_alloca;
if (ncs) rv_reserve_callee_saves(t, cs, ncs);
for (i = 0; i < frame->nslots; ++i) {
@@ -2271,7 +2260,7 @@ static const ABIArgInfo* rv_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
parts[p].align = gpr;
parts[p].src_offset = off;
}
- scratch->nparts = (u16)nparts;
+ scratch->nparts = nparts;
scratch->parts = parts;
return scratch;
}
@@ -2624,14 +2613,15 @@ static void rv_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves,
s.reg_move = rv_move;
s.scratch[NATIVE_REG_INT] = RV_TMP1;
s.scratch[NATIVE_REG_FP] = RV_FTMP1;
+ s.scratch_class_mask = (1u << NATIVE_REG_INT) | (1u << NATIVE_REG_FP);
native_arg_shuffle(&s, moves, n);
}
-static void rv_plan_call(NativeTarget* t, const NativeCallDesc* desc,
- NativeCallPlan* plan) {
+static void rv_marshal_call(NativeTarget* t, const NativeCallDesc* desc,
+ NativeCallPhase* plan) {
RvNativeTarget* a = rv_of(t);
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
- NativeCallPlanRet* rets;
+ NativeCallPhaseRet* rets;
KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
/* Right-size the result scratch to the exact number of entries the ret loops
* below write: nparts on a DIRECT register return, 1 on the !abi fallback,
@@ -2640,7 +2630,7 @@ static void rv_plan_call(NativeTarget* t, const NativeCallDesc* desc,
? abi->ret.nparts
: ((!abi && desc->nresults) ? 1u : 0u);
memset(plan, 0, sizeof *plan);
- rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPlanRet, nrets_cap) : NULL;
+ rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPhaseRet, nrets_cap) : NULL;
plan->callee = desc->callee;
plan->rets = rets;
plan->flags = desc->flags;
@@ -2800,7 +2790,7 @@ static void rv_emit_tail_site(NativeTarget* t, NativeLoc callee) {
}
}
-static void rv_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
+static void rv_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
MCEmitter* mc = t->mc;
ObjSecId sec = mc->section_id;
if (plan->flags & CG_CALL_TAIL) {
@@ -2822,14 +2812,14 @@ static void rv_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
rv_panic(rv_of(t), "unsupported call target");
}
-static void rv_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
- const NativeLoc* value, NativeCallPlanRet** out_rets,
- u32* out_nrets) {
+static void rv_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
+ const NativeLoc* value,
+ NativeCallPhaseRet** out_rets, u32* out_nrets) {
RvNativeTarget* a = rv_of(t);
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
- NativeCallPlanRet* rets = NULL;
+ NativeCallPhaseRet* rets = NULL;
u32 nr = 0;
- if (value) rets = arena_zarray(t->c->tu, NativeCallPlanRet, 4);
+ if (value) rets = arena_zarray(t->c->tu, NativeCallPhaseRet, 4);
if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
NativeLoc dstp = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
@@ -3746,17 +3736,30 @@ static void rv_intrinsic(NativeTarget* t, IntrinKind kind,
* ABI registers even though the semantic IR operands are ordinary virtual
* values. Keep live-across allocations out of those fixed destinations. */
static int rv_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
- u32 mask[NATIVE_CALL_PLAN_CLASSES]) {
- (void)t;
+ u32 mask[NATIVE_REG_CLASS_COUNT]) {
mask[0] = mask[1] = mask[2] = 0;
- if ((NativeMachineOpKind)op->kind != NATIVE_MOP_INTRINSIC ||
- (IntrinKind)op->intrin != INTRIN_SYSCALL)
- return 0;
- /* a7 = syscall number; a0..a5 = up to six arguments / a0 result. */
- mask[NATIVE_REG_INT] = (1u << RV_A0) | (1u << RV_A1) | (1u << RV_A2) |
- (1u << RV_A3) | (1u << RV_A4) | (1u << RV_A5) |
- (1u << RV_A7);
- return 1;
+ switch ((NativeMachineOpKind)op->kind) {
+ case NATIVE_MOP_INTRINSIC:
+ if ((IntrinKind)op->intrin != INTRIN_SYSCALL) return 0;
+ /* a7 = syscall number; a0..a5 = arguments / a0 result. */
+ mask[NATIVE_REG_INT] = (1u << RV_A0) | (1u << RV_A1) | (1u << RV_A2) |
+ (1u << RV_A3) | (1u << RV_A4) | (1u << RV_A5) |
+ (1u << RV_A7);
+ return 1;
+ case NATIVE_MOP_BINOP:
+ case NATIVE_MOP_BITFIELD_LOAD:
+ case NATIVE_MOP_BITFIELD_STORE:
+ case NATIVE_MOP_VA_START:
+ case NATIVE_MOP_VA_ARG:
+ case NATIVE_MOP_ATOMIC_CAS:
+ case NATIVE_MOP_ATOMIC_RMW:
+ case NATIVE_MOP_TLS_ADDR:
+ return 0;
+ case NATIVE_MOP_COUNT:
+ break;
+ }
+ rv_panic(rv_of(t), "invalid machine-effect operation");
+ return 0;
}
/* ============================ inline asm ============================ */
@@ -3839,32 +3842,6 @@ static void rv_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
/* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
-/* Pick a free register from the arch's caller-saved allocable pools for an
- * asm operand the direct path must self-allocate. */
-static Reg rv_asm_alloc_reg(NativeDirectTarget* d, NativeAllocClass cls,
- u32 allowed_mask, u32* used_int, u32* used_fp) {
- /* int: a0..a7 (10..17) then O0-owned t4..t6. */
- static const Reg int_pool[] = {10u, 11u, 12u, 13u, 14u, 15u,
- 16u, 17u, 29u, 30u, 31u};
- /* fp: fa0..fa7 (10..17) then ft caller-saved. */
- static const Reg fp_pool[] = {10u, 11u, 12u, 13u, 14u, 15u, 16u, 17u,
- 4u, 5u, 6u, 7u, 28u, 29u, 30u, 31u};
- const Reg* pool = cls == NATIVE_REG_FP ? fp_pool : int_pool;
- u32 n = cls == NATIVE_REG_FP ? (u32)(sizeof fp_pool / sizeof fp_pool[0])
- : (u32)(sizeof int_pool / sizeof int_pool[0]);
- u32* used = cls == NATIVE_REG_FP ? used_fp : used_int;
- u32 i;
- for (i = 0; i < n; ++i) {
- Reg r = pool[i];
- if (allowed_mask && (allowed_mask & (1u << r)) == 0) continue;
- if ((*used & (1u << r)) != 0) continue;
- *used |= 1u << r;
- return r;
- }
- rv_asm_panic(d, "out of registers for asm operands");
- return REG_NONE;
-}
-
/* Direct (-O0) path: resolve a semantic Operand to a NativeAddr. */
static NativeAddr rv_direct_addr(NativeDirectTarget* d, Operand op) {
NativeAddr addr;
@@ -4197,9 +4174,9 @@ NativeTarget* rv64_native_target_new(Compiler* c, ObjBuilder* obj,
t->alloca_ = rv_alloca;
t->spill = rv_spill;
t->reload = rv_reload;
- t->plan_call = rv_plan_call;
+ t->marshal_call = rv_marshal_call;
t->emit_call = rv_emit_call;
- t->plan_ret = rv_plan_ret;
+ t->marshal_ret = rv_marshal_ret;
t->ret = rv_ret;
t->atomic_load = rv_atomic_load;
t->atomic_store = rv_atomic_store;
@@ -4376,20 +4353,11 @@ static void rv_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
const Sym* clobbers, u32 nclob,
u32 clobber_abi_sets) {
static const NativeAsmDirectHooks hooks = {
- /* Reserve backend temps t0..t3, SP/GP/TP/zero/RA, and the frame pointer so
- * the direct asm operand allocator never hands them out. */
- .reserved_int_mask =
- (1u << RV_ZERO) | (1u << RV_RA) | (1u << RV_SP) | (1u << RV_GP) |
- (1u << RV_TP) | (1u << RV_TMP0) | (1u << RV_TMP1) |
- (1u << RV_TMP2) | (1u << RV_TMP3) | (1u << RV_S0),
- .reserved_fp_mask =
- (1u << RV_FTMP0) | (1u << RV_FTMP1) | (1u << 2u) | (1u << 3u),
.opk_reg = RV64_INLINE_OPK_REG,
.opcls_fp = RV64_INLINE_OPCLS_FP,
.panic = rv_asm_panic,
.bound_reg = rv_asm_bound_reg,
.bound_mem = rv_asm_bound_mem,
- .alloc_reg = rv_asm_alloc_reg,
.clobber_masks = rv_asm_clobber_masks,
.save_callee_clobbers = rv_asm_hook_save_callee_clobbers,
.restore_one = rv_asm_hook_restore_one,
diff --git a/src/arch/x64/emit.h b/src/arch/x64/emit.h
@@ -78,7 +78,7 @@ static inline void x64_abi_direct_reg_need(const ABIArgInfo* ai, u32* need_int,
*need_int = 0;
*need_fp = 0;
if (!ai || ai->kind != ABI_ARG_DIRECT) return;
- for (u16 i = 0; i < ai->nparts; ++i) {
+ for (u32 i = 0; i < ai->nparts; ++i) {
const ABIArgPart* p = &ai->parts[i];
if (p->cls == ABI_CLASS_FP)
++*need_fp;
diff --git a/src/arch/x64/native.c b/src/arch/x64/native.c
@@ -94,7 +94,6 @@ typedef struct X64NativeTarget {
u32 next_param_stack;
u8 has_sret;
u8 is_variadic;
- u8 call_callee_parked;
NativeFrameSlot sret_ptr_slot;
NativeFrameSlot reg_save_slot; /* SysV variadic 176B __va_list_tag area */
@@ -131,11 +130,6 @@ typedef struct X64NativeTarget {
NativeArgMove bind_moves[X64_MAX_BIND_MOVES];
u32 nbind_moves;
- /* Integer data scratch used while packing/unpacking a non-power-of-two ABI
- * part. Normally r10, keeping r11 exclusively available to resolve complex
- * addresses. */
- Reg part_scratch;
-
const X64ABIRegs* abi;
} X64NativeTarget;
@@ -226,13 +220,19 @@ static void emit_jcc_rel32(MCEmitter* mc, u32 cc, MCLabel l);
.flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
.spill_cost = 4u, \
.copy_cost = 1u}
-#define X64_PHYS_INT_RESERVED(r) \
- {.reg = (r), \
- .cls = NATIVE_REG_INT, \
- .abi_index = 0xffu, \
- .flags = NATIVE_REG_RESERVED, \
- .spill_cost = 0u, \
+#define X64_PHYS_INT_RESERVED_ROLE(r, role) \
+ {.reg = (r), \
+ .cls = NATIVE_REG_INT, \
+ .abi_index = 0xffu, \
+ .flags = NATIVE_REG_RESERVED | (role), \
+ .spill_cost = 0u, \
.copy_cost = 0u}
+#define X64_PHYS_INT_RESERVED(r) \
+ X64_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_NONE)
+#define X64_PHYS_INT_RESERVED_CALLER(r) \
+ X64_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_CALLER_SAVED)
+#define X64_PHYS_INT_RESERVED_CALLEE(r) \
+ X64_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_CALLEE_SAVED)
/* The NDT (-O0) value-cache / scratch pool (read only by NativeDirectTarget;
* the optimizer allocates over the phys[] ALLOCABLE flags instead, which also
@@ -247,21 +247,23 @@ static const Reg x64_ndt_int_allocable[] = {X64_RSI, X64_RDI, X64_R13,
X64_R14, X64_R15};
static const Reg x64_int_scratch[] = {X64_R8, X64_R9};
static const Reg x64_int_asm_temps[] = {X64_TMP_INT, X64_TMP_INT2};
+static const Reg x64_direct_asm_int[] = {X64_RDI, X64_RSI, X64_RDX, X64_RCX};
static const NativePhysRegInfo x64_int_phys[] = {
- X64_PHYS_INT_RESERVED(X64_RAX), /* return / div-mul (asm-pinnable) */
+ X64_PHYS_INT_RESERVED_ROLE(
+ X64_RAX, NATIVE_REG_CALLER_SAVED | NATIVE_REG_RET),
X64_PHYS_INT_ARG(X64_RCX),
X64_PHYS_INT_RET_ARG(X64_RDX),
- X64_PHYS_INT_RESERVED(X64_RBX),
+ X64_PHYS_INT_RESERVED_CALLEE(X64_RBX),
X64_PHYS_INT_RESERVED(X64_RSP), /* stack pointer */
X64_PHYS_INT_RESERVED(X64_RBP), /* frame pointer */
X64_PHYS_INT_ARG(X64_RSI),
X64_PHYS_INT_ARG(X64_RDI),
X64_PHYS_INT_ARG_RESERVED(X64_R8), /* O1 operand temp / O0 scratch */
X64_PHYS_INT_ARG_RESERVED(X64_R9), /* O1 operand temp / O0 scratch */
- X64_PHYS_INT_RESERVED(X64_R10), /* backend-private temp */
- X64_PHYS_INT_RESERVED(X64_R11), /* backend-private temp */
- X64_PHYS_INT_RESERVED(X64_R12),
+ X64_PHYS_INT_RESERVED_CALLER(X64_R10), /* backend-private temp */
+ X64_PHYS_INT_RESERVED_CALLER(X64_R11), /* backend-private temp */
+ X64_PHYS_INT_RESERVED_CALLEE(X64_R12),
X64_PHYS_INT_CALLEE(X64_R13),
X64_PHYS_INT_CALLEE(X64_R14),
X64_PHYS_INT_CALLEE(X64_R15),
@@ -296,12 +298,13 @@ static const NativePhysRegInfo x64_int_phys[] = {
.flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
.spill_cost = 1u, \
.copy_cost = 1u}
-#define X64_PHYS_FP_RESERVED(r) \
- {.reg = (r), \
- .cls = NATIVE_REG_FP, \
- .abi_index = 0xffu, \
- .flags = NATIVE_REG_RESERVED, \
- .spill_cost = 0u, \
+#define X64_PHYS_FP_RESERVED(r) \
+ {.reg = (r), \
+ .cls = NATIVE_REG_FP, \
+ .abi_index = 0xffu, \
+ .flags = NATIVE_REG_RESERVED | \
+ NATIVE_REG_CALLER_SAVED, \
+ .spill_cost = 0u, \
.copy_cost = 0u}
/* Allocable FP pool: arg/ret xmm0..xmm3 lead so -O0 fp-arg producers
@@ -316,12 +319,16 @@ static const Reg x64_ndt_fp_allocable[] = {
X64_XMM0 + 10, X64_XMM0 + 11};
static const Reg x64_fp_scratch[] = {X64_XMM4, X64_XMM5};
static const Reg x64_fp_asm_temps[] = {X64_TMP_FP, X64_TMP_FP2};
+static const Reg x64_direct_asm_fp[] = {
+ X64_XMM0, X64_XMM1, X64_XMM2, X64_XMM3,
+ X64_XMM6, X64_XMM7, X64_XMM8, X64_XMM0 + 9,
+ X64_XMM0 + 10, X64_XMM0 + 11};
static const NativePhysRegInfo x64_fp_phys[] = {
X64_PHYS_FP_ARG_RET(X64_XMM0), X64_PHYS_FP_ARG_RET(X64_XMM1),
X64_PHYS_FP_ARG(X64_XMM2), X64_PHYS_FP_ARG(X64_XMM3),
X64_PHYS_FP_ARG_RESERVED(X64_XMM4), X64_PHYS_FP_ARG_RESERVED(X64_XMM5),
- X64_PHYS_FP_CALLER(X64_XMM6), X64_PHYS_FP_CALLER(X64_XMM7),
+ X64_PHYS_FP_ARG(X64_XMM6), X64_PHYS_FP_ARG(X64_XMM7),
X64_PHYS_FP_CALLER(X64_XMM8), X64_PHYS_FP_CALLER(X64_XMM0 + 9),
X64_PHYS_FP_CALLER(X64_XMM0 + 10), X64_PHYS_FP_CALLER(X64_XMM0 + 11),
X64_PHYS_FP_RESERVED(X64_XMM0 + 12), X64_PHYS_FP_RESERVED(X64_XMM0 + 13),
@@ -340,24 +347,12 @@ static const NativeAllocClassInfo x64_classes[] = {
.nemit_temps = sizeof x64_int_scratch / sizeof x64_int_scratch[0],
.asm_temps = x64_int_asm_temps,
.nasm_temps = sizeof x64_int_asm_temps / sizeof x64_int_asm_temps[0],
+ .direct_asm_allocable = x64_direct_asm_int,
+ .ndirect_asm_allocable =
+ sizeof x64_direct_asm_int / sizeof x64_direct_asm_int[0],
.emit_cache_mask = (1u << X64_R8) | (1u << X64_R9),
.phys = x64_int_phys,
- .nphys = sizeof x64_int_phys / sizeof x64_int_phys[0],
- /* caller-saved: rax,rcx,rdx,rsi,rdi,r8,r9,r10,r11 (SysV) */
- .caller_saved_mask = (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX) |
- (1u << X64_RSI) | (1u << X64_RDI) | (1u << X64_R8) |
- (1u << X64_R9) | (1u << X64_R10) | (1u << X64_R11),
- /* callee-saved: rbx,r12,r13,r14,r15 (rbp handled by prologue head) */
- .callee_saved_mask = (1u << X64_RBX) | (1u << X64_R12) | (1u << X64_R13) |
- (1u << X64_R14) | (1u << X64_R15),
- /* SysV arg regs rdi,rsi,rdx,rcx,r8,r9 */
- .arg_mask = (1u << X64_RDI) | (1u << X64_RSI) | (1u << X64_RDX) |
- (1u << X64_RCX) | (1u << X64_R8) | (1u << X64_R9),
- .ret_mask = (1u << X64_RAX) | (1u << X64_RDX),
- /* rax/rsp/rbp reserved; r8/r9 operand temps; r10/r11 backend-private */
- .reserved_mask = (1u << X64_RAX) | (1u << X64_RSP) | (1u << X64_RBP) |
- (1u << X64_R8) | (1u << X64_R9) | (1u << X64_R10) |
- (1u << X64_R11) | (1u << X64_RBX) | (1u << X64_R12)},
+ .nphys = sizeof x64_int_phys / sizeof x64_int_phys[0]},
{.cls = NATIVE_REG_FP,
.ndt_allocable = x64_ndt_fp_allocable,
.ndt_allocable_count =
@@ -368,18 +363,12 @@ static const NativeAllocClassInfo x64_classes[] = {
.nemit_temps = sizeof x64_fp_scratch / sizeof x64_fp_scratch[0],
.asm_temps = x64_fp_asm_temps,
.nasm_temps = sizeof x64_fp_asm_temps / sizeof x64_fp_asm_temps[0],
+ .direct_asm_allocable = x64_direct_asm_fp,
+ .ndirect_asm_allocable =
+ sizeof x64_direct_asm_fp / sizeof x64_direct_asm_fp[0],
.emit_cache_mask = (1u << X64_XMM4) | (1u << X64_XMM5),
.phys = x64_fp_phys,
- .nphys = sizeof x64_fp_phys / sizeof x64_fp_phys[0],
- /* All xmm caller-saved on SysV. */
- .caller_saved_mask = 0xffffu,
- .callee_saved_mask = 0u,
- .arg_mask = 0xffu, /* xmm0..xmm7 */
- .ret_mask = (1u << X64_XMM0) | (1u << X64_XMM1),
- /* xmm4/xmm5 operand temps; xmm14/xmm15 backend-private. */
- .reserved_mask = (1u << X64_XMM4) | (1u << X64_XMM5) |
- (1u << (X64_XMM0 + 12)) | (1u << (X64_XMM0 + 13)) |
- (1u << (X64_XMM0 + 14)) | (1u << X64_XMM15)},
+ .nphys = sizeof x64_fp_phys / sizeof x64_fp_phys[0]},
};
/* Resolve a register name ("r10", "xmm3", ...) to its (class, Reg). Powers the
@@ -769,47 +758,35 @@ static int x64_native_part_width(u32 size) {
return size == 1u || size == 2u || size == 4u || size == 8u;
}
-static int x64_addr_explicitly_uses_reg(const NativeAddr* addr, Reg reg) {
- return (addr->base_kind == NATIVE_ADDR_BASE_REG &&
- (addr->base.reg & 0xfu) == reg) ||
- (addr->index_kind == NATIVE_ADDR_INDEX_REG &&
- (addr->index.reg & 0xfu) == reg);
+/* The two integer registers in the backend-private bank. Exact-width helpers
+ * assign them explicit address/data roles for their complete local phase. */
+static int x64_private_int_reg(Reg reg) {
+ return reg == X64_TMP_INT || reg == X64_TMP_INT2;
}
-/* A multi-chunk load cannot overwrite an explicit base/index with its first
- * chunk and then reuse that destroyed register for the tail. Pin the complete
- * effective address in r11 first. Generic NativeTarget destinations never use
- * the backend-private r10/r11 bank; ABI-owned r10 loads use rbp/rsp storage and
- * therefore do not enter this alias path. */
-static NativeAddr x64_stabilize_part_load_addr(X64NativeTarget* a,
- NativeAddr addr, Reg dst) {
+static Reg x64_other_private_int(Reg reg) {
+ return reg == X64_TMP_INT ? X64_TMP_INT2 : X64_TMP_INT;
+}
+
+/* Resolve the complete effective address into one explicitly selected private
+ * register. Exact-width multi-chunk operations keep this address stable while
+ * their other private register carries chunk data; callers no longer depend on
+ * x64_emit_mem's internal address-scratch choice. */
+static NativeAddr x64_stabilize_part_addr(X64NativeTarget* a, NativeAddr addr,
+ Reg addr_reg) {
u32 base;
KitCgTypeId base_type;
- if (!x64_addr_explicitly_uses_reg(&addr, dst)) return addr;
- if (dst == X64_TMP_INT || dst == X64_TMP_INT2)
- x64_panic(a, "exact-width load aliases private address scratch");
-
base_type = addr.base_type;
- base = x64_addr_to_base_reg(a, addr, X64_TMP_INT2);
- if (base != X64_TMP_INT2)
- emit_mov_rr(a->base.mc, 1, X64_TMP_INT2, base);
+ base = x64_addr_to_base_reg(a, addr, addr_reg);
+ if (base != addr_reg) emit_mov_rr(a->base.mc, 1, addr_reg, base);
memset(&addr, 0, sizeof addr);
addr.base_kind = NATIVE_ADDR_BASE_REG;
addr.cls = NATIVE_REG_INT;
- addr.base.reg = X64_TMP_INT2;
+ addr.base.reg = addr_reg;
addr.base_type = base_type;
return addr;
}
-static Reg x64_part_scratch(const X64NativeTarget* a, Reg avoid) {
- Reg scratch = a->part_scratch;
- if (scratch != X64_TMP_INT && scratch != X64_TMP_INT2)
- scratch = X64_TMP_INT2;
- if (scratch == avoid)
- scratch = scratch == X64_TMP_INT ? X64_TMP_INT2 : X64_TMP_INT;
- return scratch;
-}
-
/* Load an exact-width little-endian integer ABI part. x86 has native memory
* operations only for 1/2/4/8 bytes, so a 3/5/6/7-byte tail is assembled from
* bounded chunks in the low bits of dst. No chunk crosses the semantic part. */
@@ -818,9 +795,11 @@ static void x64_load_int_part(X64NativeTarget* a, NativeLoc dst,
NativePartChunkIter chunks;
NativeTarget* t = &a->base;
Reg rd = (Reg)loc_reg(dst);
- Reg scratch;
+ Reg addr_reg;
+ Reg data_reg;
NativeLoc tmp;
u32 chunk_off, chunk_size;
+ int private_dst;
int first = 1;
if (x64_native_part_width(size)) {
@@ -831,21 +810,30 @@ static void x64_load_int_part(X64NativeTarget* a, NativeLoc dst,
if (size == 0u || size > 8u || native_loc_is_fp(dst))
x64_panic(a, "invalid exact-width integer part load");
- addr = x64_stabilize_part_load_addr(a, addr, rd);
- scratch = x64_part_scratch(a, rd);
+ private_dst = x64_private_int_reg(rd);
+ addr_reg = private_dst ? x64_other_private_int(rd) : X64_TMP_INT2;
+ data_reg = private_dst ? addr_reg : X64_TMP_INT;
+ addr = x64_stabilize_part_addr(a, addr, addr_reg);
+ /* A private destination consumes both integer temps (one accumulator, one
+ * chunk destination). Keep the stable address in the private FP hold register
+ * and restore it immediately before each alias-safe memory load. */
+ if (private_dst)
+ emit_sse_rr_w(t->mc, 0x66, 0x6E, 1, X64_TMP_FP, addr_reg);
tmp = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- scratch);
+ data_reg);
chunks = native_part_chunks(size, 8u);
while (native_part_chunk_next(&chunks, &chunk_off, &chunk_size)) {
NativeAddr chunk_addr = addr;
NativeLoc chunk_dst = first ? dst : tmp;
chunk_addr.offset += (i32)chunk_off;
mem.size = chunk_size;
+ if (private_dst)
+ emit_sse_rr_w(t->mc, 0x66, 0x7E, 1, X64_TMP_FP, addr_reg);
x64_emit_mem(a, 1, chunk_dst, chunk_addr, mem);
if (!first) {
- emit_shift_imm(t->mc, 1, X64_SHIFT_SUB_SHL, scratch,
+ emit_shift_imm(t->mc, 1, X64_SHIFT_SUB_SHL, data_reg,
(u8)(chunk_off * 8u));
- emit_alu_rr(t->mc, 1, X64_OPC_ALU_OR, rd, scratch);
+ emit_alu_rr(t->mc, 1, X64_OPC_ALU_OR, rd, data_reg);
}
first = 0;
}
@@ -859,9 +847,11 @@ static void x64_store_int_part(X64NativeTarget* a, NativeAddr addr,
NativePartChunkIter chunks;
NativeTarget* t = &a->base;
Reg rs = (Reg)loc_reg(src);
- Reg scratch;
+ Reg addr_reg;
+ Reg data_reg;
NativeLoc tmp;
u32 chunk_off, chunk_size, shifted = 0;
+ int private_src;
int copied = 0;
if (x64_native_part_width(size)) {
@@ -872,20 +862,20 @@ static void x64_store_int_part(X64NativeTarget* a, NativeAddr addr,
if (size == 0u || size > 8u || native_loc_is_fp(src))
x64_panic(a, "invalid exact-width integer part store");
- scratch = x64_part_scratch(a, rs);
- /* x64_resolve_addr owns r11 for FRAME_VALUE/GOT materialization. The only
- * ABI paths whose packed source itself occupies r10 store to plain rbp/rsp
- * addresses; reject a future complex-address use instead of replacing the
- * shifted data with an address component. Generic emitter operands never
- * occupy either backend-private register and therefore use r10 above. */
- if (scratch == X64_TMP_INT2 &&
- (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE ||
- addr.index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE ||
- (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
- x64_use_got_for_sym(t, addr.base.global.sym))))
- x64_panic(a, "exact-width store aliases address scratch");
+ private_src = x64_private_int_reg(rs);
+ addr_reg = private_src ? x64_other_private_int(rs) : X64_TMP_INT2;
+ data_reg = private_src ? rs : X64_TMP_INT;
+ /* Preserve a backend-private source while complete address stabilization is
+ * free to use both integer temps. Restore it after the stores as well: these
+ * helpers promise not to consume their source even when another backend
+ * marshalling phase supplied it from the private bank. */
+ if (private_src)
+ emit_sse_rr_w(t->mc, 0x66, 0x6E, 1, X64_TMP_FP, rs);
+ addr = x64_stabilize_part_addr(a, addr, addr_reg);
+ if (private_src)
+ emit_sse_rr_w(t->mc, 0x66, 0x7E, 1, X64_TMP_FP, data_reg);
tmp = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- scratch);
+ data_reg);
chunks = native_part_chunks(size, 8u);
while (native_part_chunk_next(&chunks, &chunk_off, &chunk_size)) {
NativeAddr chunk_addr = addr;
@@ -893,10 +883,10 @@ static void x64_store_int_part(X64NativeTarget* a, NativeAddr addr,
chunk_addr.offset += (i32)chunk_off;
if (chunk_off != 0u) {
if (!copied) {
- emit_mov_rr(t->mc, 1, scratch, rs);
+ emit_mov_rr(t->mc, 1, data_reg, rs);
copied = 1;
}
- emit_shift_imm(t->mc, 1, X64_SHIFT_SUB_SHR, scratch,
+ emit_shift_imm(t->mc, 1, X64_SHIFT_SUB_SHR, data_reg,
(u8)((chunk_off - shifted) * 8u));
shifted = chunk_off;
chunk_src = tmp;
@@ -904,6 +894,8 @@ static void x64_store_int_part(X64NativeTarget* a, NativeAddr addr,
mem.size = chunk_size;
x64_emit_mem(a, 0, chunk_src, chunk_addr, mem);
}
+ if (private_src)
+ emit_sse_rr_w(t->mc, 0x66, 0x7E, 1, X64_TMP_FP, rs);
}
/* ============================ moves / data ============================ */
@@ -2042,13 +2034,11 @@ static void x64_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
a->next_param_stack = 0;
a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
- a->call_callee_parked = 0;
a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
a->reg_save_slot = NATIVE_FRAME_SLOT_NONE;
a->npatches = 0;
a->nalloca = 0;
a->nbind_moves = 0;
- a->part_scratch = X64_TMP_INT;
a->slim_frame = 0;
a->redzone_leaf = 0;
/* Single-pass (-O0) reservation: the NDT caches only in its allocable pool's
@@ -2148,9 +2138,9 @@ static u32 x64_live_callee_saved_mask(NativeTarget* t, NativeAllocClass cls) {
}
static u32 x64_live_caller_saved_mask(NativeTarget* t, NativeAllocClass cls) {
- const NativeAllocClassInfo* ci = native_target_class_info(t, cls);
- if (!ci) return 0;
- return ci->caller_saved_mask & ~x64_live_callee_saved_mask(t, cls);
+ u32 caller_saved =
+ native_reg_info_flag_mask(t->regs, cls, NATIVE_REG_CALLER_SAVED);
+ return caller_saved & ~x64_live_callee_saved_mask(t, cls);
}
static void x64_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
@@ -2169,33 +2159,22 @@ static void x64_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
* reserved-but-callee- saved scratch rbx/r12 (which the caller still expects
* preserved). This is the same register selection the per-block spill used,
* hoisted into the prologue. */
-static u32 x64_known_callee_saves(NativeTarget* t, const X64ABIRegs* abi,
+static int x64_asm_reg_is_callee_saved(NativeTarget* t, NativeAllocClass cls,
+ Reg r) {
+ X64NativeTarget* a = x64_of(t);
+ const X64ABIRegs* abi = a->abi ? a->abi : x64_abi_for_os(t->c->target.os);
+ return (cls == NATIVE_REG_INT && x64_reg_is_callee_int(abi, r)) ||
+ (cls == NATIVE_REG_FP && x64_reg_is_callee_fp(abi, r));
+}
+
+static u32 x64_known_callee_saves(NativeTarget* t,
const NativeKnownFrameDesc* frame, u32* out,
u32 cap) {
- u32 ncls = frame->ncallee_classes;
- u32 clob_int = 0, clob_fp = 0, abi_int, abi_fp;
- if (ncls > cap) ncls = cap;
- for (u32 c = 0; c < ncls; ++c)
- out[c] = frame->callee_saved_used ? frame->callee_saved_used[c] : 0u;
- if (frame->asm_clobbers && frame->nasm_clobbers) {
- X64NativeTarget* a = x64_of(t);
- SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
- x64_asm_clobber_masks(t->c, loc, frame->asm_clobbers, frame->nasm_clobbers,
- &clob_int, &clob_fp);
- }
- native_asm_abi_clobber_masks(t, frame->asm_clobber_abi_sets, &abi_int,
- &abi_fp);
- clob_int |= abi_int;
- clob_fp |= abi_fp;
- for (Reg r = 0; r < 16u; ++r) {
- if (NATIVE_REG_INT < ncls && (clob_int & (1u << r)) &&
- x64_reg_is_callee_int(abi, r))
- out[NATIVE_REG_INT] |= 1u << r;
- if (NATIVE_REG_FP < ncls && (clob_fp & (1u << r)) &&
- x64_reg_is_callee_fp(abi, r))
- out[NATIVE_REG_FP] |= 1u << r;
- }
- return ncls;
+ X64NativeTarget* a = x64_of(t);
+ SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
+ return native_asm_known_callee_saves(
+ t, loc, frame, out, cap, x64_asm_clobber_masks,
+ x64_asm_reg_is_callee_saved);
}
/* Optimizer entry point: the full frame is supplied up front, so the prologue
@@ -2214,9 +2193,9 @@ static void x64_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
x64_func_begin_common(t, fd);
a->frame.known_frame = 1;
if (frame) {
- u32 cs[NATIVE_CALL_PLAN_CLASSES];
+ u32 cs[NATIVE_REG_CLASS_COUNT];
u32 ncs =
- x64_known_callee_saves(t, a->abi, frame, cs, NATIVE_CALL_PLAN_CLASSES);
+ x64_known_callee_saves(t, frame, cs, NATIVE_REG_CLASS_COUNT);
a->frame.has_alloca = frame->has_alloca;
if (ncs) x64_reserve_callee_saves(t, cs, ncs);
for (i = 0; i < frame->nslots; ++i) {
@@ -2421,7 +2400,7 @@ static u32 x64_call_stack_raw_size(NativeTarget* t,
for (i = 0; i < desc->nargs; ++i) {
ABIArgInfo tmp;
const ABIArgInfo* ai = x64_param_abi(t, abi, desc, i, &tmp);
- u16 p;
+ u32 p;
if (ai->kind == ABI_ARG_IGNORE) continue;
if (ai->kind == ABI_ARG_INDIRECT) {
if (next_int < aregs->n_int_args)
@@ -2598,14 +2577,22 @@ static void x64_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
x64_load_addr(t, dst, addr);
}
-static void x64_store_outgoing_part(NativeTarget* t, int tail_call,
+typedef struct X64CallMarshalCtx {
+ u32 rsp_bias;
+ Reg cycle_scratch;
+ u8 tail;
+ u8 callee_parked;
+} X64CallMarshalCtx;
+
+static void x64_store_outgoing_part(NativeTarget* t,
+ const X64CallMarshalCtx* call,
u32 stack_off, NativeLoc src, u32 size) {
X64NativeTarget* a = x64_of(t);
NativeAddr addr;
memset(&addr, 0, sizeof addr);
addr.base_kind = NATIVE_ADDR_BASE_REG;
addr.base_type = src.type;
- if (tail_call) {
+ if (call->tail) {
/* A sibling call reuses the caller's frame: its outgoing stack args land in
* the caller's incoming-arg window. `stack_off` already includes the
* shadow-space prefix (the outgoing cursor starts at shadow_space), so the
@@ -2615,10 +2602,9 @@ static void x64_store_outgoing_part(NativeTarget* t, int tail_call,
addr.offset = (i32)(16u + stack_off);
} else {
addr.base.reg = X64_RSP;
- /* An indirect callee is transiently pushed below rsp for the whole
- * marshal phase. Keep semantic outgoing offsets anchored at the original
- * rsp; the balanced pop restores that anchor before call/jmp emission. */
- addr.offset = (i32)(stack_off + (a->call_callee_parked ? 8u : 0u));
+ /* Keep semantic outgoing offsets anchored at the pre-marshalling rsp. The
+ * per-call context owns any transient callee-save bias explicitly. */
+ addr.offset = (i32)(stack_off + call->rsp_bias);
}
{
MemAccess mem = native_mem_for_type(t, src.type, size);
@@ -2633,8 +2619,9 @@ static void x64_store_outgoing_part(NativeTarget* t, int tail_call,
* so copy a non-power-of-two memory part chunk-by-chunk instead of needlessly
* packing it into a GPR and unpacking it again. Besides being smaller, this
* needs only one private register while an indirect callee may occupy r11. */
-static void x64_copy_outgoing_part(NativeTarget* t, int tail_call,
- u32 stack_off, NativeLoc src,
+static void x64_copy_outgoing_part(NativeTarget* t,
+ const X64CallMarshalCtx* call, u32 stack_off,
+ NativeLoc src,
const ABIArgPart* part) {
NativeAllocClass cls =
part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
@@ -2648,14 +2635,14 @@ static void x64_copy_outgoing_part(NativeTarget* t, int tail_call,
u32 chunk_off, chunk_size;
while (native_part_chunk_next(&chunks, &chunk_off, &chunk_size)) {
x64_load_part(t, tmpreg, src, part->src_offset + chunk_off, chunk_size);
- x64_store_outgoing_part(t, tail_call, stack_off + chunk_off, tmpreg,
+ x64_store_outgoing_part(t, call, stack_off + chunk_off, tmpreg,
chunk_size);
}
return;
}
x64_load_part(t, tmpreg, src, part->src_offset, part->size);
- x64_store_outgoing_part(t, tail_call, stack_off, tmpreg, part->size);
+ x64_store_outgoing_part(t, call, stack_off, tmpreg, part->size);
}
/* NativeTarget bind_param: route incoming param (ABI loc) into dst. */
@@ -2702,7 +2689,7 @@ static void x64_bind_native_param(NativeTarget* t, const CGParamDesc* p,
/* Incoming stack args sit above the saved rbp + return addr (+16); Win64
* additionally reserves 32B of home space. */
i32 incoming_bias = (i32)(16u + a->abi->shadow_space);
- u16 i;
+ u32 i;
if (!ai || ai->kind == ABI_ARG_IGNORE) return;
if (ai->kind == ABI_ARG_INDIRECT) {
@@ -2863,49 +2850,19 @@ static void x64_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves, u32 n,
s.reg_move = x64_move;
s.scratch[NATIVE_REG_INT] = int_scratch;
s.scratch[NATIVE_REG_FP] = X64_TMP_FP;
+ s.scratch_class_mask = (1u << NATIVE_REG_INT) | (1u << NATIVE_REG_FP);
native_arg_shuffle(&s, moves, n);
}
-/* Clobber masks: per-call all caller-saved regs are clobbered. */
-static u32 x64_clobber_mask(const X64ABIRegs* abi, NativeAllocClass cls) {
- u32 mask = 0, r;
- if (cls == NATIVE_REG_INT) {
- for (r = 0; r < 16u; ++r) {
- if (r == X64_RSP || r == X64_RBP) continue;
- if ((abi->cs_int_mask & (1ull << r)) == 0) mask |= 1u << r;
- }
- } else if (cls == NATIVE_REG_FP) {
- for (r = 0; r < 16u; ++r)
- if ((abi->cs_fp_mask & (1ull << r)) == 0) mask |= 1u << r;
- }
- return mask;
-}
-
-static u32 x64_return_mask(const ABIFuncInfo* abi, NativeAllocClass cls) {
- u32 mask = 0, ni = 0, nf = 0;
- static const u32 iregs[2] = {X64_RAX, X64_RDX};
- u16 i;
- if (!abi || abi->ret.kind == ABI_ARG_IGNORE ||
- abi->ret.kind == ABI_ARG_INDIRECT)
- return 0;
- for (i = 0; i < abi->ret.nparts; ++i) {
- const ABIArgPart* p = &abi->ret.parts[i];
- if (cls == NATIVE_REG_INT && p->cls == ABI_CLASS_INT && ni < 2)
- mask |= 1u << iregs[ni++];
- else if (cls == NATIVE_REG_FP && p->cls == ABI_CLASS_FP && nf < 2)
- mask |= 1u << (X64_XMM0 + nf++);
- }
- return mask;
-}
-
-static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
- NativeCallPlan* plan) {
+static void x64_marshal_call(NativeTarget* t, const NativeCallDesc* desc,
+ NativeCallPhase* plan) {
X64NativeTarget* a = x64_of(t);
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
const X64ABIRegs* aregs = a->abi ? a->abi : x64_abi_for_os(t->c->target.os);
- NativeCallPlanRet* rets;
+ NativeCallPhaseRet* rets;
+ X64CallMarshalCtx call;
KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
- u32 c;
+ int tail = (desc->flags & CG_CALL_TAIL) != 0;
/* Right-size the result scratch to the exact number of entries the ret loops
* below write: nparts on a DIRECT register return, 1 on the !abi fallback,
* 0 (NULL) otherwise (IGNORE / sret / no results). */
@@ -2913,7 +2870,10 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
? abi->ret.nparts
: ((!abi && desc->nresults) ? 1u : 0u);
memset(plan, 0, sizeof *plan);
- rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPlanRet, nrets_cap) : NULL;
+ memset(&call, 0, sizeof call);
+ call.tail = (u8)tail;
+ call.cycle_scratch = X64_TMP_INT2;
+ rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPhaseRet, nrets_cap) : NULL;
plan->callee = desc->callee;
plan->rets = rets;
plan->flags = desc->flags;
@@ -2922,10 +2882,6 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
plan->stack_arg_size = x64_call_stack_bytes(t, desc);
if (plan->stack_arg_size > a->frame.max_outgoing)
a->frame.max_outgoing = plan->stack_arg_size;
- for (c = 0; c < NATIVE_CALL_PLAN_CLASSES; ++c) {
- plan->clobber_mask[c] = x64_clobber_mask(aregs, (NativeAllocClass)c);
- plan->return_mask[c] = x64_return_mask(abi, (NativeAllocClass)c);
- }
/* Argument transport owns r10 (part data), r11 (address stabilization), and
* rax (parallel-copy cycle breaking). A register-indirect callee therefore
* lives in a balanced stack home for the complete marshal phase, independent
@@ -2933,18 +2889,17 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
if (plan->callee.kind == NATIVE_LOC_REG) {
if ((NativeAllocClass)plan->callee.cls != NATIVE_REG_INT)
x64_panic(a, "indirect callee is not in an integer register");
- if (a->call_callee_parked)
- x64_panic(a, "nested indirect-callee marshal");
x64_push_reg(t->mc, (Reg)loc_reg(plan->callee));
- a->call_callee_parked = 1;
+ call.rsp_bias = 8u;
+ call.callee_parked = 1u;
+ call.cycle_scratch = X64_RAX;
plan->callee = native_loc_reg(plan->callee.type, NATIVE_REG_INT, X64_R11);
}
{
u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
u32 next_fp = 0, stack = aregs->shadow_space, nmoves = 0, i;
- int tail = (desc->flags & CG_CALL_TAIL) != 0;
u32 tail_payload = tail ? x64_call_stack_raw_size(t, desc) : 0u;
- u16 p;
+ u32 p;
X64ArgMove moves[X64_MAX_REG_ARG_MOVES];
x64_sync_slot(aregs, &next_int, &next_fp);
for (i = 0; i < desc->nargs; ++i) {
@@ -2983,7 +2938,7 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
} else {
NativeLoc ptr = native_loc_reg(i64t, NATIVE_REG_INT, X64_RAX);
x64_addr_of_loc(t, ptr, src);
- x64_store_outgoing_part(t, tail, stack, ptr, 8);
+ x64_store_outgoing_part(t, &call, stack, ptr, 8);
stack += 8u;
}
x64_sync_slot(aregs, &next_int, &next_fp);
@@ -2993,7 +2948,7 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
x64_direct_to_stack(aregs, ai, next_int, next_fp)) {
for (p = 0; p < ai->nparts; ++p) {
const ABIArgPart* part = &ai->parts[p];
- x64_copy_outgoing_part(t, tail, stack, desc->args[i], part);
+ x64_copy_outgoing_part(t, &call, stack, desc->args[i], part);
stack += 8u;
}
continue;
@@ -3027,7 +2982,7 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
m->size = part->size;
x64_sync_slot(aregs, &next_int, &next_fp);
} else {
- x64_copy_outgoing_part(t, tail, stack, desc->args[i], part);
+ x64_copy_outgoing_part(t, &call, stack, desc->args[i], part);
stack += 8u;
x64_sync_slot(aregs, &next_int, &next_fp);
}
@@ -3035,11 +2990,7 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
}
/* If an indirect callee was staged in r11 above, the cycle scratch must
* avoid it; rax is free here (not an int arg reg; AL count comes later). */
- x64_emit_reg_arg_moves(
- t, moves, nmoves,
- (plan->callee.kind == NATIVE_LOC_REG && plan->callee.v.reg == X64_R11)
- ? X64_RAX
- : X64_TMP_INT2);
+ x64_emit_reg_arg_moves(t, moves, nmoves, call.cycle_scratch);
if (abi && abi->has_sret) {
/* sret pointer in the first int-arg reg. A tail call forwards the
* caller's own incoming sret pointer (spilled at entry); otherwise pass
@@ -3054,16 +3005,17 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
/* Variadic call: AL = number of vector regs used. */
if (abi && abi->variadic)
x64_emit_load_imm(t->mc, 0, X64_RAX, (i64)next_fp);
- if (a->call_callee_parked) {
+ if (call.callee_parked) {
x64_pop_reg(t->mc, X64_R11);
- a->call_callee_parked = 0;
+ call.rsp_bias = 0u;
+ call.callee_parked = 0u;
}
}
/* Return value receipt. */
if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
u32 nr = 0, ni = 0, nf = 0;
static const u32 ret_int_regs[2] = {X64_RAX, X64_RDX};
- u16 p;
+ u32 p;
for (p = 0; p < abi->ret.nparts; ++p) {
const ABIArgPart* part = &abi->ret.parts[p];
NativeAllocClass cls =
@@ -3105,7 +3057,7 @@ static void x64_emit_tail_site(NativeTarget* t, NativeLoc callee) {
ObjSecId sec = mc->section_id;
/* Restore callee-saves before the frame teardown (O1 path; none at -O0).
* Their rbp-relative offsets are frame-size-independent, and the indirect
- * callee was staged in r11 by plan_call — a caller-saved scratch — so these
+ * callee was staged in r11 by marshal_call — a caller-saved scratch — so these
* restores never clobber it. Mirrors the x64_func_end epilogue. */
x64_emit_callee_restores(a);
emit_leave(mc);
@@ -3118,7 +3070,7 @@ static void x64_emit_tail_site(NativeTarget* t, NativeLoc callee) {
mc_emit_reloc_at(mc, sec, disp_pos, R_X64_PLT32, callee.v.global.sym,
callee.v.global.addend - 4, 1, 0);
} else if (callee.kind == NATIVE_LOC_REG) {
- /* indirect callee was staged in r11 by plan_call */
+ /* indirect callee was staged in r11 by marshal_call */
u32 r = loc_reg(callee);
x64_emit_indirect_rm(mc, r, 4u); /* jmp r/m, /4 */
} else {
@@ -3126,7 +3078,7 @@ static void x64_emit_tail_site(NativeTarget* t, NativeLoc callee) {
}
}
-static void x64_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
+static void x64_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
MCEmitter* mc = t->mc;
ObjSecId sec = mc->section_id;
if (plan->flags & CG_CALL_TAIL) {
@@ -3151,14 +3103,14 @@ static void x64_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
x64_panic(x64_of(t), "unsupported call target");
}
-static void x64_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
- const NativeLoc* value, NativeCallPlanRet** out_rets,
- u32* out_nrets) {
+static void x64_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
+ const NativeLoc* value,
+ NativeCallPhaseRet** out_rets, u32* out_nrets) {
X64NativeTarget* a = x64_of(t);
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
- NativeCallPlanRet* rets = NULL;
+ NativeCallPhaseRet* rets = NULL;
u32 nr = 0;
- if (value) rets = arena_zarray(t->c->tu, NativeCallPlanRet, 4);
+ if (value) rets = arena_zarray(t->c->tu, NativeCallPhaseRet, 4);
if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
/* sret: reload the destination pointer (spilled at entry) into rax and
* copy the source aggregate into [rax]. Keep it out of R10/R11: those are
@@ -3190,7 +3142,7 @@ static void x64_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
u32 ni = 0, nf = 0;
static const u32 ret_int_regs[2] = {X64_RAX, X64_RDX};
- u16 p;
+ u32 p;
int exact_parts = 0;
for (p = 0; p < abi->ret.nparts; ++p)
if (abi->ret.parts[p].cls == ABI_CLASS_INT &&
@@ -4349,31 +4301,6 @@ static void x64_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
/* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
-/* Pick a free register from caller-saved allocable pools for an asm operand the
- * direct path self-allocates. */
-static Reg x64_asm_alloc_reg(NativeDirectTarget* d, NativeAllocClass cls,
- u32 allowed_mask, u32* used_int, u32* used_fp) {
- static const Reg int_pool[] = {X64_RDI, X64_RSI, X64_RDX,
- X64_RCX, X64_R8, X64_R9};
- static const Reg fp_pool[] = {
- X64_XMM0, X64_XMM1, X64_XMM2, X64_XMM3, X64_XMM4, X64_XMM5,
- X64_XMM6, X64_XMM7, X64_XMM8, X64_XMM0 + 9, X64_XMM0 + 10, X64_XMM0 + 11};
- const Reg* pool = cls == NATIVE_REG_FP ? fp_pool : int_pool;
- u32 n = cls == NATIVE_REG_FP ? (u32)(sizeof fp_pool / sizeof fp_pool[0])
- : (u32)(sizeof int_pool / sizeof int_pool[0]);
- u32* used = cls == NATIVE_REG_FP ? used_fp : used_int;
- u32 i;
- for (i = 0; i < n; ++i) {
- Reg r = pool[i];
- if (allowed_mask && (allowed_mask & (1u << r)) == 0) continue;
- if ((*used & (1u << r)) != 0) continue;
- *used |= 1u << r;
- return r;
- }
- x64_asm_panic(d, "out of registers for asm operands");
- return REG_NONE;
-}
-
/* Direct (-O0) path: resolve a semantic Operand to a NativeAddr. */
static NativeAddr x64_direct_addr(NativeDirectTarget* d, Operand op) {
NativeAddr addr;
@@ -4640,6 +4567,33 @@ static void x64_set_loc(NativeTarget* t, SrcLoc loc) {
mc_set_loc(t->mc, loc);
}
+static u32 x64_binop_visible_clobbers(const NativeMachineOp* op) {
+ switch ((BinOp)op->binop) {
+ case BO_SDIV:
+ case BO_UDIV:
+ case BO_SREM:
+ case BO_UREM:
+ return (1u << X64_RAX) | (1u << X64_RDX);
+ case BO_SHL:
+ case BO_SHR_S:
+ case BO_SHR_U:
+ return op->second_is_reg ? (1u << X64_RCX) : 0u;
+ default:
+ return 0u;
+ }
+}
+
+static u32 x64_intrinsic_visible_clobbers(IntrinKind kind) {
+ if (kind == INTRIN_UMUL_OVERFLOW || kind == INTRIN_SMUL_HIGH ||
+ kind == INTRIN_UMUL_HIGH || kind == INTRIN_READCYCLECOUNTER)
+ return (1u << X64_RAX) | (1u << X64_RDX);
+ if (kind == INTRIN_SYSCALL)
+ return (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX) |
+ (1u << X64_RSI) | (1u << X64_RDI) | (1u << X64_R8) |
+ (1u << X64_R9);
+ return 0u;
+}
+
/* Physical registers each x86-64 instruction's encoding clobbers as a side
* effect, so the optimizer keeps values live across them out of those registers
* (the backend is then free to use them). idiv/div write rax (quotient) and rdx
@@ -4648,7 +4602,7 @@ static void x64_set_loc(NativeTarget* t, SrcLoc loc) {
* All other ordinary helpers use declared results or backend-private R10/R11
* and XMM14/XMM15 temporaries. */
static int x64_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
- u32 mask[NATIVE_CALL_PLAN_CLASSES]) {
+ u32 mask[NATIVE_REG_CLASS_COUNT]) {
mask[0] = mask[1] = mask[2] = 0;
switch ((NativeMachineOpKind)op->kind) {
case NATIVE_MOP_TLS_ADDR:
@@ -4662,24 +4616,8 @@ static int x64_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
mask[NATIVE_REG_INT] = (1u << X64_RAX) | (1u << X64_RDI);
return 1;
case NATIVE_MOP_BINOP:
- switch ((BinOp)op->binop) {
- case BO_SDIV:
- case BO_UDIV:
- case BO_SREM:
- case BO_UREM:
- mask[NATIVE_REG_INT] = (1u << X64_RAX) | (1u << X64_RDX);
- return 1;
- case BO_SHL:
- case BO_SHR_S:
- case BO_SHR_U:
- if (op->second_is_reg) {
- mask[NATIVE_REG_INT] = (1u << X64_RCX);
- return 1;
- }
- return 0;
- default:
- return 0;
- }
+ mask[NATIVE_REG_INT] = x64_binop_visible_clobbers(op);
+ return mask[NATIVE_REG_INT] != 0u;
case NATIVE_MOP_BITFIELD_LOAD:
/* The only non-destination temporary is backend-private R11. */
return 0;
@@ -4701,29 +4639,16 @@ static int x64_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
* two-operand IMUL (no fixed-register clobber). Linux syscall writes rax
* and the CPU instruction itself clobbers rcx/r11; the kernel ABI treats
* the integer caller-saved syscall registers as volatile. */
- if ((IntrinKind)op->intrin == INTRIN_UMUL_OVERFLOW ||
- (IntrinKind)op->intrin == INTRIN_SMUL_HIGH ||
- (IntrinKind)op->intrin == INTRIN_UMUL_HIGH ||
- (IntrinKind)op->intrin == INTRIN_READCYCLECOUNTER) {
- /* MUL's rdx:rax product / RDTSC's edx:eax counter both write both
- * registers; keep live values out of them across the op. */
- mask[NATIVE_REG_INT] = (1u << X64_RAX) | (1u << X64_RDX);
- return 1;
- }
- if ((IntrinKind)op->intrin == INTRIN_SYSCALL) {
- /* R8/R9 are cache-capable emitter temps, so the effect must invalidate
- * retained spill values there even though they are not MIR homes.
- * R10/R11 are pure backend-private temps and remain excluded. */
- mask[NATIVE_REG_INT] =
- (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX) |
- (1u << X64_RSI) | (1u << X64_RDI) | (1u << X64_R8) |
- (1u << X64_R9);
- return 1;
- }
- return 0;
- default:
- return 0;
+ /* R8/R9 are cache-capable emitter temps, so the syscall effect includes
+ * them. R10/R11 are pure backend-private temps and remain excluded. */
+ mask[NATIVE_REG_INT] =
+ x64_intrinsic_visible_clobbers((IntrinKind)op->intrin);
+ return mask[NATIVE_REG_INT] != 0u;
+ case NATIVE_MOP_COUNT:
+ break;
}
+ x64_panic(x64_of(t), "invalid machine-effect operation");
+ return 0;
}
/* ============================ construction ============================ */
@@ -4788,9 +4713,9 @@ NativeTarget* x64_native_target_new(Compiler* c, ObjBuilder* obj,
t->alloca_ = x64_alloca;
t->spill = x64_spill;
t->reload = x64_reload;
- t->plan_call = x64_plan_call;
+ t->marshal_call = x64_marshal_call;
t->emit_call = x64_emit_call;
- t->plan_ret = x64_plan_ret;
+ t->marshal_ret = x64_marshal_ret;
t->ret = x64_ret;
t->atomic_load = x64_atomic_load;
t->atomic_store = x64_atomic_store;
@@ -4943,20 +4868,11 @@ static void x64_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
const Sym* clobbers, u32 nclob,
u32 clobber_abi_sets) {
static const NativeAsmDirectHooks hooks = {
- /* Reserve backend temps R10/R11, O0 scratch R8/R9, RAX (reserved; only
- * self-allocated here when explicitly pinned), and SP/BP. */
- .reserved_int_mask =
- (1u << X64_RAX) | (1u << X64_R11) | (1u << X64_RSP) |
- (1u << X64_RBP) | (1u << X64_R8) | (1u << X64_R9) |
- (1u << X64_R10),
- .reserved_fp_mask = (1u << X64_XMM4) | (1u << X64_XMM5) |
- (1u << (X64_XMM0 + 14)) | (1u << X64_XMM15),
.opk_reg = X64_INLINE_OPK_REG,
.opcls_fp = X64_INLINE_OPCLS_FP,
.panic = x64_asm_panic,
.bound_reg = x64_asm_bound_reg,
.bound_mem = x64_asm_bound_mem,
- .alloc_reg = x64_asm_alloc_reg,
.clobber_masks = x64_asm_clobber_masks,
.save_callee_clobbers = x64_asm_hook_save_callee_clobbers,
.restore_one = x64_asm_hook_restore_one,
diff --git a/src/cg/native_argmove.c b/src/cg/native_argmove.c
@@ -58,11 +58,36 @@ void native_arg_shuffle(const NativeArgShuffle* s, NativeArgMove* moves,
u32 k = 0;
NativeAllocClass bc, sc;
NativeLoc scratchloc;
+ Reg scratch;
Reg broken_reg;
KitCgTypeId val_type;
while (k < n && (done[k] || nam_src_reg(&moves[k], &sc) == REG_NONE)) ++k;
+ if (k == n)
+ compiler_panic(s->t->c, (SrcLoc){0, 0, 0},
+ "native arg shuffle: blocked move set has no register "
+ "cycle");
bc = (NativeAllocClass)moves[k].dst.cls;
broken_reg = moves[k].dst.v.reg;
+ if (bc >= NATIVE_REG_CLASS_COUNT)
+ compiler_panic(s->t->c, (SrcLoc){0, 0, 0},
+ "native arg shuffle: invalid class scratch");
+ if ((s->scratch_class_mask & (1u << bc)) == 0u)
+ compiler_panic(s->t->c, (SrcLoc){0, 0, 0},
+ "native arg shuffle: missing class scratch");
+ scratch = s->scratch[bc];
+ if (scratch >= NATIVE_MAX_HARD_REGS)
+ compiler_panic(s->t->c, (SrcLoc){0, 0, 0},
+ "native arg shuffle: invalid class scratch");
+ for (j = 0; j < n; ++j) {
+ Reg sr = nam_src_reg(&moves[j], &sc);
+ if (done[j]) continue;
+ if (((NativeAllocClass)moves[j].dst.cls == bc &&
+ moves[j].dst.v.reg == scratch) ||
+ (sr != REG_NONE && sc == bc && sr == scratch))
+ compiler_panic(s->t->c, (SrcLoc){0, 0, 0},
+ "native arg shuffle: cycle scratch aliases a live "
+ "move register");
+ }
/* The value parked in broken_reg is whatever the cycle's readers consume,
* and its width is their *source* type — not moves[k].dst's own incoming
* type, which can be narrower (e.g. a 4-byte int rotating into the slot a
@@ -81,7 +106,7 @@ void native_arg_shuffle(const NativeArgShuffle* s, NativeArgMove* moves,
scratchloc.kind = NATIVE_LOC_REG;
scratchloc.cls = (u8)bc;
scratchloc.type = val_type;
- scratchloc.v.reg = s->scratch[bc];
+ scratchloc.v.reg = scratch;
s->reg_move(s->t, scratchloc, moves[k].dst);
for (j = 0; j < n; ++j) {
Reg sr = nam_src_reg(&moves[j], &sc);
diff --git a/src/cg/native_argmove.h b/src/cg/native_argmove.h
@@ -50,8 +50,10 @@ typedef struct NativeArgShuffle {
void (*reg_move)(NativeTarget* t, NativeLoc dst, NativeLoc src);
/* Scratch register per NativeAllocClass (index by class id) for breaking
* cycles. A cycle's ring is register-to-register, so the scratch needs no
- * address arithmetic — a bare register suffices. */
- Reg scratch[NATIVE_REG_VEC + 1];
+ * address arithmetic — a bare register suffices. The mask makes configured
+ * classes explicit even when physical register zero is a valid scratch. */
+ Reg scratch[NATIVE_REG_CLASS_COUNT];
+ u32 scratch_class_mask;
} NativeArgShuffle;
/* Emit `moves[0..n)` as a parallel register copy via `s`. Mutates `moves`
diff --git a/src/cg/native_asm.c b/src/cg/native_asm.c
@@ -46,6 +46,35 @@ void native_asm_abi_clobber_masks(NativeTarget* t, u32 abi_sets, u32* int_mask,
}
}
+u32 native_asm_known_callee_saves(
+ NativeTarget* t, SrcLoc loc, const NativeKnownFrameDesc* frame, u32* out,
+ u32 cap, NativeAsmClobberMasksFn clobber_masks,
+ NativeAsmCalleeSavedRegFn is_callee_saved) {
+ u32 ncls = frame ? frame->ncallee_classes : 0u;
+ u32 clob_int = 0, clob_fp = 0, abi_int = 0, abi_fp = 0;
+ if (ncls > cap) ncls = cap;
+ for (u32 c = 0; c < cap; ++c) out[c] = 0;
+ for (u32 c = 0; c < ncls; ++c)
+ out[c] = frame->callee_saved_used ? frame->callee_saved_used[c] : 0u;
+ if (frame && frame->asm_clobbers && frame->nasm_clobbers && clobber_masks)
+ clobber_masks(t->c, loc, frame->asm_clobbers, frame->nasm_clobbers,
+ &clob_int, &clob_fp);
+ if (frame)
+ native_asm_abi_clobber_masks(t, frame->asm_clobber_abi_sets, &abi_int,
+ &abi_fp);
+ clob_int |= abi_int;
+ clob_fp |= abi_fp;
+ for (Reg r = 0; r < NATIVE_MAX_HARD_REGS; ++r) {
+ if (NATIVE_REG_INT < ncls && (clob_int & (1u << r)) &&
+ is_callee_saved(t, NATIVE_REG_INT, r))
+ out[NATIVE_REG_INT] |= 1u << r;
+ if (NATIVE_REG_FP < ncls && (clob_fp & (1u << r)) &&
+ is_callee_saved(t, NATIVE_REG_FP, r))
+ out[NATIVE_REG_FP] |= 1u << r;
+ }
+ return ncls;
+}
+
int native_asm_constraint_reg_class(const char* constraint,
NativeAllocClass* cls_out) {
const char* body = native_asm_constraint_body(constraint);
@@ -63,8 +92,8 @@ int native_asm_constraint_reg_class(const char* constraint,
static int native_asm_default_operand_reg_ok(const NativeRegInfo* ri,
NativeAllocClass cls, Reg reg) {
- if (!ri || cls >= ri->nclasses) return 0;
- const NativeAllocClassInfo* ci = &ri->classes[cls];
+ const NativeAllocClassInfo* ci = native_reg_info_class_info(ri, cls);
+ if (!ci) return 0;
for (u32 i = 0; i < ci->nphys; ++i) {
const NativePhysRegInfo* pi = &ci->phys[i];
if (pi->reg != reg) continue;
@@ -194,6 +223,32 @@ static int native_asm_direct_resolve_pin(NativeDirectTarget* d,
return 1;
}
+static Reg native_asm_direct_alloc_reg(NativeDirectTarget* d,
+ const NativeAsmDirectHooks* h,
+ NativeAllocClass cls, u32 allowed_mask,
+ u32* used_int, u32* used_fp) {
+ const NativeAllocClassInfo* ci = native_target_class_info(d->native, cls);
+ u32* used = cls == NATIVE_REG_FP ? used_fp : used_int;
+ for (u32 i = 0; ci && i < ci->ndirect_asm_allocable; ++i) {
+ Reg reg = ci->direct_asm_allocable[i];
+ if (reg >= NATIVE_MAX_HARD_REGS) continue;
+ if (allowed_mask && (allowed_mask & (1u << reg)) == 0) continue;
+ if ((*used & (1u << reg)) != 0) continue;
+ *used |= 1u << reg;
+ return reg;
+ }
+ h->panic(d, "out of registers for asm operands");
+ return REG_NONE;
+}
+
+static KitCgTypeId native_asm_pointer_carrier_type(NativeTarget* t) {
+ if (t->c->target.ptr_size == 4u) return builtin_id(KIT_CG_BUILTIN_I32);
+ if (t->c->target.ptr_size == 8u) return builtin_id(KIT_CG_BUILTIN_I64);
+ compiler_panic(t->c, (SrcLoc){0, 0, 0},
+ "native asm: unsupported pointer carrier width %u",
+ (unsigned)t->c->target.ptr_size);
+}
+
void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
const AsmConstraint* outs, u32 nout,
Operand* out_ops, const AsmConstraint* ins,
@@ -212,8 +267,8 @@ void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
native_asm_abi_clobber_masks(d->native, clobber_abi_sets, &abi_int, &abi_fp);
clob_int |= abi_int;
clob_fp |= abi_fp;
- used_int = clob_int | h->reserved_int_mask;
- used_fp = clob_fp | h->reserved_fp_mask;
+ used_int = clob_int;
+ used_fp = clob_fp;
for (i = 0; i < nout; ++i) {
const char* body = native_asm_constraint_body(outs[i].str);
@@ -234,8 +289,9 @@ void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
if (native_asm_constraint_reg_info(d->native, outs[i].str, &info)) {
Reg reg = info.fixed_reg != REG_NONE
? info.fixed_reg
- : h->alloc_reg(d, info.cls, info.allowed_mask, &used_int,
- &used_fp);
+ : native_asm_direct_alloc_reg(
+ d, h, info.cls, info.allowed_mask, &used_int,
+ &used_fp);
if (info.cls == NATIVE_REG_FP) {
used_fp |= 1u << reg;
/* Binding itself overwrites the staging register, independently of
@@ -249,7 +305,8 @@ void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
}
h->bound_reg(&bound_outs[i], type, info.cls, reg);
} else if (body[0] == 'm') {
- Reg reg = h->alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
+ Reg reg = native_asm_direct_alloc_reg(
+ d, h, NATIVE_REG_INT, 0, &used_int, &used_fp);
clob_int |= 1u << reg;
h->bound_mem(&bound_outs[i], type, reg);
} else {
@@ -287,8 +344,9 @@ void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
if (native_asm_constraint_reg_info(d->native, ins[i].str, &info)) {
Reg reg = info.fixed_reg != REG_NONE
? info.fixed_reg
- : h->alloc_reg(d, info.cls, info.allowed_mask, &used_int,
- &used_fp);
+ : native_asm_direct_alloc_reg(
+ d, h, info.cls, info.allowed_mask, &used_int,
+ &used_fp);
if (info.cls == NATIVE_REG_FP) {
used_fp |= 1u << reg;
clob_fp |= 1u << reg;
@@ -302,7 +360,8 @@ void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
h->panic(d, "immediate constraint requires immediate operand");
bound_ins[i] = in_ops[i];
} else if (body[0] == 'm') {
- Reg reg = h->alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
+ Reg reg = native_asm_direct_alloc_reg(
+ d, h, NATIVE_REG_INT, 0, &used_int, &used_fp);
clob_int |= 1u << reg;
h->bound_mem(&bound_ins[i], type, reg);
} else {
@@ -322,9 +381,9 @@ void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
(Reg)bound_outs[i].v.local));
}
} else if (bound_outs[i].kind == OPK_INDIRECT) {
- NativeLoc loc =
- native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- (Reg)bound_outs[i].v.ind.base);
+ NativeLoc loc = native_loc_reg(native_asm_pointer_carrier_type(d->native),
+ NATIVE_REG_INT,
+ (Reg)bound_outs[i].v.ind.base);
h->load_address_to_reg(d, out_ops[i], loc);
}
}
@@ -336,9 +395,9 @@ void native_asm_bind_direct_operands(NativeDirectTarget* d, const char* tmpl,
d, in_ops[i],
native_loc_reg(bound_ins[i].type, cls, (Reg)bound_ins[i].v.local));
} else if (bound_ins[i].kind == OPK_INDIRECT) {
- NativeLoc loc =
- native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
- (Reg)bound_ins[i].v.ind.base);
+ NativeLoc loc = native_loc_reg(native_asm_pointer_carrier_type(d->native),
+ NATIVE_REG_INT,
+ (Reg)bound_ins[i].v.ind.base);
h->load_address_to_reg(d, in_ops[i], loc);
}
}
diff --git a/src/cg/native_asm.h b/src/cg/native_asm.h
@@ -43,6 +43,19 @@ int native_asm_match_index(const char* s);
void native_asm_abi_clobber_masks(NativeTarget* t, u32 abi_sets, u32* int_mask,
u32* fp_mask);
+typedef void (*NativeAsmClobberMasksFn)(Compiler*, SrcLoc, const Sym*, u32,
+ u32*, u32*);
+typedef int (*NativeAsmCalleeSavedRegFn)(NativeTarget*, NativeAllocClass, Reg);
+
+/* Merge allocator-used and inline-asm-clobbered callee-saved registers for a
+ * known frame. Target parsing remains behind clobber_masks; is_callee_saved
+ * defines the live ABI's preservable register set and excludes structural
+ * registers such as the frame pointer. */
+u32 native_asm_known_callee_saves(
+ NativeTarget* t, SrcLoc loc, const NativeKnownFrameDesc* frame, u32* out,
+ u32 cap, NativeAsmClobberMasksFn clobber_masks,
+ NativeAsmCalleeSavedRegFn is_callee_saved);
+
typedef enum NativeAsmRegPinStatus {
NATIVE_ASM_REG_PIN_ABSENT = 0,
NATIVE_ASM_REG_PIN_OK = 1,
@@ -96,11 +109,6 @@ int native_asm_constraint_reg_class(const char* constraint,
typedef struct NativeDirectTarget NativeDirectTarget;
typedef struct NativeAsmDirectHooks {
- /* Registers the O0 direct-asm operand allocator must never hand out
- * (backend-private temporaries, ABI fixed regs, frame pointer, ...), seeded
- * into the used masks alongside the clobber masks. Per-arch. */
- u32 reserved_int_mask;
- u32 reserved_fp_mask;
/* Arch inline-operand discriminators: Operand.kind for a bound register
* pseudo-operand, and the Operand.pad[0] value marking the FP class. */
u8 opk_reg;
@@ -113,11 +121,6 @@ typedef struct NativeAsmDirectHooks {
void (*bound_reg)(Operand* out, KitCgTypeId type, NativeAllocClass cls,
Reg reg);
void (*bound_mem)(Operand* out, KitCgTypeId type, Reg base);
- /* Pick a free register from the arch's allocable pool for cls, honoring
- * allowed_mask and the running used masks (which it updates). Panics on
- * exhaustion. */
- Reg (*alloc_reg)(NativeDirectTarget* d, NativeAllocClass cls,
- u32 allowed_mask, u32* used_int, u32* used_fp);
/* Parse the asm clobber list into per-class register masks (arch register
* naming). */
void (*clobber_masks)(Compiler* c, SrcLoc loc, const Sym* clobbers, u32 nclob,
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -138,7 +138,8 @@ static NativeAllocClass nd_class_for_type(NativeDirectTarget* d,
static const NativeAllocClassInfo* nd_class_info(NativeDirectTarget* d,
NativeAllocClass cls) {
- const NativeAllocClassInfo* ci = (u32)cls < 3u ? d->class_info[cls] : NULL;
+ const NativeAllocClassInfo* ci =
+ (u32)cls < NATIVE_REG_CLASS_COUNT ? d->class_info[cls] : NULL;
if (!ci) nd_panic(d, "target has no requested register class");
return ci;
}
@@ -170,7 +171,7 @@ static u32 nd_caller_saved_mask(NativeDirectTarget* d, NativeAllocClass cls) {
static void nd_note_reg_used(NativeDirectTarget* d, NativeAllocClass cls,
Reg reg) {
- if ((u32)cls >= 3u || reg >= 32u) return;
+ if ((u32)cls >= NATIVE_REG_CLASS_COUNT || reg >= NATIVE_MAX_HARD_REGS) return;
if (nd_callee_saved_mask(d, cls) & (1u << reg))
d->callee_saved_used[cls] |= 1u << reg;
}
@@ -1271,7 +1272,7 @@ static void nd_func_end(CgTarget* t) {
* saves/restores any callee-save, which is what lets the backend reserve
* only a tiny tcc-style deferred-`sub` prologue region. Assert it so a
* future regression can't silently overflow that region. */
- for (u32 cls = 0; cls < 3u; ++cls) {
+ for (u32 cls = 0; cls < NATIVE_REG_CLASS_COUNT; ++cls) {
if (d->callee_saved_used[cls])
nd_panic(d, "ndt_caller_saved_only target used a callee-saved register");
}
@@ -1280,7 +1281,7 @@ static void nd_func_end(CgTarget* t) {
* pressure; have the backend reserve save slots for them. The
* optimizer/known-frame path is separate and always callee-save-capable. */
u32 ncallee_classes = 0;
- for (u32 cls = 0; cls < 3u; ++cls) {
+ for (u32 cls = 0; cls < NATIVE_REG_CLASS_COUNT; ++cls) {
if (d->callee_saved_used[cls]) ncallee_classes = cls + 1u;
}
if (ncallee_classes) {
@@ -2157,7 +2158,7 @@ static void nd_convert(CgTarget* t, ConvKind op, Operand dst, Operand src) {
static void nd_call(CgTarget* t, const CGCallDesc* desc) {
NativeDirectTarget* d = nd_of(t);
- NativeCallPlan plan;
+ NativeCallPhase plan;
NativeCallDesc nd;
NativeLoc* args;
NativeLoc* results;
@@ -2241,11 +2242,11 @@ static void nd_call(CgTarget* t, const CGCallDesc* desc) {
nd.tail_policy = desc->tail_policy;
nd.inline_policy = desc->inline_policy;
- if (d->ops && d->ops->plan_call)
- d->ops->plan_call(d, &nd, &plan);
+ if (d->ops && d->ops->marshal_call)
+ d->ops->marshal_call(d, &nd, &plan);
else {
- ND_REQUIRE_NATIVE(d, plan_call, "target does not plan calls");
- d->native->plan_call(d->native, &nd, &plan);
+ ND_REQUIRE_NATIVE(d, marshal_call, "target does not marshal calls");
+ d->native->marshal_call(d->native, &nd, &plan);
}
if (plan.stack_arg_size > d->max_outgoing)
d->max_outgoing = plan.stack_arg_size;
@@ -2311,7 +2312,7 @@ static void nd_ret(CgTarget* t, CGLocal value) {
NativeDirectTarget* d = nd_of(t);
NativeLoc loc;
const NativeLoc* locp = NULL;
- NativeCallPlanRet* rets = NULL;
+ NativeCallPhaseRet* rets = NULL;
u32 nrets = 0;
/* The emit_ret ops path reads `value` from its home, so it needs the cache
* spilled first (no production arch installs it today). */
@@ -2335,8 +2336,8 @@ static void nd_ret(CgTarget* t, CGLocal value) {
locp = &loc;
}
nd_drop_all(d);
- ND_REQUIRE_NATIVE(d, plan_ret, "target does not plan returns");
- d->native->plan_ret(d->native, d->func, locp, &rets, &nrets);
+ ND_REQUIRE_NATIVE(d, marshal_ret, "target does not marshal returns");
+ d->native->marshal_ret(d->native, d->func, locp, &rets, &nrets);
for (u32 i = 0; i < nrets; ++i)
nd_write_loc(d, rets[i].dst, rets[i].src, rets[i].mem);
ND_REQUIRE_NATIVE(d, ret, "target does not emit returns");
@@ -2576,12 +2577,13 @@ CgTarget* native_direct_target_new(Compiler* c, ObjBuilder* obj,
/* Resolve register/class info once; it is constant for the program. */
d->reg_info = cfg->native ? cfg->native->regs : NULL;
- for (u32 i = 0; i < 3u; ++i) d->class_info[i] = NULL;
+ native_reg_info_validate(c, d->reg_info);
+ for (u32 i = 0; i < NATIVE_REG_CLASS_COUNT; ++i) d->class_info[i] = NULL;
if (d->reg_info) {
const NativeRegInfo* ri = d->reg_info;
for (u32 i = 0; i < ri->nclasses; ++i) {
u32 cls = ri->classes[i].cls;
- if (cls < 3u) d->class_info[cls] = &ri->classes[i];
+ if (cls < NATIVE_REG_CLASS_COUNT) d->class_info[cls] = &ri->classes[i];
}
}
@@ -2593,7 +2595,7 @@ CgTarget* native_direct_target_new(Compiler* c, ObjBuilder* obj,
* (nd_note_reg_used) and reserved by reserve_callee_saves. */
{
int caller_only = d->reg_info && d->reg_info->ndt_caller_saved_only;
- for (u32 cls = 0; cls < 3u; ++cls) {
+ for (u32 cls = 0; cls < NATIVE_REG_CLASS_COUNT; ++cls) {
const NativeAllocClassInfo* ci = d->class_info[cls];
u32 mask, n = 0;
if (!ci) continue;
diff --git a/src/cg/native_direct_target.h b/src/cg/native_direct_target.h
@@ -86,11 +86,11 @@ struct NativeOps {
NativeDirectAddrLegality (*semantic_addr_legal)(NativeDirectTarget*,
Operand addr, MemAccess);
- void (*plan_call)(NativeDirectTarget*, const NativeCallDesc*,
- NativeCallPlan*);
+ void (*marshal_call)(NativeDirectTarget*, const NativeCallDesc*,
+ NativeCallPhase*);
const char* (*tail_call_unrealizable_reason)(NativeDirectTarget*,
const CGCallDesc*);
- void (*emit_call)(NativeDirectTarget*, const NativeCallPlan*);
+ void (*emit_call)(NativeDirectTarget*, const NativeCallPhase*);
/* `value` is the single returned local, or CG_LOCAL_NONE for void. */
void (*emit_ret)(NativeDirectTarget*, CGLocal value);
@@ -135,7 +135,7 @@ struct NativeDirectTarget {
* lookup instead of re-resolving reg_info and linearly scanning ri->classes.
*/
const NativeRegInfo* reg_info;
- const NativeAllocClassInfo* class_info[3];
+ const NativeAllocClassInfo* class_info[NATIVE_REG_CLASS_COUNT];
/* Per-class -O0 value-cache register pool, precomputed at construction in
* `allocable[]` order. On ndt_caller_saved_only targets (aa64) it is the
@@ -149,8 +149,8 @@ struct NativeDirectTarget {
* Note: `allocable[]` is read only by the NDT, never the optimizer (which
* allocates over the phys[] ALLOCABLE flags), so each arch tunes it for -O0
* register caching independently of -O1. */
- Reg cache_pool[3][32];
- u32 ncache_pool[3];
+ Reg cache_pool[NATIVE_REG_CLASS_COUNT][NATIVE_MAX_HARD_REGS];
+ u32 ncache_pool[NATIVE_REG_CLASS_COUNT];
const CGFuncDesc* func;
SrcLoc loc;
@@ -175,16 +175,16 @@ struct NativeDirectTarget {
u32 nscopes;
u32 scopes_cap;
- u32 scratch_used[3];
+ u32 scratch_used[NATIVE_REG_CLASS_COUNT];
/* Per-function callee-saved registers borrowed by direct scratch/cache
* allocation. Reported to the native backend before prologue patching. */
- u32 callee_saved_used[3];
+ u32 callee_saved_used[NATIVE_REG_CLASS_COUNT];
/* Local register cache (write-back, basic-block-scoped). reg_owner[cls][reg]
* names the semantic local currently cached in that physical register, or
* CG_LOCAL_NONE. scratch_used doubles as the per-class "pinned for the
* current instruction" mask. Per-local cache state (reg/cls/dirty) lives on
* NativeDirectLocal. See doc/CODEGEN.md "local register cache". */
- CGLocal reg_owner[3][32];
+ CGLocal reg_owner[NATIVE_REG_CLASS_COUNT][NATIVE_MAX_HARD_REGS];
u32 use_tick; /* monotonic counter stamped onto NativeDirectLocal.last_use */
/* Flat mirror of the currently-owning local's last_use, keyed by physical
* register, so the LRU victim scan reads one contiguous array instead of
@@ -192,7 +192,7 @@ struct NativeDirectTarget {
* whenever an owned register's last_use changes; only read for registers with
* reg_owner[cls][reg] != CG_LOCAL_NONE, where it equals that local's
* last_use. */
- u32 reg_last_use[3][32];
+ u32 reg_last_use[NATIVE_REG_CLASS_COUNT][NATIVE_MAX_HARD_REGS];
/* Head/tail of the intrusive cached-locals list (in caching order), -1 when
* empty; ncached is its length. Lets nd_flush_all run in O(cached) instead of
* scanning all nlocals on every control-flow / barrier op; cache_tail makes
diff --git a/src/opt/ir.h b/src/opt/ir.h
@@ -219,8 +219,7 @@ typedef struct OptCGCallPlan {
u32 nargs;
u32 nrets;
u32 stack_arg_size;
- u32 clobber_mask[3];
- u32 return_mask[3];
+ u32 clobber_mask[NATIVE_REG_CLASS_COUNT];
u16 flags;
u8 has_sret;
u8 is_variadic;
@@ -421,11 +420,11 @@ typedef struct IRBitFieldAux {
BitFieldAccess access;
} IRBitFieldAux;
-#define OPT_REG_CLASSES 3u
-#define OPT_MAX_HARD_REGS 32u
+#define OPT_REG_CLASSES NATIVE_REG_CLASS_COUNT
+#define OPT_MAX_HARD_REGS NATIVE_MAX_HARD_REGS
/* One per-class clobber bitmask set; see Func.inst_clobbers. */
-typedef u32 OptInstClobberMask[OPT_REG_CLASSES];
+typedef NativeRegMaskSet OptInstClobberMask;
#define OPT_MAX_EMIT_TEMPS 4u
typedef struct IRGepAux {
diff --git a/src/opt/pass_lower.c b/src/opt/pass_lower.c
@@ -396,7 +396,7 @@ static void set_preg_pref_for_params(Func* f) {
}
/* Advance the ABI cursors for every part of this param's home, regardless
* of whether we hinted, so subsequent params see the right slot. */
- for (u16 j = 0; j < ai->nparts; ++j) {
+ for (u32 j = 0; j < ai->nparts; ++j) {
u32* c = (ai->parts[j].cls == ABI_CLASS_FP) ? &next_fp : &next_int;
*c += 1u;
}
@@ -407,7 +407,7 @@ static void set_preg_pref_for_params(Func* f) {
* PReg to the matching ABI arg register. Sequential int/fp counters mirror
* the per-class arg slot assignment used by set_preg_pref_for_params. Skips
* variadic, has_sret, and indirect/aggregate args: they need per-target
- * counter logic that hasn't been factored out of plan_call. */
+ * counter logic that hasn't been factored out of marshal_call. */
static void set_preg_pref_for_call_args(Func* f, const CGCallDesc* desc) {
if (!f || !desc) return;
/* Tail calls handle arg routing through the tail-call shuffle in the
@@ -439,7 +439,7 @@ static void set_preg_pref_for_call_args(Func* f, const CGCallDesc* desc) {
if (phys_arg_reg_for_index(f, cls, *counter, &hint))
set_preg_pref_to_arg_reg(f, &a->storage, hint);
}
- for (u16 p = 0; p < ai->nparts; ++p) {
+ for (u32 p = 0; p < ai->nparts; ++p) {
u32* c = (ai->parts[p].cls == ABI_CLASS_FP) ? &next_fp : &next_int;
*c += 1u;
}
diff --git a/src/opt/pass_machinize.c b/src/opt/pass_machinize.c
@@ -113,110 +113,23 @@ static void machinize_prepare_insts(Func* f, NativeTarget* target) {
}
}
-static const NativePhysRegInfo* class_phys_reg(
- const NativeAllocClassInfo* ci, Reg reg) {
- for (u32 i = 0; ci && i < ci->nphys; ++i)
- if (ci->phys[i].reg == reg) return &ci->phys[i];
- return NULL;
-}
-
-static u32 validate_reg_list(Func* f, const NativeAllocClassInfo* ci,
- const Reg* regs, u32 nregs,
- const char* list_name) {
- u32 mask = 0;
- if ((nregs != 0u) != (regs != NULL))
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u has inconsistent %s list",
- (unsigned)ci->cls, list_name);
- if (nregs > OPT_MAX_HARD_REGS)
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u has too many %s registers (%u)",
- (unsigned)ci->cls, list_name, (unsigned)nregs);
- for (u32 i = 0; i < nregs; ++i) {
- Reg reg = regs[i];
- const NativePhysRegInfo* phys;
- if (reg >= OPT_MAX_HARD_REGS || (mask & (1u << reg)))
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u has invalid/duplicate %s reg %u",
- (unsigned)ci->cls, list_name, (unsigned)reg);
- phys = class_phys_reg(ci, reg);
- if (!phys || phys->cls != ci->cls)
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u %s reg %u is not a matching "
- "physical register",
- (unsigned)ci->cls, list_name, (unsigned)reg);
- mask |= 1u << reg;
- }
- return mask;
-}
-
static void collect_class(Func* f, NativeTarget* target,
const NativeAllocClassInfo* ci) {
u32 cls = ci->cls;
- u32 known_mask = 0;
- u32 emit_mask;
- u32 asm_mask;
+ u32 emit_mask = 0;
+ u32 asm_mask = 0;
if (cls >= OPT_REG_CLASSES)
compiler_panic(f->c, (SrcLoc){0, 0, 0},
"opt_machinize: invalid register class %u", (unsigned)cls);
- if ((ci->nphys != 0u) != (ci->phys != NULL) ||
- ci->nphys > OPT_MAX_HARD_REGS)
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u has invalid physical table size %u",
- (unsigned)cls, (unsigned)ci->nphys);
- for (u32 i = 0; i < ci->nphys; ++i) {
- const NativePhysRegInfo* phys = &ci->phys[i];
- if (phys->reg >= OPT_MAX_HARD_REGS || phys->cls != cls ||
- (known_mask & (1u << phys->reg)))
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u has invalid physical reg %u",
- (unsigned)cls, (unsigned)phys->reg);
- known_mask |= 1u << phys->reg;
- }
- (void)validate_reg_list(f, ci, ci->ndt_allocable, ci->ndt_allocable_count,
- "O0 allocable");
- (void)validate_reg_list(f, ci, ci->scratch, ci->nscratch, "O0 scratch");
if (ci->nemit_temps > OPT_MAX_EMIT_TEMPS)
compiler_panic(f->c, (SrcLoc){0, 0, 0},
"opt_machinize: class %u needs %u emit temps (capacity %u)",
(unsigned)cls, (unsigned)ci->nemit_temps,
(unsigned)OPT_MAX_EMIT_TEMPS);
- emit_mask = validate_reg_list(f, ci, ci->emit_temps, ci->nemit_temps,
- "emit temp");
- asm_mask = validate_reg_list(f, ci, ci->asm_temps, ci->nasm_temps,
- "asm-internal temp");
- if (emit_mask & asm_mask)
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u emit and asm temp banks overlap",
- (unsigned)cls);
- if (asm_mask & ~ci->reserved_mask)
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u asm temp bank is not reserved",
- (unsigned)cls);
- if (ci->emit_cache_mask & ~emit_mask)
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u cache mask escapes emit temp bank",
- (unsigned)cls);
- if ((ci->caller_saved_mask | ci->callee_saved_mask | ci->arg_mask |
- ci->ret_mask | ci->reserved_mask) &
- ~known_mask)
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u register mask names unknown phys",
- (unsigned)cls);
- for (u32 i = 0; i < ci->nemit_temps; ++i) {
- const NativePhysRegInfo* phys = class_phys_reg(ci, ci->emit_temps[i]);
- if (phys && (phys->flags & NATIVE_REG_ALLOCABLE))
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u emit temp %u is allocable",
- (unsigned)cls, (unsigned)ci->emit_temps[i]);
- }
- for (u32 i = 0; i < ci->nasm_temps; ++i) {
- const NativePhysRegInfo* phys = class_phys_reg(ci, ci->asm_temps[i]);
- if (phys && (phys->flags & NATIVE_REG_ALLOCABLE))
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: class %u asm temp %u is allocable",
- (unsigned)cls, (unsigned)ci->asm_temps[i]);
- }
+ for (u32 i = 0; i < ci->nemit_temps; ++i)
+ emit_mask |= 1u << ci->emit_temps[i];
+ for (u32 i = 0; i < ci->nasm_temps; ++i)
+ asm_mask |= 1u << ci->asm_temps[i];
f->opt_caller_saved[cls] =
native_target_caller_saved_mask(target, (NativeAllocClass)cls);
f->opt_callee_saved[cls] =
@@ -226,10 +139,13 @@ static void collect_class(Func* f, NativeTarget* target,
"opt_machinize: class %u emit temp bank is not entirely "
"caller-saved",
(unsigned)cls);
- f->opt_reserved_regs[cls] = ci->reserved_mask | emit_mask;
+ /* Static reservations come from register roles. The emitter temp bank is
+ * separately policy-owned, so reserve it for this function here too. */
+ f->opt_reserved_regs[cls] =
+ native_target_reserved_mask(target, (NativeAllocClass)cls) | emit_mask;
f->asm_temp_mask[cls] = asm_mask;
- f->opt_arg_regs[cls] = ci->arg_mask;
- f->opt_ret_regs[cls] = ci->ret_mask;
+ f->opt_arg_regs[cls] = native_target_arg_mask(target, (NativeAllocClass)cls);
+ f->opt_ret_regs[cls] = native_target_ret_mask(target, (NativeAllocClass)cls);
for (u32 i = 0; i < ci->nphys; ++i) {
const NativePhysRegInfo* src = &ci->phys[i];
CGPhysRegInfo* dst = &f->opt_phys_regs[cls][f->opt_phys_reg_count[cls]++];
@@ -250,10 +166,7 @@ static void collect_class(Func* f, NativeTarget* target,
static void machinize_collect_regs(Func* f, NativeTarget* target) {
u32 seen = 0;
if (!target || !target->regs) return;
- if (target->regs->nclasses > OPT_REG_CLASSES ||
- (target->regs->nclasses != 0u) != (target->regs->classes != NULL))
- compiler_panic(f->c, (SrcLoc){0, 0, 0},
- "opt_machinize: invalid register class table");
+ native_reg_info_validate(f->c, target->regs);
for (u32 i = 0; i < target->regs->nclasses; ++i) {
u32 cls = target->regs->classes[i].cls;
if (cls >= OPT_REG_CLASSES || (seen & (1u << cls)))
@@ -326,7 +239,7 @@ void opt_refresh_machine_clobbers(Func* f, NativeTarget* target) {
for (u32 i = 0; i < bl->ninsts; ++i) {
Inst* in = &bl->insts[i];
NativeMachineOp mop;
- u32 mask[NATIVE_CALL_PLAN_CLASSES];
+ u32 mask[NATIVE_REG_CLASS_COUNT];
int has_clobbers;
memset(&mop, 0, sizeof mop);
switch ((IROp)in->op) {
diff --git a/src/opt/pass_native_emit.c b/src/opt/pass_native_emit.c
@@ -1314,7 +1314,7 @@ done:
}
/* Execute one move in a multi-move ABI phase and retain its register
- * destination. The backend call/return planners promise that moves are
+ * destination. The backend call/return marshallers promise that moves are
* source-safe in the order returned; this helper supplies the complementary
* destination-lifetime guarantee for emitter-side materialization. */
static void write_phase_loc(NativeEmitCtx* e, NativeLoc dst, NativeLoc src,
@@ -1649,7 +1649,7 @@ static NativeLoc abi_storage_loc(NativeEmitCtx* e, const OptCGABIValue* v,
static void emit_call(NativeEmitCtx* e, Inst* in) {
IRCallAux* aux = (IRCallAux*)in->extra.aux;
NativeCallDesc d;
- NativeCallPlan plan;
+ NativeCallPhase plan;
NativeLoc* args = NULL;
NativeLoc* results = NULL;
NativeLoc final_result = loc_none();
@@ -1666,9 +1666,9 @@ static void emit_call(NativeEmitCtx* e, Inst* in) {
KitCgTypeId rty = aux->desc.ret.type;
results = arena_zarray(e->f->arena, NativeLoc, 1);
final_result = abi_storage_loc(e, &aux->desc.ret, in->loc);
- /* Hand plan_call the value's real destination directly whenever it is a
+ /* Hand marshal_call the value's real destination directly whenever it is a
* register or a frame slot: a scalar result is a single move out of the ABI
- * result register, and an aggregate / oversized result — which plan_call or
+ * result register, and an aggregate / oversized result — which marshal_call or
* the callee writes in parts and so must land in memory — lands straight in
* its frame home. Routing either through a fresh temp slot (store then
* reload / copy_bytes) was a pure round trip on every call. The temp slot
@@ -1693,16 +1693,16 @@ static void emit_call(NativeEmitCtx* e, Inst* in) {
d.flags = aux->desc.flags;
d.tail_policy = aux->desc.tail_policy;
d.inline_policy = aux->desc.inline_policy;
- /* Materialize a frame-resident indirect callee before plan_call starts its
+ /* Materialize a frame-resident indirect callee before marshal_call starts its
* argument phase. Real backends marshal register arguments inside
- * plan_call; doing this afterward allowed the emitter to reuse an already
+ * marshal_call; doing this afterward allowed the emitter to reuse an already
* populated ABI argument register for the callee. The acquired lease also
- * keeps generic plan.args moves from selecting the callee register. */
+ * keeps generic phase argument moves from selecting the callee register. */
if (d.callee.kind != NATIVE_LOC_REG &&
d.callee.kind != NATIVE_LOC_GLOBAL)
d.callee = materialize(e, d.callee, NATIVE_REG_INT, d.callee.type,
REG_NONE, REG_NONE, in->loc);
- e->target->plan_call(e->target, &d, &plan);
+ e->target->marshal_call(e->target, &d, &plan);
for (u32 i = 0; i < plan.nargs; ++i)
write_phase_loc(e, plan.args[i].dst, plan.args[i].src, plan.args[i].mem,
in->loc);
@@ -1720,7 +1720,7 @@ static void emit_call(NativeEmitCtx* e, Inst* in) {
type_size_or(e->c, rty, 8u) > e->c->target.ptr_size)) {
/* Aggregate / oversized result: move bytes rather than a scalar copy
* (which would exceed the single-register width). The result was either
- * written in parts by plan_call's rets, or by the callee via the sret
+ * written in parts by marshal_call's rets, or by the callee via the sret
* pointer; either way it now lives in the temp slot. */
AggregateAccess acc;
NativeAddr da = addr_from_loc(e, final_result, in->loc);
@@ -1741,19 +1741,19 @@ static void emit_ret(NativeEmitCtx* e, Inst* in, const CGFuncDesc* fd) {
IRRetAux* aux = (IRRetAux*)in->extra.aux;
NativeLoc value = loc_none();
const NativeLoc* values = NULL;
- NativeCallPlanRet* rets = NULL;
+ NativeCallPhaseRet* rets = NULL;
u32 nrets = 0;
if (aux && aux->present) {
- /* Hand plan_ret the value's location directly. For an aggregate / oversized
- * result it is a memory location (plan_ret copies to the sret pointer or
+ /* Hand marshal_ret the value's location directly. For an aggregate / oversized
+ * result it is a memory location (marshal_ret copies to the sret pointer or
* reads parts into the return registers); for a scalar it is the value's
- * register or slot, which plan_ret moves into the return register. The old
+ * register or slot, which marshal_ret moves into the return register. The old
* code spilled scalars to a fresh slot and reloaded them, a pure round
* trip on every return. */
value = abi_storage_loc(e, &aux->val, in->loc);
values = &value;
}
- e->target->plan_ret(e->target, fd, values, &rets, &nrets);
+ e->target->marshal_ret(e->target, fd, values, &rets, &nrets);
for (u32 i = 0; i < nrets; ++i)
write_phase_loc(e, rets[i].dst, rets[i].src, rets[i].mem, in->loc);
/* Skip the trailing branch-to-epilogue when this IR_RET is the very last
@@ -2567,7 +2567,7 @@ static void plan_one_asm_staging(NativeEmitCtx* e, Inst* in, IRAsmAux* aux) {
temp_scope_end(e, &scope);
}
-static NativeLoc* plan_call_args_reserve(NativeEmitCtx* e, NativeLoc* args,
+static NativeLoc* marshal_call_args_reserve(NativeEmitCtx* e, NativeLoc* args,
u32* cap, u32 want) {
u32 grown;
if (want <= *cap) return args;
@@ -2643,7 +2643,7 @@ static void plan_frame(NativeEmitCtx* e, const CGFuncDesc* fd) {
if (aux && t->call_stack_bytes) {
NativeCallDesc d;
u32 sb;
- call_args = plan_call_args_reserve(
+ call_args = marshal_call_args_reserve(
e, call_args, &call_args_cap, aux->desc.nargs);
memset(&d, 0, sizeof d);
d.fn_type = aux->desc.fn_type;
diff --git a/test/api/abi_classify_test.c b/test/api/abi_classify_test.c
@@ -32,7 +32,8 @@ static void expect_direct_1x_int(const char* tag, const ABIArgInfo* ai,
static KitTargetSpec test_target_spec(KitArchKind arch, KitOSKind os,
KitObjFmt obj) {
KitTargetSpec t = kit_unit_target(arch, os, obj);
- if (arch == KIT_ARCH_RV32 || arch == KIT_ARCH_WASM) {
+ if (arch == KIT_ARCH_ARM_32 || arch == KIT_ARCH_RV32 ||
+ arch == KIT_ARCH_WASM) {
t.ptr_size = 4;
t.ptr_align = 4;
}
@@ -80,6 +81,8 @@ static const char* arch_name(KitArchKind a) {
return "x64";
case KIT_ARCH_ARM_64:
return "aarch64";
+ case KIT_ARCH_ARM_32:
+ return "arm32";
case KIT_ARCH_RV32:
return "rv32";
case KIT_ARCH_RV64:
@@ -191,6 +194,155 @@ static KitCgTypeId record2(KitCompiler* c, KitCgTypeId a, KitCgTypeId b) {
return kit_cg_type_record(c, &desc);
}
+static KitCgTypeId record_i32s(KitCompiler* c, u32 n) {
+ KitCgFieldDesc fields[8];
+ KitCgRecordDesc desc;
+ KitCgTypeId i32 = builtin(c, KIT_CG_BUILTIN_I32);
+ if (n == 0u || n > sizeof fields / sizeof fields[0]) return KIT_CG_TYPE_NONE;
+ memset(fields, 0, sizeof fields);
+ for (u32 i = 0; i < n; ++i) fields[i].type = i32;
+ memset(&desc, 0, sizeof desc);
+ desc.fields = fields;
+ desc.nfields = n;
+ return kit_cg_type_record(c, &desc);
+}
+
+static void expect_arm32_word_parts(const char* tag, const ABIArgInfo* ai,
+ u32 nwords) {
+ EXPECT(ai->kind == ABI_ARG_DIRECT, "%s: kind=%d want DIRECT", tag,
+ (int)ai->kind);
+ EXPECT(ai->nparts == nwords, "%s: nparts=%u want %u", tag,
+ (unsigned)ai->nparts, (unsigned)nwords);
+ if (ai->nparts != nwords || !ai->parts) return;
+ for (u32 i = 0; i < nwords; ++i) {
+ EXPECT(ai->parts[i].cls == ABI_CLASS_INT,
+ "%s: parts[%u].cls=%d want INT", tag, (unsigned)i,
+ (int)ai->parts[i].cls);
+ EXPECT(ai->parts[i].size == 4u,
+ "%s: parts[%u].size=%u want 4", tag, (unsigned)i,
+ (unsigned)ai->parts[i].size);
+ EXPECT(ai->parts[i].src_offset == i * 4u,
+ "%s: parts[%u].src_offset=%u want %u", tag, (unsigned)i,
+ (unsigned)ai->parts[i].src_offset, (unsigned)(i * 4u));
+ }
+}
+
+static void test_aapcs32_specifics(void) {
+ KitCompiler* c =
+ new_compiler(KIT_ARCH_ARM_32, KIT_OS_FREESTANDING, KIT_OBJ_ELF);
+ static const u32 words[] = {1u, 2u, 4u, 8u};
+ char tag[64];
+
+ for (u32 i = 0; i < sizeof words / sizeof words[0]; ++i) {
+ KitCgTypeId rec = record_i32s(c, words[i]);
+ const ABIFuncInfo* fi = classify_fn(c, rec, rec);
+ snprintf(tag, sizeof tag, "arm32 record%u arg", (unsigned)(words[i] * 4u));
+ expect_arm32_word_parts(tag, &fi->params[0], words[i]);
+ snprintf(tag, sizeof tag, "arm32 record%u ret", (unsigned)(words[i] * 4u));
+ if (words[i] == 1u) {
+ expect_arm32_word_parts(tag, &fi->ret, 1u);
+ EXPECT(fi->has_sret == 0, "%s: one-word result should not use sret", tag);
+ EXPECT(fi->sret_consumes_int_arg == 0,
+ "%s: one-word result should not consume an argument register",
+ tag);
+ } else {
+ EXPECT(fi->ret.kind == ABI_ARG_INDIRECT,
+ "%s: kind=%d want INDIRECT", tag, (int)fi->ret.kind);
+ EXPECT(fi->ret.nparts == 0, "%s: nparts=%u want 0", tag,
+ (unsigned)fi->ret.nparts);
+ EXPECT((fi->ret.flags & ABI_AF_SRET) != 0,
+ "%s: flags=0x%x missing SRET", tag, (unsigned)fi->ret.flags);
+ EXPECT(fi->ret.indirect_align == 4u,
+ "%s: indirect_align=%u want 4", tag,
+ (unsigned)fi->ret.indirect_align);
+ EXPECT(fi->has_sret == 1, "%s: composite result should use sret", tag);
+ EXPECT(fi->sret_consumes_int_arg == 1,
+ "%s: sret pointer should consume r0", tag);
+ }
+ }
+
+ {
+ KitCgFieldDesc fields[3];
+ KitCgRecordDesc desc;
+ KitCgTypeId i8 = builtin(c, KIT_CG_BUILTIN_I8);
+ const ABIFuncInfo* fi;
+ memset(fields, 0, sizeof fields);
+ for (u32 i = 0; i < 3u; ++i) fields[i].type = i8;
+ memset(&desc, 0, sizeof desc);
+ desc.fields = fields;
+ desc.nfields = 3u;
+ fi = classify_fn(c, kit_cg_type_record(c, &desc),
+ kit_cg_type_record(c, &desc));
+ EXPECT(fi->params[0].kind == ABI_ARG_DIRECT,
+ "arm32 record3 arg: want DIRECT");
+ EXPECT(fi->params[0].nparts == 1u,
+ "arm32 record3 arg: nparts=%u want 1",
+ (unsigned)fi->params[0].nparts);
+ EXPECT(fi->params[0].parts && fi->params[0].parts[0].size == 3u,
+ "arm32 record3 arg: tail size should be exact");
+ EXPECT(fi->ret.kind == ABI_ARG_DIRECT,
+ "arm32 record3 ret: want DIRECT");
+ EXPECT(fi->ret.nparts == 1u,
+ "arm32 record3 ret: nparts=%u want 1",
+ (unsigned)fi->ret.nparts);
+ EXPECT(fi->ret.parts && fi->ret.parts[0].size == 3u,
+ "arm32 record3 ret: tail size should be exact");
+ EXPECT(fi->has_sret == 0,
+ "arm32 record3 ret: one-word result should not use sret");
+ }
+
+ {
+ KitCgTypeId i64 = builtin(c, KIT_CG_BUILTIN_I64);
+ KitCgTypeId rec = record2(c, i64, i64);
+ const ABIFuncInfo* fi = classify_fn(c, rec, rec);
+ expect_arm32_word_parts("arm32 aligned record16 arg", &fi->params[0],
+ 4u);
+ EXPECT(fi->params[0].parts && fi->params[0].parts[0].align == 8u,
+ "arm32 aligned record16 arg: first-part align=%u want 8",
+ fi->params[0].parts
+ ? (unsigned)fi->params[0].parts[0].align
+ : 0u);
+ EXPECT(fi->ret.kind == ABI_ARG_INDIRECT,
+ "arm32 aligned record16 ret: want INDIRECT");
+ EXPECT(fi->ret.indirect_align == 8u,
+ "arm32 aligned record16 ret: indirect_align=%u want 8",
+ (unsigned)fi->ret.indirect_align);
+ EXPECT((fi->ret.flags & ABI_AF_SRET) != 0,
+ "arm32 aligned record16 ret: missing SRET");
+ }
+
+ {
+ static const u32 nwords = 65537u;
+ KitCgFieldDesc field;
+ KitCgRecordDesc desc;
+ KitCgTypeId i32 = builtin(c, KIT_CG_BUILTIN_I32);
+ KitCgTypeId array = kit_cg_type_array(c, i32, nwords);
+ const ABIFuncInfo* fi;
+ memset(&field, 0, sizeof field);
+ field.type = array;
+ memset(&desc, 0, sizeof desc);
+ desc.fields = &field;
+ desc.nfields = 1u;
+ fi = classify_fn(c, kit_cg_type_record(c, &desc),
+ kit_cg_type_record(c, &desc));
+ EXPECT(fi->params[0].kind == ABI_ARG_DIRECT,
+ "arm32 huge record arg: want DIRECT");
+ EXPECT(fi->params[0].nparts == nwords,
+ "arm32 huge record arg: nparts=%u want %u",
+ (unsigned)fi->params[0].nparts, (unsigned)nwords);
+ EXPECT(fi->params[0].parts &&
+ fi->params[0].parts[nwords - 1u].src_offset ==
+ (nwords - 1u) * 4u &&
+ fi->params[0].parts[nwords - 1u].size == 4u,
+ "arm32 huge record arg: last carrier lane was truncated");
+ EXPECT(fi->ret.kind == ABI_ARG_INDIRECT &&
+ (fi->ret.flags & ABI_AF_SRET) != 0,
+ "arm32 huge record ret: want INDIRECT SRET");
+ }
+
+ kit_compiler_free(c);
+}
+
static void check_target(KitArchKind arch, KitOSKind os, KitObjFmt obj) {
KitCompiler* c = new_compiler(arch, os, obj);
KitCgTypeId i128_ty = builtin(c, KIT_CG_BUILTIN_I128);
@@ -622,6 +774,8 @@ static void test_scalar_split_lane_size(void) {
* integer-register pairs regardless of the float ABI. */
check_scalar_split_lane_target(KIT_ARCH_RV32, KIT_OS_LINUX, KIT_OBJ_ELF,
"rv32", 4, 4);
+ check_scalar_split_lane_target(KIT_ARCH_ARM_32, KIT_OS_FREESTANDING,
+ KIT_OBJ_ELF, "arm32", 4, 4);
check_scalar_split_lane_target(KIT_ARCH_RV64, KIT_OS_LINUX, KIT_OBJ_ELF,
"rv64", 0, 0);
check_scalar_split_lane_target(KIT_ARCH_WASM, KIT_OS_WASI, KIT_OBJ_WASM,
@@ -639,6 +793,7 @@ int main(void) {
test_win64_specifics();
test_aarch64_windows_variadic();
test_apple_arm64_stack_traits();
+ test_aapcs32_specifics();
test_scalar_split_lane_size();
kit_unit_summary(&g_u, "abi_classify_test");
return kit_unit_status(&g_u);
diff --git a/test/arch/arm32_inline_test.c b/test/arch/arm32_inline_test.c
@@ -24,6 +24,20 @@ static void arm32_bad_operand(KitCompiler* c, KitCg* cg, KitCgTypeId i64_ty) {
it_inline_asm(c, cg, "mov %9, r0", NULL, 0, NULL, 0, NULL, 0);
}
+static void arm32_named_r4_clobber(KitCompiler* c, KitCg* cg,
+ KitCgTypeId i64_ty) {
+ static const char* const clobbers[] = {"r4"};
+ (void)i64_ty;
+ it_inline_asm(c, cg, "movs r4, #99", NULL, 0, NULL, 0, clobbers, 1);
+}
+
+static void arm32_bad_frame_clobber(KitCompiler* c, KitCg* cg,
+ KitCgTypeId i64_ty) {
+ static const char* const clobbers[] = {"r7"};
+ (void)i64_ty;
+ it_inline_asm(c, cg, "", NULL, 0, NULL, 0, clobbers, 1);
+}
+
/* r0-r3 are the direct binder's caller-saved operand pool. Five simultaneous
* generic inputs therefore put the fifth value in callee-saved r4. */
static void arm32_generic_callee_saved_operand(KitCompiler* c, KitCg* cg,
@@ -60,6 +74,49 @@ static int arm32_is_r4_mem(const KitInsn* insn, const char* mnemonic) {
insn->operands.s[1] == '4';
}
+static int arm32_is_r4_reglist(const KitInsn* insn, const char* mnemonic) {
+ size_t n = strlen(mnemonic);
+ if (!insn->mnemonic.s || insn->mnemonic.len < n ||
+ memcmp(insn->mnemonic.s, mnemonic, n) != 0 || !insn->operands.s)
+ return 0;
+ for (size_t i = 0; i + 1u < insn->operands.len; ++i)
+ if (insn->operands.s[i] == 'r' && insn->operands.s[i + 1u] == '4')
+ return 1;
+ return 0;
+}
+
+static int arm32_has_known_frame_r4_preservation(InlineTestEnv* env,
+ const InlineText* text) {
+ KitTargetOptions opts;
+ KitTarget* target = NULL;
+ KitDisasmContext dc;
+ KitDisasmIter* it = NULL;
+ KitInsn insn;
+ int saw_push = 0;
+ int saw_pop = 0;
+
+ memset(&opts, 0, sizeof opts);
+ opts.spec = it_target(KIT_ARCH_ARM_32);
+ if (kit_target_new(&env->ctx, &opts, &target) != KIT_OK || !target)
+ goto done;
+ memset(&dc, 0, sizeof dc);
+ dc.target = target;
+ dc.context = env->ctx;
+ if (kit_disasm_iter_new(&dc, text->data, text->len, 0, text->file, &it) !=
+ KIT_OK ||
+ !it)
+ goto done;
+ while (kit_disasm_iter_next(it, &insn) == KIT_ITER_ITEM) {
+ if (arm32_is_r4_reglist(&insn, "push")) saw_push = 1;
+ if (arm32_is_r4_reglist(&insn, "pop")) saw_pop = 1;
+ }
+
+done:
+ if (it) kit_disasm_iter_free(it);
+ kit_target_free(target);
+ return saw_push && saw_pop;
+}
+
static int arm32_has_generic_r4_preservation(InlineTestEnv* env,
const InlineText* text) {
KitTargetOptions opts;
@@ -272,6 +329,12 @@ int main(void) {
it_expect_panic(&env, KIT_ARCH_ARM_32, "arm32_bad_operand",
arm32_bad_operand, "operand index"),
"expected out-of-range arm32 asm operand to panic");
+ IT_EXPECT(&env,
+ it_expect_panic(&env, KIT_ARCH_ARM_32,
+ "arm32_bad_frame_clobber",
+ arm32_bad_frame_clobber,
+ "structural register cannot be clobbered"),
+ "expected arm32 r7 clobber to panic");
IT_EXPECT(&env, arm32_check_thumb_func(&env),
"arm32 .thumb_func symbol regression failed: %s", env.last_diag);
@@ -286,6 +349,16 @@ int main(void) {
"generic r4 asm operand was not saved and restored");
it_text_close(&text);
+ IT_EXPECT(&env,
+ it_emit_text_opt(&env, KIT_ARCH_ARM_32,
+ "arm32_named_r4_clobber_o1",
+ arm32_named_r4_clobber, 1, &text),
+ "failed to emit O1 named r4 clobber: %s", env.last_diag);
+ if (text.data)
+ IT_EXPECT(&env, arm32_has_known_frame_r4_preservation(&env, &text),
+ "O1 known frame did not preserve named r4 clobber");
+ it_text_close(&text);
+
if (env.fails) {
fprintf(stderr, "%d failure(s)\n", env.fails);
return 1;
diff --git a/test/cg/native_direct_target_test.c b/test/cg/native_direct_target_test.c
@@ -7,6 +7,7 @@
#include <string.h>
#include "core/arena.h"
+#include "cg/native_asm.h"
#include "lib/kit_unit.h"
/* Shared test context replaces the per-file heap/diag/counter globals;
@@ -53,9 +54,9 @@ typedef enum MockEventKind {
EV_LOAD_IMM,
EV_BINOP,
EV_BARRIER,
- EV_PLAN_CALL,
+ EV_MARSHAL_CALL,
EV_EMIT_CALL,
- EV_PLAN_RET,
+ EV_MARSHAL_RET,
EV_MOVE,
EV_RET,
} MockEventKind;
@@ -82,23 +83,51 @@ static const Reg mock_fp_scratch[] = {4u, 5u};
static const Reg mock_ndt_int_allocable[] = {1u, 2u, 3u, 6u};
static const Reg mock_ndt_fp_allocable[] = {4u, 5u, 6u};
+#define MOCK_PHYS(cls_, reg_, flags_) \
+ {.reg = (reg_), \
+ .cls = (cls_), \
+ .abi_index = 0xffu, \
+ .flags = (flags_), \
+ .spill_cost = 1u, \
+ .copy_cost = 1u}
+
+static const NativePhysRegInfo mock_int_phys[] = {
+ MOCK_PHYS(NATIVE_REG_INT, 0u,
+ NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | NATIVE_REG_RET),
+ MOCK_PHYS(NATIVE_REG_INT, 1u,
+ NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED),
+ MOCK_PHYS(NATIVE_REG_INT, 2u,
+ NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED),
+ MOCK_PHYS(NATIVE_REG_INT, 3u,
+ NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED),
+ MOCK_PHYS(NATIVE_REG_INT, 6u, NATIVE_REG_ALLOCABLE),
+};
+
+static const NativePhysRegInfo mock_fp_phys[] = {
+ MOCK_PHYS(NATIVE_REG_FP, 0u,
+ NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | NATIVE_REG_RET),
+ MOCK_PHYS(NATIVE_REG_FP, 4u,
+ NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED),
+ MOCK_PHYS(NATIVE_REG_FP, 5u,
+ NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED),
+ MOCK_PHYS(NATIVE_REG_FP, 6u, NATIVE_REG_ALLOCABLE),
+};
+
static const NativeAllocClassInfo mock_classes[] = {
{.cls = NATIVE_REG_INT,
.ndt_allocable = mock_ndt_int_allocable,
.ndt_allocable_count = 4,
.scratch = mock_int_scratch,
.nscratch = 3,
- .caller_saved_mask = (1u << 1) | (1u << 2) | (1u << 3),
- .arg_mask = 1u << 0,
- .ret_mask = 1u << 0},
+ .phys = mock_int_phys,
+ .nphys = sizeof mock_int_phys / sizeof mock_int_phys[0]},
{.cls = NATIVE_REG_FP,
.ndt_allocable = mock_ndt_fp_allocable,
.ndt_allocable_count = 3,
.scratch = mock_fp_scratch,
.nscratch = 2,
- .caller_saved_mask = (1u << 4) | (1u << 5),
- .arg_mask = 1u << 0,
- .ret_mask = 1u << 0},
+ .phys = mock_fp_phys,
+ .nphys = sizeof mock_fp_phys / sizeof mock_fp_phys[0]},
};
static const NativeRegInfo mock_reg_info = {
@@ -106,6 +135,48 @@ static const NativeRegInfo mock_reg_info = {
.nclasses = sizeof mock_classes / sizeof mock_classes[0],
};
+static int reordered_constraint_reg(const NativeRegInfo* ri, const char* body,
+ NativeAllocClass* cls_out, Reg* fixed_out,
+ u32* allowed_mask_out) {
+ (void)ri;
+ if (strcmp(body, "x") != 0) return 0;
+ *cls_out = NATIVE_REG_FP;
+ *fixed_out = 4u;
+ *allowed_mask_out = 1u << 4u;
+ return 1;
+}
+
+static void test_register_class_lookup_is_not_positional(void) {
+ NativePhysRegInfo int_phys =
+ MOCK_PHYS(NATIVE_REG_INT, 1u, NATIVE_REG_ALLOCABLE);
+ NativePhysRegInfo fp_phys =
+ MOCK_PHYS(NATIVE_REG_FP, 4u, NATIVE_REG_ALLOCABLE);
+ NativeAllocClassInfo classes[2];
+ NativeRegInfo regs;
+ NativeTarget target;
+ NativeAsmConstraintInfo info;
+ memset(classes, 0, sizeof classes);
+ classes[0].cls = NATIVE_REG_FP;
+ classes[0].phys = &fp_phys;
+ classes[0].nphys = 1u;
+ classes[1].cls = NATIVE_REG_INT;
+ classes[1].phys = &int_phys;
+ classes[1].nphys = 1u;
+ memset(®s, 0, sizeof regs);
+ regs.classes = classes;
+ regs.nclasses = 2u;
+ regs.asm_constraint_reg = reordered_constraint_reg;
+ memset(&target, 0, sizeof target);
+ target.regs = ®s;
+
+ EXPECT(native_reg_info_class_info(®s, NATIVE_REG_FP) == &classes[0] &&
+ native_reg_info_class_info(®s, NATIVE_REG_INT) == &classes[1],
+ "register class lookup depends on table order");
+ EXPECT(native_asm_constraint_reg_info(&target, "x", &info) &&
+ info.cls == NATIVE_REG_FP && info.fixed_reg == 4u,
+ "inline-asm fallback used a positional register class");
+}
+
static MockNative* mock_of(NativeTarget* t) { return (MockNative*)t; }
static void ev(MockNative* m, MockEventKind kind, u32 a, u32 b, u32 c) {
@@ -193,18 +264,18 @@ static void mock_move_rr(NativeTarget* t, NativeRegLoc dst, NativeRegLoc src) {
ev(mock_of(t), EV_MOVE, dst.v.reg, src.v.reg, dst.type);
}
-static void mock_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
+static void mock_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
EXPECT(plan->callee.kind == NATIVE_LOC_REG,
"frame callee should be materialized for call");
ev(mock_of(t), EV_EMIT_CALL, plan->nargs, plan->nrets, plan->callee.v.reg);
}
-static void mock_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
- const NativeLoc* value, NativeCallPlanRet** out_rets,
- u32* out_nrets) {
- NativeCallPlanRet* r;
+static void mock_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
+ const NativeLoc* value,
+ NativeCallPhaseRet** out_rets, u32* out_nrets) {
+ NativeCallPhaseRet* r;
(void)fd;
- r = arena_zarray(t->c->tu, NativeCallPlanRet, 1);
+ r = arena_zarray(t->c->tu, NativeCallPhaseRet, 1);
if (value) {
r[0].src = *value;
r[0].dst.kind = NATIVE_LOC_REG;
@@ -217,7 +288,7 @@ static void mock_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
}
*out_rets = r;
*out_nrets = value ? 1u : 0u;
- ev(mock_of(t), EV_PLAN_RET, 0, value ? 1u : 0u, 0);
+ ev(mock_of(t), EV_MARSHAL_RET, 0, value ? 1u : 0u, 0);
}
static void mock_ret(NativeTarget* t) { ev(mock_of(t), EV_RET, 0, 0, 0); }
@@ -240,7 +311,7 @@ static void mock_native_init(MockNative* m, Compiler* c) {
m->base.binop = mock_binop;
m->base.move_rr = mock_move_rr;
m->base.emit_call = mock_emit_call;
- m->base.plan_ret = mock_plan_ret;
+ m->base.marshal_ret = mock_marshal_ret;
m->base.ret = mock_ret;
}
@@ -250,13 +321,13 @@ static void mock_barrier(NativeDirectTarget* d, u32 flags) {
ev(m, EV_BARRIER, 0, 0, flags);
}
-static void mock_plan_call(NativeDirectTarget* d, const NativeCallDesc* desc,
- NativeCallPlan* plan) {
+static void mock_marshal_call(NativeDirectTarget* d, const NativeCallDesc* desc,
+ NativeCallPhase* plan) {
MockNative* m = (MockNative*)d->native;
- NativeCallPlanMove* args =
- arena_zarray(d->base.c->tu, NativeCallPlanMove, desc->nargs);
- NativeCallPlanRet* rets =
- arena_zarray(d->base.c->tu, NativeCallPlanRet, desc->nresults);
+ NativeCallPhaseMove* args =
+ arena_zarray(d->base.c->tu, NativeCallPhaseMove, desc->nargs);
+ NativeCallPhaseRet* rets =
+ arena_zarray(d->base.c->tu, NativeCallPhaseRet, desc->nresults);
memset(plan, 0, sizeof *plan);
plan->callee = desc->callee;
plan->args = args;
@@ -285,11 +356,11 @@ static void mock_plan_call(NativeDirectTarget* d, const NativeCallDesc* desc,
rets[i].mem.align = 4;
}
m->last_stack_arg_size = plan->stack_arg_size;
- ev(m, EV_PLAN_CALL, desc->nargs, desc->nresults, plan->stack_arg_size);
+ ev(m, EV_MARSHAL_CALL, desc->nargs, desc->nresults, plan->stack_arg_size);
}
static const NativeOps mock_ops = {
- .plan_call = mock_plan_call,
+ .marshal_call = mock_marshal_call,
.barrier = mock_barrier,
};
@@ -432,7 +503,8 @@ static void test_frame_locals_scratch_storeback_and_branches(void) {
EXPECT(count_event(&native, EV_CMP_BRANCH) == 1,
"expected one compare branch");
EXPECT(count_event(&native, EV_JUMP) == 1, "expected one jump");
- EXPECT(count_event(&native, EV_PLAN_RET) == 1 && count_event(&native, EV_RET),
+ EXPECT(count_event(&native, EV_MARSHAL_RET) == 1 &&
+ count_event(&native, EV_RET),
"return should plan moves and emit ret");
tc_fini(&tc);
}
@@ -470,8 +542,9 @@ static void test_call_barrier_storeback_and_max_outgoing(void) {
(NATIVE_DIRECT_BARRIER_CALL | NATIVE_DIRECT_BARRIER_MEMORY),
"call should request call+memory barrier");
EXPECT(native.last_stack_arg_size == 24 && nd->max_outgoing == 24,
- "call planning should track max outgoing stack size");
- EXPECT(count_event(&native, EV_PLAN_CALL) == 1, "expected one call plan");
+ "call marshalling should track max outgoing stack size");
+ EXPECT(count_event(&native, EV_MARSHAL_CALL) == 1,
+ "expected one call marshalling phase");
EXPECT(count_event(&native, EV_EMIT_CALL) == 1, "expected one emitted call");
EXPECT(count_event(&native, EV_STORE) == 2,
"selective call setup should emit only its required stores (stores=%d)",
@@ -618,6 +691,7 @@ static void test_b_call_still_flushes(void) {
int main(void) {
kit_unit_init(&g_u);
g_u.ctx.now = -1;
+ test_register_class_lookup_is_not_positional();
test_frame_locals_scratch_storeback_and_branches();
test_call_barrier_storeback_and_max_outgoing();
test_b_cached_pointer_base_not_reloaded();
diff --git a/test/opt/frame_value_backend_test.c b/test/opt/frame_value_backend_test.c
@@ -241,7 +241,7 @@ static void check_typed_frame_address(KitUnit *unit, const BackendCase *tc) {
NativeLoc call_arg;
NativeLoc exact_arg;
NativeCallDesc call_desc;
- NativeCallPlan call_plan;
+ NativeCallPhase call_plan;
DecodedInsn body[12];
DecodedInsn call_body[8];
DecodedInsn global_body[12];
@@ -385,7 +385,7 @@ static void check_typed_frame_address(KitUnit *unit, const BackendCase *tc) {
call_desc.nargs = 1u;
memset(&call_plan, 0, sizeof call_plan);
call_begin = mc_pos(mc);
- native->plan_call(native, &call_desc, &call_plan);
+ native->marshal_call(native, &call_desc, &call_plan);
call_end = mc_pos(mc);
/* A GLOBAL NativeLoc is the scalar address value `&symbol + addend`, never
@@ -404,7 +404,7 @@ static void check_typed_frame_address(KitUnit *unit, const BackendCase *tc) {
call_desc.nargs = 1u;
memset(&call_plan, 0, sizeof call_plan);
global_begin = mc_pos(mc);
- native->plan_call(native, &call_desc, &call_plan);
+ native->marshal_call(native, &call_desc, &call_plan);
global_end = mc_pos(mc);
/* ARM32 splits an i64 direct argument into low/high 32-bit ABI lanes. A
@@ -422,7 +422,7 @@ static void check_typed_frame_address(KitUnit *unit, const BackendCase *tc) {
call_desc.nargs = 1u;
memset(&call_plan, 0, sizeof call_plan);
imm_begin = mc_pos(mc);
- native->plan_call(native, &call_desc, &call_plan);
+ native->marshal_call(native, &call_desc, &call_plan);
imm_end = mc_pos(mc);
}
@@ -448,7 +448,7 @@ static void check_typed_frame_address(KitUnit *unit, const BackendCase *tc) {
call_desc.nargs = 1u;
memset(&call_plan, 0, sizeof call_plan);
exact_call_begin = mc_pos(mc);
- native->plan_call(native, &call_desc, &call_plan);
+ native->marshal_call(native, &call_desc, &call_plan);
native->emit_call(native, &call_plan);
exact_call_end = mc_pos(mc);
}
@@ -479,7 +479,7 @@ static void check_typed_frame_address(KitUnit *unit, const BackendCase *tc) {
call_desc.nargs = 1u;
memset(&call_plan, 0, sizeof call_plan);
forced_vararg_begin = mc_pos(mc);
- native->plan_call(native, &call_desc, &call_plan);
+ native->marshal_call(native, &call_desc, &call_plan);
native->emit_call(native, &call_plan);
forced_vararg_end = mc_pos(mc);
synthetic->vararg_on_stack = saved_vararg_on_stack;
diff --git a/test/opt/location_mir_test.c b/test/opt/location_mir_test.c
@@ -157,7 +157,6 @@ static void machinize_rejects_cross_owner_registers(KitUnit* u) {
cls.nemit_temps = 1u;
cls.phys = phys;
cls.nphys = 2u;
- cls.caller_saved_mask = (1u << emit_temp) | (1u << asm_temp);
memset(®s, 0, sizeof regs);
regs.classes = &cls;
regs.nclasses = 1u;
@@ -173,13 +172,16 @@ static void machinize_rejects_cross_owner_registers(KitUnit* u) {
"machinize accepted a register shared by allocation and the "
"emitter-temp owner");
- /* The asm bank has a separate lifetime and must be explicitly reserved.
- * Exercise that ownership rule independently after making the emit bank
- * valid, without trying to mirror every table-shape validation branch. */
- phys[0].flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_RESERVED;
+ phys[0].flags = NATIVE_REG_CALLER_SAVED;
cls.asm_temps = &asm_temp;
cls.nasm_temps = 1u;
- cls.reserved_mask = 1u << emit_temp;
+ opt_machinize_native(f, &target);
+ CU_EXPECT(u,
+ (f->opt_reserved_regs[NATIVE_REG_INT] & (1u << emit_temp)) != 0,
+ "machinize did not reserve the policy-owned emitter temp");
+
+ /* The asm bank has a separate lifetime and must be statically reserved. */
+ phys[1].flags = NATIVE_REG_CALLER_SAVED;
CU_EXPECT(u,
expect_compiler_panic(u, c, run_machinize, &call,
"asm temp bank is not reserved"),
diff --git a/test/opt/native_emit_frame_dst_test.c b/test/opt/native_emit_frame_dst_test.c
@@ -14,12 +14,13 @@ typedef struct MockNative {
NativeTarget base;
KitUnit *unit;
NativeAllocClassInfo classes[2];
+ NativePhysRegInfo int_phys[1];
NativeRegInfo regs;
Reg int_temps[4];
- NativeCallPlanMove call_arg;
- NativeCallPlanRet call_ret;
- NativeCallPlanMove phase_call_args[2];
- NativeCallPlanRet ret_parts[2];
+ NativeCallPhaseMove call_arg;
+ NativeCallPhaseRet call_ret;
+ NativeCallPhaseMove phase_call_args[2];
+ NativeCallPhaseRet ret_parts[4];
u32 callback_mask;
u32 writeback_slots;
u32 load_imms;
@@ -42,7 +43,7 @@ typedef struct MockNative {
Reg machine_clobber_reg;
u8 report_bitfield_store_clobber;
u8 plan_two_arg_indirect_call;
- u8 plan_two_part_ret;
+ u8 plan_four_part_ret;
u8 reject_indexed_addr;
MCLabel next_label;
} MockNative;
@@ -231,7 +232,7 @@ static void mock_bitfield_store(NativeTarget *target, NativeAddr addr,
static int mock_machine_op_clobbers(
NativeTarget *target, const NativeMachineOp *op,
- u32 mask[NATIVE_CALL_PLAN_CLASSES]) {
+ u32 mask[NATIVE_REG_CLASS_COUNT]) {
MockNative *mock = mock_of(target);
mask[0] = mask[1] = mask[2] = 0u;
if (!mock->report_bitfield_store_clobber ||
@@ -380,8 +381,8 @@ static NativeLoc mock_reg_loc(KitCgTypeId type, NativeAllocClass cls, Reg reg) {
/* Deliberately source the post-call copy from the same frame value used by
* the argument. The call clobbers the sole staging register, so the emitter
* must end the pre-call temp/cache phase and reload this source afterwards. */
-static void mock_plan_call(NativeTarget *target, const NativeCallDesc *desc,
- NativeCallPlan *plan) {
+static void mock_marshal_call(NativeTarget *target, const NativeCallDesc *desc,
+ NativeCallPhase *plan) {
MockNative *mock = mock_of(target);
if (mock->plan_two_arg_indirect_call) {
NativeLoc staged_callee;
@@ -444,10 +445,9 @@ static void mock_plan_call(NativeTarget *target, const NativeCallDesc *desc,
mock->call_ret.mem.align = 8u;
plan->rets = &mock->call_ret;
plan->nrets = 1u;
- plan->clobber_mask[NATIVE_REG_INT] = 1u << 8u;
}
-static void mock_emit_call(NativeTarget *target, const NativeCallPlan *plan) {
+static void mock_emit_call(NativeTarget *target, const NativeCallPhase *plan) {
MockNative *mock = mock_of(target);
if (mock->plan_two_arg_indirect_call) {
CU_EXPECT(mock->unit,
@@ -474,24 +474,24 @@ static void mock_emit_call(NativeTarget *target, const NativeCallPlan *plan) {
++mock->calls;
}
-static void mock_plan_ret(NativeTarget *target, const CGFuncDesc *desc,
+static void mock_marshal_ret(NativeTarget *target, const CGFuncDesc *desc,
const NativeLoc *value,
- NativeCallPlanRet **out_rets, u32 *out_nrets) {
+ NativeCallPhaseRet **out_rets, u32 *out_nrets) {
MockNative *mock = mock_of(target);
(void)desc;
- if (!mock->plan_two_part_ret) {
+ if (!mock->plan_four_part_ret) {
*out_rets = NULL;
*out_nrets = 0u;
return;
}
CU_EXPECT(mock->unit, value && value->kind == NATIVE_LOC_FRAME,
- "two-part return test requires a frame-resident source");
+ "four-part return test requires a frame-resident source");
if (!value || value->kind != NATIVE_LOC_FRAME) {
*out_rets = NULL;
*out_nrets = 0u;
return;
}
- for (u32 i = 0; i < 2u; ++i) {
+ for (u32 i = 0; i < 4u; ++i) {
memset(&mock->ret_parts[i], 0, sizeof mock->ret_parts[i]);
mock->ret_parts[i].src =
native_loc_stack(value->type, value->v.frame, (i32)(i * 8u));
@@ -502,7 +502,7 @@ static void mock_plan_ret(NativeTarget *target, const CGFuncDesc *desc,
mock->ret_parts[i].mem.align = 8u;
}
*out_rets = mock->ret_parts;
- *out_nrets = 2u;
+ *out_nrets = 4u;
}
static void mock_ret(NativeTarget *target) { (void)target; }
@@ -548,10 +548,10 @@ static void mock_init(MockNative *mock, KitUnit *unit, Compiler *c) {
mock->base.atomic_rmw = mock_atomic_rmw;
mock->base.atomic_cas = mock_atomic_cas;
mock->base.intrinsic = mock_intrinsic;
- mock->base.plan_call = mock_plan_call;
+ mock->base.marshal_call = mock_marshal_call;
mock->base.emit_call = mock_emit_call;
mock->base.machine_op_clobbers = mock_machine_op_clobbers;
- mock->base.plan_ret = mock_plan_ret;
+ mock->base.marshal_ret = mock_marshal_ret;
mock->base.ret = mock_ret;
}
@@ -1395,7 +1395,7 @@ static void allocator_cache_waits_for_hard_liveness(KitUnit *unit) {
FrameSlot clobbered_slot, preserved_slot, record_slot;
Inst *in, *clobber;
NativeMachineOp mop;
- u32 clobber_mask[NATIVE_CALL_PLAN_CLASSES];
+ u32 clobber_mask[NATIVE_REG_CLASS_COUNT];
u32 block;
CU_EXPECT(unit,
@@ -1482,7 +1482,11 @@ static void allocator_cache_waits_for_hard_liveness(KitUnit *unit) {
/* The O0 direct-target value bank is deliberately empty. In particular, it
* must not make the machinized allocation registers cache-eligible before
* the lazy hard-liveness table has actually been built. */
- mock.classes[NATIVE_REG_INT].caller_saved_mask = 1u << 10u;
+ mock.int_phys[0].reg = 10u;
+ mock.int_phys[0].cls = NATIVE_REG_INT;
+ mock.int_phys[0].flags = NATIVE_REG_CALLER_SAVED;
+ mock.classes[NATIVE_REG_INT].phys = mock.int_phys;
+ mock.classes[NATIVE_REG_INT].nphys = 1u;
memset(&mop, 0, sizeof mop);
mop.kind = NATIVE_MOP_BITFIELD_STORE;
@@ -1526,7 +1530,7 @@ static void reserved_cache_obeys_machine_clobbers(KitUnit *unit) {
FrameSlot clobbered_slot, preserved_slot, record_slot;
Inst *in, *clobber;
NativeMachineOp mop;
- u32 clobber_mask[NATIVE_CALL_PLAN_CLASSES];
+ u32 clobber_mask[NATIVE_REG_CLASS_COUNT];
u32 block;
CU_EXPECT(unit,
@@ -1739,7 +1743,7 @@ static void return_parts_keep_prior_abi_destinations_live(KitUnit *unit) {
f->opt_reserved_regs[RC_INT] |= 1u << (8u + i);
}
f->emit_temp_reg_count[RC_INT] = 4u;
- value_slot = add_slot(f, i64_type, 16u, 8u);
+ value_slot = add_slot(f, i64_type, 32u, 8u);
ret = emit_with_ops(f, block, IR_RET, 0u);
aux = arena_znew(f->arena, IRRetAux);
@@ -1749,7 +1753,7 @@ static void return_parts_keep_prior_abi_destinations_live(KitUnit *unit) {
ret->extra.aux = aux;
mock_init(&mock, unit, c);
- mock.plan_two_part_ret = 1u;
+ mock.plan_four_part_ret = 1u;
mock.int_temps[2] = 10u;
mock.int_temps[3] = 11u;
mock.classes[NATIVE_REG_INT].nscratch = 4u;
@@ -1758,16 +1762,15 @@ static void return_parts_keep_prior_abi_destinations_live(KitUnit *unit) {
(1u << 8u) | (1u << 9u) | (1u << 10u) | (1u << 11u);
opt_emit_native(c, f, &mock.base);
- CU_EXPECT(unit, mock.nframe_load_dsts == 2u,
- "two return parts should each load their frame source "
+ CU_EXPECT(unit, mock.nframe_load_dsts == 4u,
+ "four return parts should each load their frame source "
"(loads=%u)",
mock.nframe_load_dsts);
- if (mock.nframe_load_dsts == 2u) {
- CU_EXPECT(unit, mock.frame_load_dsts[0] == 8u,
- "first return part should load directly into r8");
- CU_EXPECT(unit, mock.frame_load_dsts[1] == 9u,
- "second return part should load directly into r9");
- }
+ if (mock.nframe_load_dsts == 4u)
+ for (u32 i = 0; i < 4u; ++i)
+ CU_EXPECT(unit, mock.frame_load_dsts[i] == (Reg)(8u + i),
+ "return part %u should load directly into r%u", i,
+ (unsigned)(8u + i));
CU_EXPECT(unit, mock.moves == 0u,
"frame return parts should not route through emitter temps");
diff --git a/test/opt/reg_effects_test.c b/test/opt/reg_effects_test.c
@@ -5,6 +5,7 @@
#include <kit/frontend.h>
#include "arch/aa64/aa64.h"
+#include "arch/arm32/arm32.h"
#include "arch/riscv/rv64.h"
#include "arch/x64/x64.h"
#include "core/pool.h"
@@ -63,7 +64,7 @@ typedef struct MachinizeRunCtx {
static int invalid_machine_effect(
NativeTarget* target, const NativeMachineOp* op,
- u32 mask[NATIVE_CALL_PLAN_CLASSES]) {
+ u32 mask[NATIVE_REG_CLASS_COUNT]) {
(void)target;
(void)op;
mask[0] = mask[1] = mask[2] = 0;
@@ -912,6 +913,165 @@ static void aa64_tls_model_fixed_effects(KitUnit* u) {
}
}
+static void x64_machine_effect_families(KitUnit* u) {
+ static const struct {
+ NativeMachineOp op;
+ u32 expected;
+ const char* name;
+ } cases[] = {
+ {{.kind = NATIVE_MOP_BINOP, .binop = BO_SDIV},
+ (1u << 0u) | (1u << 2u), "signed division"},
+ {{.kind = NATIVE_MOP_BINOP, .binop = BO_SHL, .second_is_reg = 1u},
+ 1u << 1u, "variable shift"},
+ {{.kind = NATIVE_MOP_BINOP, .binop = BO_SHL}, 0u, "immediate shift"},
+ {{.kind = NATIVE_MOP_BITFIELD_LOAD}, 0u, "bitfield load"},
+ {{.kind = NATIVE_MOP_BITFIELD_STORE},
+ (1u << 0u) | (1u << 1u) | (1u << 2u), "bitfield store"},
+ {{.kind = NATIVE_MOP_ATOMIC_CAS},
+ (1u << 0u) | (1u << 1u) | (1u << 2u), "atomic compare-exchange"},
+ {{.kind = NATIVE_MOP_ATOMIC_RMW},
+ (1u << 0u) | (1u << 1u) | (1u << 2u), "atomic read-modify-write"},
+ {{.kind = NATIVE_MOP_VA_START}, 0u, "va_start"},
+ {{.kind = NATIVE_MOP_VA_ARG}, 0u, "va_arg"},
+ {{.kind = NATIVE_MOP_INTRINSIC, .intrin = INTRIN_UMUL_OVERFLOW},
+ (1u << 0u) | (1u << 2u), "unsigned multiply overflow"},
+ {{.kind = NATIVE_MOP_INTRINSIC, .intrin = INTRIN_READCYCLECOUNTER},
+ (1u << 0u) | (1u << 2u), "cycle counter"},
+ {{.kind = NATIVE_MOP_INTRINSIC, .intrin = INTRIN_SYSCALL},
+ (1u << 0u) | (1u << 1u) | (1u << 2u) | (1u << 6u) | (1u << 7u) |
+ (1u << 8u) | (1u << 9u),
+ "syscall"},
+ {{.kind = NATIVE_MOP_TLS_ADDR}, 0u, "ELF local-exec TLS"},
+ };
+ KitCompiler* kc = NULL;
+ NativeTarget* target;
+
+ CU_EXPECT(u,
+ kit_unit_compiler_new(
+ u, kit_unit_target(KIT_ARCH_X86_64, KIT_OS_LINUX, KIT_OBJ_ELF),
+ &kc) == KIT_OK &&
+ kc != NULL,
+ "compiler allocation failed for x64 machine-effect family test");
+ if (!kc) return;
+ target = x64_native_target_new((Compiler*)kc, NULL, NULL);
+ for (u32 i = 0; i < sizeof cases / sizeof cases[0]; ++i) {
+ NativeRegMaskSet mask = {0};
+ int any = target->machine_op_clobbers(target, &cases[i].op, mask);
+ CU_EXPECT(u,
+ mask[NATIVE_REG_INT] == cases[i].expected &&
+ any == (cases[i].expected != 0u),
+ "x64 %s fixed effects mismatch: got %#x, expected %#x",
+ cases[i].name, mask[NATIVE_REG_INT], cases[i].expected);
+ }
+ kit_compiler_free(kc);
+}
+
+typedef NativeTarget* (*RoleTargetNew)(Compiler*, ObjBuilder*, MCEmitter*);
+
+typedef struct RoleMaskCase {
+ KitArchKind arch;
+ KitOSKind os;
+ KitObjFmt obj;
+ RoleTargetNew target_new;
+ const char* name;
+ NativeRegMaskSet caller;
+ NativeRegMaskSet callee;
+ NativeRegMaskSet arg;
+ NativeRegMaskSet ret;
+ NativeRegMaskSet reserved;
+} RoleMaskCase;
+
+static void register_role_flags_are_authoritative(KitUnit* u) {
+ static const RoleMaskCase cases[] = {
+ {.arch = KIT_ARCH_X86_64,
+ .os = KIT_OS_LINUX,
+ .obj = KIT_OBJ_ELF,
+ .target_new = x64_native_target_new,
+ .name = "x64 SysV",
+ .caller = {0x00000fc7u, 0x0000ffffu, 0u},
+ .callee = {0x0000f008u, 0u, 0u},
+ .arg = {0x000003c6u, 0x000000ffu, 0u},
+ .ret = {0x00000005u, 0x00000003u, 0u},
+ .reserved = {0x00001f39u, 0x0000f030u, 0u}},
+ {.arch = KIT_ARCH_X86_64,
+ .os = KIT_OS_WINDOWS,
+ .obj = KIT_OBJ_COFF,
+ .target_new = x64_native_target_new,
+ .name = "x64 Win64",
+ .caller = {0x00000f07u, 0x0000003fu, 0u},
+ .callee = {0x0000f0c8u, 0x0000ffc0u, 0u},
+ /* Argument/return flags describe the shared static x64 register table;
+ * caller/callee preservation is the only live-ABI override. */
+ .arg = {0x000003c6u, 0x000000ffu, 0u},
+ .ret = {0x00000005u, 0x00000003u, 0u},
+ .reserved = {0x00001f39u, 0x0000f030u, 0u}},
+ {.arch = KIT_ARCH_ARM_64,
+ .os = KIT_OS_LINUX,
+ .obj = KIT_OBJ_ELF,
+ .target_new = aa64_native_target_new,
+ .name = "aa64",
+ .caller = {0x0007ffffu, 0xffff00ffu, 0u},
+ .callee = {0x1ff80000u, 0x0000ff00u, 0u},
+ .arg = {0x000000ffu, 0x000000ffu, 0u},
+ .ret = {0x00000003u, 0x0000000fu, 0u},
+ .reserved = {0xe0070e00u, 0x00300000u, 0u}},
+ {.arch = KIT_ARCH_RV64,
+ .os = KIT_OS_LINUX,
+ .obj = KIT_OBJ_ELF,
+ .target_new = rv64_native_target_new,
+ .name = "rv64",
+ .caller = {0xf003fce0u, 0xf003fcffu, 0u},
+ .callee = {0x0ffc0300u, 0x0ffc0300u, 0u},
+ .arg = {0x0003fc00u, 0x0003fc00u, 0u},
+ .ret = {0x00000c00u, 0x00000c00u, 0u},
+ .reserved = {0xf00001ffu, 0x0000000fu, 0u}},
+ {.arch = KIT_ARCH_ARM_32,
+ .os = KIT_OS_FREESTANDING,
+ .obj = KIT_OBJ_ELF,
+ .target_new = arm32_native_target_new,
+ .name = "arm32",
+ .caller = {0x0000100fu, 0u, 0u},
+ .callee = {0x00000ff0u, 0u, 0u},
+ .arg = {0x0000000fu, 0u, 0u},
+ .ret = {0x00000003u, 0u, 0u},
+ .reserved = {0x0000f080u, 0u, 0u}},
+ };
+
+ for (u32 i = 0; i < sizeof cases / sizeof cases[0]; ++i) {
+ const RoleMaskCase* tc = &cases[i];
+ KitCompiler* kc = NULL;
+ NativeTarget* target;
+ CU_EXPECT(u,
+ kit_unit_compiler_new(
+ u, kit_unit_target(tc->arch, tc->os, tc->obj), &kc) ==
+ KIT_OK &&
+ kc != NULL,
+ "compiler allocation failed for %s register roles", tc->name);
+ if (!kc) continue;
+ target = tc->target_new((Compiler*)kc, NULL, NULL);
+ CU_EXPECT(u, target != NULL, "%s target allocation failed", tc->name);
+ if (target) {
+ native_reg_info_validate((Compiler*)kc, target->regs);
+ for (u32 cls = 0; cls < NATIVE_REG_CLASS_COUNT; ++cls) {
+ CU_EXPECT(u,
+ native_target_caller_saved_mask(
+ target, (NativeAllocClass)cls) == tc->caller[cls] &&
+ native_target_callee_saved_mask(
+ target, (NativeAllocClass)cls) == tc->callee[cls] &&
+ native_target_arg_mask(target, (NativeAllocClass)cls) ==
+ tc->arg[cls] &&
+ native_target_ret_mask(target, (NativeAllocClass)cls) ==
+ tc->ret[cls] &&
+ native_target_reserved_mask(
+ target, (NativeAllocClass)cls) == tc->reserved[cls],
+ "%s class %u derived register roles mismatch", tc->name,
+ cls);
+ }
+ }
+ kit_compiler_free(kc);
+ }
+}
+
int main(void) {
KitUnit u;
kit_unit_init(&u);
@@ -926,6 +1086,8 @@ int main(void) {
aa64_syscall_fixed_effects(&u);
x64_tls_model_fixed_effects(&u);
aa64_tls_model_fixed_effects(&u);
+ x64_machine_effect_families(&u);
+ register_role_flags_are_authoritative(&u);
x64_dying_fixed_sources_are_staged(&u);
fprintf(stderr, "reg-effects: %d checks, %d failures\n", u.checks, u.fails);
return kit_unit_status(&u);
diff --git a/test/parse/run.sh b/test/parse/run.sh
@@ -244,7 +244,11 @@ if [ $is_native_target -eq 1 ]; then
fi
# Cached start.o — build once for the harness run rather than per case.
-START_OBJ="$BUILD_DIR/parse_start.o"
+# This harness is routinely run for several architectures at once (for
+# example, independent cross-test workers). Keep the cached startup object
+# architecture-owned: sharing parse_start.o lets one worker replace another's
+# object between the probe above and the per-case link below.
+START_OBJ="$BUILD_DIR/parse_start.$TEST_ARCH.o"
have_start_obj=0
if [ $have_clang_cross -eq 1 ]; then
if clang $CLANG_TARGET -O1 -ffreestanding -fno-stack-protector \
@@ -363,6 +367,17 @@ kit_lane_R() {
}
kit_lane_E() {
+ # The ldbl128_* execution fixtures assert IEEE binary128 semantics. x64
+ # targets use either x87 extended precision (SysV/macOS) or double-aliased
+ # long double (Win64), so their deliberate return-0 guard cannot match the
+ # nonzero binary128 oracle. Keep the layout/macro probe, whose expected
+ # result is already zero, and classify the semantic fixtures as inapplicable.
+ if [ "$TEST_ARCH" = "x64" ] && \
+ [[ "$KIT_BASE" == ldbl128_* ]] && \
+ [[ "$KIT_BASE" != ldbl128_01_* ]]; then
+ kit_skip "$KIT_NAME/E" "x64 long double is not IEEE binary128"
+ return
+ fi
# rv32: freestanding bare-metal. parse-runner --emit -> kit ld with a startup
# that calls main() and reports its return via a SiFive finisher -> run under
# qemu-system-riscv32 (test/lib/exec_bare.sh). The qemu exit equals
