commit 965e6ceb27de27a316c8a251786d8717317ff6e0
parent a66c55af01ab541d4e379ef57d1ab8becdcf26b3
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Mon, 15 Jun 2026 19:13:20 -0700
opt: make the O1 whole-program inliner linear; fix static-inline hint
The -O1 whole-program inliner was superlinear (~O(n^2.5-3)): 9.6s / 86% of an
11.2s sqlite compile, exploding to 332x -O0 at 1600 funcs. Its hot gates used
linear scans -- funcset_find/funcset_index walked all funcs per call, and
recursive_or_scc -> func_reaches was a per-candidate reachability DFS paying that
O(n) lookup at every hop.
Attach an InlineIndex to the FuncSet: a symbol->index hash map (O(1)
funcset_find) plus per-function SCC ids from one iterative Tarjan pass. The
recursion gate is only ever asked where a caller->callee edge exists, where
"callee reaches caller" is exactly "same SCC", so it becomes an
scc[caller]==scc[callee] compare. Verdicts are identical to the old DFS -- the
sqlite -O1 object is byte-for-byte unchanged (same 730 inlines) -- only the
asymptotics change. opt.inline.total 9.6s -> 0.24s; -O1 sqlite 11.2s -> 1.6s
(~10x -O0, was ~70x); the synthetic call-graph sweep is now linear (doubling
funcs ~doubles -O1 time). The streaming tiny-inliner's ad-hoc 2-element sets
leave FuncSet.index NULL and keep the direct linear/DFS fallback.
Inline hints: the C "inline" keyword is tracked on the specifier flags, separate
from the __attribute__ flags decl building copied in, so "static inline" silently
fell back to the DEFAULT inline policy (cost cap 20) instead of HINT (cap 40) --
only always_inline/noinline (attributes) reached codegen. Merge specs DF_INLINE
onto the decl so static-inline bodies inline at sizes a plain static would not.
always_inline (unbounded) and noinline (never) already worked.
doc/OPT.md sec.8 updated to the fixed numbers + the hint table;
test/opt/whole_program_inline.sh gains a 4-policy (HINT/ALWAYS/DEFAULT/NEVER)
regression check on all three arches.
Gates: -O0 sqlite byte-identical; -O1 sqlite byte-identical to pre-perf-change +
links/runs (sum=42); ASan+verifiers clean; test-parse/cg-api/opt/pp/toy/smoke-x64
/smoke-rv64/isa all green.
Diffstat:
5 files changed, 305 insertions(+), 48 deletions(-)
diff --git a/doc/OPT.md b/doc/OPT.md
@@ -381,7 +381,13 @@ transform or analysis; the file paths orient the reader.
lazily re-lowered callee cache), gates on a tiny straightline-cost cap and a
whitelist that excludes calls/control-rich constructs, refuses self/recursive
callees, and splices the cloned body in. The whole-program inliner machinery
- (`inline_call_site` and its gates) also lives here.
+ (`opt_inline`, `inline_call_site`, and the cost/growth/policy gates) also lives
+ here; it builds an `InlineIndex` over the `FuncSet` once — a symbol→index hash
+ map plus per-function SCC ids (iterative Tarjan) — so callee resolution and the
+ recursion gate (`recursive_or_scc`, an `scc[caller] == scc[callee]` compare)
+ are O(1), keeping the pass linear in call sites (Section 8). Both inliners
+ honor the per-function/per-call `KitCgInlinePolicy` (DEFAULT/HINT/ALWAYS/NEVER),
+ which is how C `inline` / `always_inline` / `noinline` reach codegen.
- **Address folding** (`src/opt/pass_addr_fold.c`): the always-on O1 HIR folds —
`opt_addr_xform_pregs` (fold `ADDR_OF(local)` into direct `OPK_LOCAL`
load/store operands and clear `FSF_ADDR_TAKEN` when all such defs retire),
@@ -565,62 +571,76 @@ recording, *not* the finalize sweep that follows it.
```
$ KIT_METRICS=1 kit cc -O1 -c sqlite3.c --sysroot "$SDK" -o sqlite3.o
kit metrics:
- compile.tu 4.7M ticks (frontend: lex+pp+parse+CG recording)
- opt.inline.total 230.6M ticks (whole-program inliner, 1 call)
- opt.o1.total 25.7M ticks (per-function pipeline, summed over 2580 funcs)
- opt.regalloc 11.5M ticks (biggest per-function pass)
+ compile.tu ~10M ticks (frontend: lex+pp+parse+CG recording)
+ opt.inline.total ~6M ticks (whole-program inliner, 1 call)
+ opt.o1.total ~38M ticks (per-function pipeline, summed over 2580 funcs)
+ opt.regalloc ~17M ticks (biggest per-function pass)
...
opt.funcs=2580 opt.blocks=122208 opt.tiny_inline.inlined=13 opt.inline.inlined=730
```
### Measured cost and scaling (sqlite amalgamation, arm64/Darwin)
-Compiling the ~263 K-line sqlite amalgamation (`-c`, release kit) the wall-clock
-split is stark — `-O1` is **~70× slower than `-O0`** (11.2 s vs 0.16 s; tcc does
-the whole file in 0.06 s):
+Compiling the ~263 K-line sqlite amalgamation (`-c`, release kit), `-O1` runs in
+**~1.6 s — about 10× `-O0`** (0.16 s; tcc does the whole file in 0.06 s). The
+time is dominated by the per-function pipeline, with the linear-scan allocator
+and live-range construction the largest line items — exactly the shape a no-SSA
+`-O1` should have:
| phase (24 MHz ticks → s) | seconds | share |
|-------------------------------------|----------:|------:|
-| frontend (`compile.tu`) | 0.20 | 2% |
-| CG→opt lowering (`cg_ir_lower`) | 0.23 | 2% |
-| **whole-program inliner** (`opt.inline.total`) | **9.61** | **86%** |
-| per-function pipeline (`opt.o1.total`) | 1.07 | 10% |
-| — of which regalloc + live-ranges | 0.64 | |
-
-The whole-program inliner dominates, and it is **superlinear**. A synthetic
-sweep (N small functions in a dense call graph, best-of-3) shows the `-O1` time
-exploding while `-O0` stays flat:
-
-| funcs | `-O0` | `-O1` | O1/O0 |
-|------:|------:|------:|------:|
-| 200 | 0.027 | 0.060 | 2.3× |
-| 400 | 0.027 | 0.240 | 8.8× |
-| 800 | 0.030 | 1.524 | 50.5× |
-| 1600 | 0.035 | 11.48 | 332× |
-
-Doubling the function count multiplies `-O1` time by ~6–8× — an exponent near
-2.5–3. The root cause is in `pass_inline.c`: the inliner runs up to four
-fixpoint iterations over the whole `FuncSet`, and its hot gates resolve callees
-and check for recursion/SCC membership with **linear scans** — `funcset_find`
-and `funcset_index` walk all functions on every call, and `recursive_or_scc` →
-`func_reaches` is a per-candidate DFS whose every hop pays that O(n) lookup. With
-2580 functions and ~24 K candidate call sites this is the textbook
-"superlinear axis = bug" trap (`doc/plan/PERF.md`). The payoff is also small:
-730 of 24 K candidates inline.
-
-By contrast the per-function pipeline is well-behaved (~1 s total, linear in
-function count); within it the linear-scan allocator + live-range construction
-are the largest line items, as expected for a no-SSA `-O1`.
-
-**Implication for the "fast O1" goal.** The non-SSA per-function pipeline already
-meets the bar — strip the inliner and `-O1` would be ~1.5 s on sqlite (linear,
-~10× `-O0`). The whole-program inliner is what breaks it. The fixes are
-self-contained and do not touch correctness: (1) index the `FuncSet` by symbol
-(hash map) so `funcset_find`/`funcset_index` are O(1); (2) memoize the
-reachability/SCC check instead of re-running `func_reaches` per candidate; and/or
-(3) reconsider whether whole-program inlining belongs at `-O1` at all versus
-behind `-O2`/`-flto`, leaving `-O1` with only the cheap streaming `try_tiny_inline`
-(0.28 s on sqlite). Any of these restores the linear, "fast" `-O1` design point.
+| frontend (`compile.tu`) | ~0.3 | ~20% |
+| CG→opt lowering (`cg_ir_lower`) | ~0.4 | ~25% |
+| whole-program inliner (`opt.inline.total`) | ~0.24 | ~15% |
+| per-function pipeline (machinize→regalloc→emit) | ~0.7 | ~45% |
+| — of which regalloc + live-ranges | ~0.6 | |
+
+The cost scales **linearly** in function count. A synthetic sweep (N small
+functions in a dense call graph, best-of-3) — doubling N roughly doubles `-O1`
+time, converging on the same slope as `-O0`:
+
+| funcs | `-O0` | `-O1` | O1/O0 | `-O1` ×/2× funcs |
+|------:|------:|------:|------:|-----------------:|
+| 400 | 0.027 | 0.038 | 1.4× | — |
+| 800 | 0.030 | 0.049 | 1.6× | 1.3× |
+| 1600 | 0.034 | 0.074 | 2.2× | 1.5× |
+| 3200 | 0.043 | 0.131 | 3.1× | 1.8× |
+| 6400 | 0.062 | 0.264 | 4.3× | 2.0× |
+
+**This was not always so.** The whole-program inliner used to be superlinear
+(≈O(n^2.5–3): 9.6 s / 86 % of an 11.2 s `-O1` sqlite compile, exploding to 332×
+`-O0` at 1600 functions). The hot gates resolved callees and checked recursion
+with linear scans — `funcset_find`/`funcset_index` walked all functions on every
+call, and `recursive_or_scc` → `func_reaches` was a per-candidate reachability
+DFS whose every hop paid that O(n) lookup. The fix (in `pass_inline.c`) attaches
+an `InlineIndex` to the `FuncSet`: a symbol→index hash map (O(1) `funcset_find`)
+plus **per-function SCC ids computed once by iterative Tarjan**. The recursion
+gate is then a single observation — the inliner only asks "is this call
+recursive?" where a `caller→callee` edge exists, and there "callee reaches
+caller" is *exactly* "caller and callee share an SCC", so it reduces to an
+`scc[caller] == scc[callee]` compare. The verdicts are identical to the old DFS
+(the sqlite object stayed **byte-for-byte unchanged**, same 730 inlines); only
+the asymptotics changed. `opt.inline.total` dropped 9.6 s → ~0.24 s.
+
+### Inline hints
+
+The inliner honors the frontend's per-function `KitCgInlinePolicy`, which the C
+frontend derives from declaration hints:
+
+| C hint | policy | inliner behavior |
+|-------------------------------------|----------|------------------|
+| (none) | DEFAULT | inline if cost ≤ 20 |
+| `inline` / `static inline` | HINT | inline if cost ≤ 40 |
+| `__attribute__((always_inline))` | ALWAYS | inline regardless of cost (recursion still blocks) |
+| `__attribute__((noinline))` | NEVER | never inline |
+
+So a `static inline` body inlines at sizes a plain `static` one would not, and
+`always_inline` ignores the budget entirely, while `noinline` is always honored.
+(The `inline` *keyword* lives on the specifier flags, separate from the
+`__attribute__` flags; it has to be merged onto the declaration so the policy
+reaches codegen — a step that was missing, which had silently demoted every
+`static inline` to DEFAULT until corrected.) `test/opt/whole_program_inline.sh`
+guards all four policies on every arch.
### Code quality
diff --git a/lang/c/parse/parse.c b/lang/c/parse/parse.c
@@ -1242,6 +1242,15 @@ static SymEntry* declare_function(Parser* p, Sym fname, const Type* fn_ty,
decl_in.linkage == DL_EXTERNAL ? p->default_visibility : SV_DEFAULT;
attr_list_to_decl(p->c, p->decls, specs->attrs, &decl_in);
attr_list_to_decl(p->c, p->decls, dattrs, &decl_in);
+ /* The `inline` specifier is a keyword carried on specs->flags, separate from
+ * the __attribute__ flags attr_list_to_decl applies. It must be merged onto
+ * the decl so decl_inline_policy() sees it on both the recorded symbol attrs
+ * (decl_declare below) and the returned decl flags (out_decl_flags, used to
+ * build the function descriptor). Without this, `static inline` functions
+ * silently fell back to the DEFAULT inline policy — only the always_inline /
+ * noinline attributes reached codegen — so a `static inline` body got no
+ * stronger inlining than a plain static one. */
+ decl_in.flags |= (specs->flags & DF_INLINE);
did = decl_declare(p->decls, &decl_in);
fsym = decl_obj_sym(p->decls, did);
e = scope_define(p, fname, SEK_FUNC, fn_ty);
diff --git a/src/opt/opt_internal.h b/src/opt/opt_internal.h
@@ -23,6 +23,12 @@ struct FuncSet {
Func** funcs;
u32 nfuncs;
u32 cap;
+ /* Opaque InlineIndex* (pass_inline.c): a symbol->index map plus per-function
+ * SCC ids, built once by opt_inline so callee resolution and the
+ * recursion/SCC gate are O(1) instead of linear scans over `funcs`. NULL for
+ * the ad-hoc 2-element sets the streaming tiny-inliner builds, which fall back
+ * to the direct linear scan / reachability DFS. */
+ void* index;
};
typedef struct OptBlockList {
diff --git a/src/opt/pass_inline.c b/src/opt/pass_inline.c
@@ -2,6 +2,7 @@
#include "core/arena.h"
#include "core/core.h"
+#include "core/hashmap.h"
#include "core/metrics.h"
#include "opt/opt_internal.h"
@@ -75,8 +76,29 @@ static void instvec_push(InstVec* iv, const Inst* in) {
iv->v[iv->n++] = *in;
}
+/* Symbol -> index-into-fs->funcs map. Key 0 (OBJ_SYM_NONE) is the empty-slot
+ * sentinel in the hashmap, which is fine: a NONE-named function is never a
+ * callee (no symbol to call) and is handled by the linear fallback as a
+ * caller. */
+HASHMAP_DEFINE(InlineSymMap, ObjSymId, u32, hash_u32);
+
+/* O(1)-lookup acceleration for opt_inline, attached to FuncSet.index. Built
+ * once over the whole-program set; the streaming tiny-inliner leaves
+ * FuncSet.index NULL and the helpers below fall back to a linear scan over its
+ * 2-element set. */
+typedef struct InlineIndex {
+ InlineSymMap sym2idx; /* ObjSymId -> index into fs->funcs */
+ u32* scc; /* SCC id per fs index; same id <=> mutually recursive */
+ u32 nfuncs; /* size of scc[] (== fs->nfuncs at build time) */
+} InlineIndex;
+
static Func* funcset_find(FuncSet* fs, ObjSymId sym) {
if (!fs || sym == OBJ_SYM_NONE) return NULL;
+ if (fs->index) {
+ InlineIndex* idx = (InlineIndex*)fs->index;
+ u32* slot = InlineSymMap_get(&idx->sym2idx, sym);
+ return slot ? fs->funcs[*slot] : NULL;
+ }
for (u32 i = 0; i < fs->nfuncs; ++i)
if (fs->funcs[i] && fs->funcs[i]->name == sym) return fs->funcs[i];
return NULL;
@@ -84,6 +106,13 @@ static Func* funcset_find(FuncSet* fs, ObjSymId sym) {
static int funcset_index(FuncSet* fs, Func* f) {
if (!fs || !f) return -1;
+ if (fs->index && f->name != OBJ_SYM_NONE) {
+ InlineIndex* idx = (InlineIndex*)fs->index;
+ u32* slot = InlineSymMap_get(&idx->sym2idx, f->name);
+ /* Guard against a (deduped-away in practice) name collision: only trust the
+ * mapped slot when it actually points back at f. */
+ if (slot && fs->funcs[*slot] == f) return (int)*slot;
+ }
for (u32 i = 0; i < fs->nfuncs; ++i)
if (fs->funcs[i] == f) return (int)i;
return -1;
@@ -140,6 +169,21 @@ static int func_reaches(FuncSet* fs, Func* from, Func* target, u8* seen) {
static int recursive_or_scc(FuncSet* fs, Func* caller, Func* callee) {
if (caller == callee) return 1;
+ /* Whole-program path: the inliner only ever asks this where a caller->callee
+ * call edge exists, so "callee can transitively reach caller" is exactly
+ * "caller and callee share a strongly-connected component". With the SCC ids
+ * precomputed (inline_index_build) this is O(1) and gives the identical
+ * verdict the linear func_reaches DFS would. */
+ if (fs->index) {
+ InlineIndex* idx = (InlineIndex*)fs->index;
+ if (idx->scc) {
+ int ci = funcset_index(fs, caller);
+ int ce = funcset_index(fs, callee);
+ if (ci >= 0 && ce >= 0 && (u32)ci < idx->nfuncs && (u32)ce < idx->nfuncs)
+ return idx->scc[ci] == idx->scc[ce];
+ }
+ }
+ /* Ad-hoc 2-element set (streaming tiny-inliner): direct reachability DFS. */
u8* seen = arena_zarray(fs->arena, u8, fs->nfuncs ? fs->nfuncs : 1u);
return func_reaches(fs, callee, caller, seen);
}
@@ -664,9 +708,121 @@ static void inline_order_visit(InlineOrderCtx* ctx, Func* f) {
ctx->order[ctx->norder++] = f;
}
+/* Count the direct in-set callees of f (CSR degree / fill helper). */
+static u32 inline_collect_callees(FuncSet* fs, Func* f, u32* out, u32 base) {
+ u32 n = 0;
+ if (!f) return 0;
+ for (u32 b = 0; b < f->nblocks; ++b) {
+ Block* bl = &f->blocks[b];
+ for (u32 k = 0; k < bl->ninsts; ++k) {
+ Func* c = direct_callee(fs, &bl->insts[k]);
+ int ci = c ? funcset_index(fs, c) : -1;
+ if (ci < 0) continue;
+ if (out) out[base + n] = (u32)ci;
+ ++n;
+ }
+ }
+ return n;
+}
+
+/* Build FuncSet.index: a symbol->index map and per-function SCC ids over the
+ * direct-call graph, so funcset_find and the recursion/SCC gate are O(1). The
+ * SCC partition is invariant under the inlining the pass performs (it only adds
+ * caller->X edges where X already sat strictly below the caller in the
+ * condensation DAG, and only removes cross-SCC call edges), so one build serves
+ * every fixpoint iteration. Tarjan's algorithm, iterative to bound stack depth
+ * on deep call chains. O(V + E). */
+static void inline_index_build(FuncSet* fs, InlineIndex* idx) {
+ u32 n = fs->nfuncs;
+ u32 *deg, *off, *edges, *comp, *st_node, *st_ei, *low;
+ i32* dfsnum;
+ u8* onstack;
+ u32 nedges, comp_top = 0, st_top = 0, counter = 0, ncomp = 0;
+ memset(idx, 0, sizeof *idx);
+ InlineSymMap_init_cap(&idx->sym2idx, fs->c->ctx->heap, 0);
+ for (u32 i = 0; i < n; ++i)
+ if (fs->funcs[i] && fs->funcs[i]->name != OBJ_SYM_NONE)
+ (void)InlineSymMap_set(&idx->sym2idx, fs->funcs[i]->name, i);
+ fs->index = idx; /* enables the O(1) funcset_find/index used just below */
+ idx->nfuncs = n;
+ if (n == 0) return;
+
+ deg = arena_zarray(fs->arena, u32, n);
+ for (u32 i = 0; i < n; ++i) deg[i] = inline_collect_callees(fs, fs->funcs[i], NULL, 0);
+ off = arena_array(fs->arena, u32, n + 1u);
+ off[0] = 0;
+ for (u32 i = 0; i < n; ++i) off[i + 1u] = off[i] + deg[i];
+ nedges = off[n];
+ edges = arena_array(fs->arena, u32, nedges ? nedges : 1u);
+ for (u32 i = 0; i < n; ++i)
+ (void)inline_collect_callees(fs, fs->funcs[i], edges, off[i]);
+
+ idx->scc = arena_array(fs->arena, u32, n);
+ dfsnum = arena_array(fs->arena, i32, n);
+ low = arena_array(fs->arena, u32, n);
+ onstack = arena_zarray(fs->arena, u8, n);
+ comp = arena_array(fs->arena, u32, n);
+ st_node = arena_array(fs->arena, u32, n);
+ st_ei = arena_array(fs->arena, u32, n);
+ for (u32 i = 0; i < n; ++i) {
+ dfsnum[i] = -1;
+ idx->scc[i] = 0;
+ }
+ for (u32 root = 0; root < n; ++root) {
+ if (dfsnum[root] != -1) continue;
+ st_node[st_top] = root;
+ st_ei[st_top] = 0;
+ ++st_top;
+ while (st_top) {
+ u32 v = st_node[st_top - 1u];
+ u32 ei = st_ei[st_top - 1u];
+ if (ei == 0) { /* pre-order visit */
+ dfsnum[v] = (i32)counter;
+ low[v] = counter;
+ ++counter;
+ comp[comp_top++] = v;
+ onstack[v] = 1;
+ }
+ if (ei < deg[v]) {
+ u32 w = edges[off[v] + ei];
+ st_ei[st_top - 1u] = ei + 1u;
+ if (dfsnum[w] == -1) { /* tree edge: descend */
+ st_node[st_top] = w;
+ st_ei[st_top] = 0;
+ ++st_top;
+ } else if (onstack[w] && (u32)dfsnum[w] < low[v]) {
+ low[v] = (u32)dfsnum[w]; /* back/cross edge to a live ancestor */
+ }
+ } else { /* post-order: v's subtree done */
+ if (low[v] == (u32)dfsnum[v]) {
+ for (;;) {
+ u32 w = comp[--comp_top];
+ onstack[w] = 0;
+ idx->scc[w] = ncomp;
+ if (w == v) break;
+ }
+ ++ncomp;
+ }
+ --st_top;
+ if (st_top) {
+ u32 p = st_node[st_top - 1u];
+ if (low[v] < low[p]) low[p] = low[v];
+ }
+ }
+ }
+ }
+}
+
+static void inline_index_fini(FuncSet* fs, InlineIndex* idx) {
+ InlineSymMap_fini(&idx->sym2idx);
+ fs->index = NULL;
+}
+
void opt_inline(FuncSet* fs, int max_iters) {
if (!fs || fs->nfuncs == 0 || max_iters <= 0) return;
if (max_iters > 4) max_iters = 4;
+ InlineIndex idx;
+ inline_index_build(fs, &idx);
u32* base_cost = arena_array(fs->arena, u32, fs->nfuncs);
for (u32 i = 0; i < fs->nfuncs; ++i)
base_cost[i] = func_inline_cost(fs->funcs[i]);
@@ -697,6 +853,7 @@ void opt_inline(FuncSet* fs, int max_iters) {
}
if (!changed) break;
}
+ inline_index_fini(fs, &idx);
}
/* Streaming single-caller variant for the O1 pipeline. Same gates as
diff --git a/test/opt/whole_program_inline.sh b/test/opt/whole_program_inline.sh
@@ -135,4 +135,69 @@ if ! "$KIT" build-exe -O1 "$WORK/verb_exe.c" -o "$WORK/verb_exe" \
fi
printf 'whole-program-inline build-exe correct + fused\n'
+# Inline-policy hints: a `static inline` (HINT) callee raises the inliner's cost
+# budget above a plain static (DEFAULT); always_inline ignores the cap, noinline
+# blocks. The inline decision runs on target-independent IR, so the verdict is
+# identical on every arch. A ~30-op callee sits above the DEFAULT cap (20) and
+# below the HINT cap (40): the static-inline one fuses while the byte-identical
+# plain-static one does not. Regression guard for the keyword `inline` reaching
+# the codegen inline policy (it used to be dropped, so static inline silently
+# behaved like a plain static).
+mul() { local n=$1 s="x" i; for ((i = 1; i < n; i++)); do s="$s*x"; done; printf '%s' "$s"; }
+H30=$(mul 31)
+A60=$(mul 61)
+read -r -d '' HINT_SRC <<EOF || true
+static inline int ih(int x){ return $H30; }
+static int id(int x){ return $H30; }
+__attribute__((always_inline)) static int ia(int x){ return $A60; }
+__attribute__((noinline)) static int iv(int x){ return x + 1; }
+int use_ih(int x){ return ih(x); }
+int use_id(int x){ return id(x); }
+int use_ia(int x){ return ia(x); }
+int use_iv(int x){ return iv(x); }
+EOF
+check_inline_hints() {
+ local triple=$1
+ local tag=$2
+ local src="$WORK/hints_$tag.c"
+ local obj="$WORK/hints_$tag.o"
+ local dis="$WORK/hints_$tag.dis"
+ printf '%s\n' "$HINT_SRC" > "$src"
+ "$KIT" cc -target "$triple" -O1 -ffreestanding -std=c11 -c "$src" -o "$obj" \
+ > "$WORK/hints_$tag.cc.out" 2>&1
+ "$KIT" objdump -d "$obj" > "$dis" 2>&1
+ # Isolate one function body: from its `<name>:` header to the next symbol
+ # header (the disassembly has no blank separators, so a /^$/ window would run
+ # to EOF and bleed across functions).
+ calls_in() {
+ awk -v f="<$1>:" '$0 ~ f {g = 1; next} /<[A-Za-z_].*>:/ {g = 0} g' "$dis" \
+ | grep -cE "$call_mnemonics" || true
+ }
+ local ch cd ca cv
+ ch=$(calls_in use_ih)
+ cd=$(calls_in use_id)
+ ca=$(calls_in use_ia)
+ cv=$(calls_in use_iv)
+ if [ "$ch" -ne 0 ]; then
+ printf 'inline-hints FAILED: %s static-inline (HINT) callee not fused (%s call[s])\n' "$tag" "$ch" >&2
+ exit 1
+ fi
+ if [ "$cd" -eq 0 ]; then
+ printf 'inline-hints FAILED: %s plain-static (DEFAULT) callee fused above its budget\n' "$tag" >&2
+ exit 1
+ fi
+ if [ "$ca" -ne 0 ]; then
+ printf 'inline-hints FAILED: %s always_inline callee not fused\n' "$tag" >&2
+ exit 1
+ fi
+ if [ "$cv" -eq 0 ]; then
+ printf 'inline-hints FAILED: %s noinline callee was inlined\n' "$tag" >&2
+ exit 1
+ fi
+ printf 'inline-hints %-8s HINT+ALWAYS fused, DEFAULT+NEVER kept out-of-line\n' "$tag"
+}
+check_inline_hints aarch64-linux-gnu aa64
+check_inline_hints x86_64-linux-gnu x64
+check_inline_hints riscv64-linux-gnu rv64
+
printf 'whole-program-inline: ok\n'
