commit c89c5909d47ae0b854d9018f0bec63db7c559277
parent 9c3ce7acc8fe15631b220d004e0e5406340fa775
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Thu, 11 Jun 2026 09:43:56 -0700
perf: byte-identical -O0 throughput sprint (9 segments, +10–15%)
Constant-factor wins across every compile segment, each gated
byte-identical to the pre-change binary (objects -O0/-O1, -g DWARF,
-S, -E, diagnostics, linked exe, line-splice battery — see
scripts/perf_identity_gate.sh) and validated against the full test
suites. Measured (best-of-11, integrated kit vs HEAD golden):
type-decl +15.0%, fn-count +10.8%, body-size +10.3%, pp-macro -E
+8.5%, ref-density +7.1%, global-decl +4.8%, locals +3.0%.
Per segment:
- intern (core/pool): inline sym_eq + miss-path copy to drop the
freestanding memcmp/memcpy libcalls; precompute the grow threshold.
- codegen-dispatch (cg/arith,value): cache a 1-byte wide_kind tag on
the value-stack node (set once in api_push); the per-binop i128/
wide64/f128 stack-top gauntlet becomes a load+compare. Biggest
body-size lever. Also CSE the foldable-int can_delay check.
- codegen-regcache (cg/native_direct_target): flat reg_last_use[cls]
[reg] mirror for the LRU victim scan + precomputed caller-saved-
allocable sublist for the free-reg scan (~+2.2% body-size).
- parser (lang/c/parse): fuse reject_same_scope_redefinition +
scope_define into one hash probe (NAME##_replace); O(1) keyword
classification (KwMap) replacing the ~52-entry linear scan; drop a
redundant current-scope lookup in declare_function.
- lexer (cpp/lex): drop the per-token memset in lex_next (+ the two
src_next_raw EOF fallbacks); splice-free fast-path ident scan
(~+6% type-decl).
- pp (cpp/pp): out-pointer token readers (pp_next_raw_into /
src_next_raw_into) killing the 24B sret round-trip; inline the
SRC_BUF replay fast path; skip subst_phase2 placemarker compaction
when none emitted (~+8.5% pp-macro -E).
- emit (core/buf, obj, arch/mc): inline buf_write small-write fast
path (buf_write_slow tail); cache the active section Buf* on the
MCEmitter; pre-size SymNameIndex.
- memset-abi (arch/{aa64,x64,riscv}/native, mc): right-size the
NativeCallPlanRet scratch in plan_ret/plan_call (skip the alloc on
the sret path; size to nparts); drop the dead labels_grow tail
memset.
- include-io (cpp/pp): resolved-path header-content cache + include-
resolution memo (collapses re-opens of repeatedly-included headers;
helps real header graphs, not the single-include synthetic axis).
Diffstat:
24 files changed, 795 insertions(+), 162 deletions(-)
diff --git a/include/kit/support/hashmap.h b/include/kit/support/hashmap.h
@@ -122,6 +122,37 @@ static inline uint32_t kit_hash_u64(uint64_t x) {
return 1; \
} \
\
+ /* Like NAME##_set (newest writer wins) but reports the prior value: writes \
+ * the old value to *old_out on overwrite, or a zero-initialized VT on \
+ * insert. Lets a caller fuse a read-then-write of the same key into one \
+ * hash+probe (e.g. the redefinition check + define in the C parser). */ \
+ KIT_HASHMAP_UNUSED static inline int NAME##_replace(NAME* m, KT k, VT v, \
+ VT* old_out) { \
+ uint32_t mask, j; \
+ if (m->cap == 0 || \
+ m->used * KIT_HASHMAP_LOAD_DEN >= m->cap * KIT_HASHMAP_LOAD_NUM) \
+ NAME##_resize(m, m->cap ? m->cap * 2u : KIT_HASHMAP_INIT_CAP); \
+ if (m->cap == 0) { \
+ if (old_out) memset(old_out, 0, sizeof *old_out); \
+ return -1; /* resize OOM'd from empty: avoid NULL deref */ \
+ } \
+ mask = m->cap - 1u; \
+ j = HASH_FN(k) & mask; \
+ while (m->slots[j].k) { \
+ if (m->slots[j].k == (k)) { \
+ if (old_out) *old_out = m->slots[j].v; \
+ m->slots[j].v = (v); \
+ return 0; \
+ } \
+ j = (j + 1u) & mask; \
+ } \
+ if (old_out) memset(old_out, 0, sizeof *old_out); \
+ m->slots[j].k = (k); \
+ m->slots[j].v = (v); \
+ m->used++; \
+ return 1; \
+ } \
+ \
KIT_HASHMAP_UNUSED static inline int NAME##_try_insert(NAME* m, KT k, VT v, \
VT* existing_out) { \
uint32_t mask, j; \
diff --git a/lang/c/parse/parse.c b/lang/c/parse/parse.c
@@ -363,15 +363,22 @@ SymEntry* scope_lookup_current(Parser* p, Sym name) {
return p->scope ? scope_entries_find(p->scope, name) : NULL;
}
-SymEntry* scope_lookup(Parser* p, Sym name) {
+/* Walk the scope chain starting at `from` (inclusive), returning the first
+ * binding for `name`. scope_lookup is the common case starting at p->scope; a
+ * caller that has already probed the current scope can start at its parent. */
+static SymEntry* scope_lookup_from(Scope* from, Sym name) {
Scope* s;
- for (s = p->scope; s; s = s->parent) {
+ for (s = from; s; s = s->parent) {
SymEntry* e = scope_entries_find(s, name);
if (e) return e;
}
return NULL;
}
+SymEntry* scope_lookup(Parser* p, Sym name) {
+ return scope_lookup_from(p->scope, name);
+}
+
static void sym_set_decl(SymEntry* e, DeclId id, DeclStorage storage,
DeclLinkage linkage, u32 flags, DeclState state) {
e->decl_id = id;
@@ -397,18 +404,62 @@ static int is_ordinary_decl_kind(SymEntryKind k) {
k == SEK_TYPEDEF || k == SEK_ENUM_CST;
}
-static void reject_same_scope_redefinition(Parser* p, Sym name,
- SymEntryKind kind,
- const Type* type) {
- SymEntry* e = scope_lookup_current(p, name);
- if (!e || !is_ordinary_decl_kind((SymEntryKind)e->kind)) return;
- if (e->kind == SEK_TYPEDEF && kind == SEK_TYPEDEF &&
- type_compatible(e->type, type)) {
+/* Apply the same-scope redefinition rule against `prior` (the binding present
+ * before the new entry is installed): reject an ordinary-kind redeclaration
+ * unless it is a typedef of a compatible type. */
+static void reject_redef_on(Parser* p, const SymEntry* prior, SymEntryKind kind,
+ const Type* type) {
+ if (!prior || !is_ordinary_decl_kind((SymEntryKind)prior->kind)) return;
+ if (prior->kind == SEK_TYPEDEF && kind == SEK_TYPEDEF &&
+ type_compatible(prior->type, type)) {
return;
}
perr(p, "redefinition of identifier");
}
+/* Fused redefinition-check + define: one hash+probe on the scope index reads
+ * the prior binding (for the redef rule) AND installs the new entry, instead of
+ * probing the same key twice (a get for the redef check then a set in
+ * scope_define). Behavior is identical to a redef check followed by
+ * return scope_define(p, name, kind, type);
+ * — the redef rule fires on the captured prior before the new entry shadows it,
+ * and the LIFO list + newest-wins index semantics are preserved. Use this only
+ * at the rejecting declaration sites; plain scope_define stays for the
+ * non-rejecting redeclaration-merge sites. */
+static SymEntry* scope_define_checked(Parser* p, Sym name, SymEntryKind kind,
+ const Type* type) {
+ SymEntry* e = arena_new(p->pool->arena, SymEntry);
+ Scope* s = p->scope;
+ SymEntry* prior;
+ if (!e) perr(p, "out of memory in scope_define_checked");
+ memset(e, 0, sizeof *e);
+ e->name = name;
+ e->kind = (u8)kind;
+ e->type = type;
+ if (name && s->emap.cap) {
+ /* Active index: prepend to the LIFO list, then fold the prior-read and the
+ * index install into a single probe (newest define wins). */
+ SymEntry* old = NULL;
+ e->next = s->entries;
+ s->entries = e;
+ s->nentries++;
+ (void)SymEntryMap_replace(&s->emap, name, e, &old);
+ prior = old;
+ } else {
+ /* No index yet (or anonymous): capture the prior binding before the prepend
+ * shadows it, then mirror scope_entries_index_put's threshold-trip build.
+ * Anonymous entries (name == 0) stay off the index, as before. */
+ prior = scope_entries_find(s, name);
+ e->next = s->entries;
+ s->entries = e;
+ s->nentries++;
+ if (name && s->nentries > SCOPE_INDEX_THRESHOLD)
+ scope_entries_index_build(p, s);
+ }
+ reject_redef_on(p, prior, kind, type);
+ return e;
+}
+
static void tag_index_build(Parser* p, Scope* s) {
TagEntry* e;
TagEntryMap_init_cap(&s->tmap, &p->pool->arena_heap, 64u);
@@ -493,8 +544,7 @@ FrameSlot make_local_aligned(Parser* p, Sym name, const Type* type, SrcLoc loc,
fsd.kind = FS_LOCAL;
fsd.flags = FSF_NONE;
s = pcg_local(p, &fsd);
- reject_same_scope_redefinition(p, name, SEK_LOCAL, type);
- e = scope_define(p, name, SEK_LOCAL, type);
+ e = scope_define_checked(p, name, SEK_LOCAL, type);
e->v.slot = s;
sym_set_decl(e, DECL_NONE, DS_AUTO, DL_NONE, DF_NONE, DSTATE_DEFINED);
return s;
@@ -754,8 +804,7 @@ static void parse_init_declarator(Parser* p, const DeclSpecs* specs) {
perr(p, "typedef declarator cannot have initializer");
}
{
- reject_same_scope_redefinition(p, name, SEK_TYPEDEF, var_ty);
- SymEntry* e = scope_define(p, name, SEK_TYPEDEF, var_ty);
+ SymEntry* e = scope_define_checked(p, name, SEK_TYPEDEF, var_ty);
sym_set_decl(e, DECL_NONE, DS_TYPEDEF, DL_NONE, specs->flags,
DSTATE_DECLARED);
if (p->vla_pending && var_ty && var_ty->kind == TY_ARRAY) {
@@ -822,8 +871,7 @@ static void parse_init_declarator(Parser* p, const DeclSpecs* specs) {
attr_list_to_decl(p->c, p->decls, specs->attrs, &decl_in);
did = decl_declare(p->decls, &decl_in);
sym = decl_obj_sym(p->decls, did);
- reject_same_scope_redefinition(p, name, SEK_GLOBAL, var_ty);
- e = scope_define(p, name, SEK_GLOBAL, var_ty);
+ e = scope_define_checked(p, name, SEK_GLOBAL, var_ty);
e->v.sym = sym;
sym_set_decl(e, did, DS_STATIC, DL_NONE, decl_in.flags, DSTATE_DEFINED);
align_eff = (specs->align > decl_in.align) ? specs->align : decl_in.align;
@@ -870,8 +918,7 @@ static void parse_init_declarator(Parser* p, const DeclSpecs* specs) {
attr_list_to_decl(p->c, p->decls, specs->attrs, &decl_in);
did = decl_declare(p->decls, &decl_in);
sym = decl_obj_sym(p->decls, did);
- reject_same_scope_redefinition(p, name, SEK_GLOBAL, var_ty);
- e = scope_define(p, name, SEK_GLOBAL, var_ty);
+ e = scope_define_checked(p, name, SEK_GLOBAL, var_ty);
e->v.sym = sym;
sym_set_decl(e, did, DS_EXTERN, DL_EXTERNAL, decl_in.flags,
DSTATE_DECLARED);
@@ -1113,7 +1160,10 @@ static SymEntry* declare_function(Parser* p, Sym fname, const Type* fn_ty,
external_func_remember(p, fname, existing);
return existing;
}
- visible = scope_lookup(p, fname);
+ /* `existing` is provably NULL here (both branches above perr or return), so
+ * the current scope has no binding for fname. Skip re-probing it: walk from
+ * the parent scope. */
+ visible = scope_lookup_from(p->scope ? p->scope->parent : NULL, fname);
if (!existing && visible && visible->kind == SEK_FUNC) {
existing = visible;
}
@@ -1252,9 +1302,7 @@ static void parse_function_body(Parser* p, ObjSymId fsym, const Type* fn_ty,
s = pcg_param_slot(p, i, &fsd);
pds[i].slot = s;
if (infos[i].name) {
- reject_same_scope_redefinition(p, infos[i].name, SEK_LOCAL,
- infos[i].type);
- e = scope_define(p, infos[i].name, SEK_LOCAL, infos[i].type);
+ e = scope_define_checked(p, infos[i].name, SEK_LOCAL, infos[i].type);
e->v.slot = s;
sym_set_decl(e, DECL_NONE, DS_AUTO, DL_NONE, DF_NONE, DSTATE_DEFINED);
e->vla_bounds = build_param_vla_bounds(p, &infos[i], infos[i].loc);
@@ -1312,9 +1360,8 @@ static void parse_external_decl(Parser* p) {
if (is_punct(&p->cur, '=')) {
perr(p, "typedef declarator cannot have initializer");
}
- reject_same_scope_redefinition(p, tname, SEK_TYPEDEF, tty);
{
- SymEntry* te = scope_define(p, tname, SEK_TYPEDEF, tty);
+ SymEntry* te = scope_define_checked(p, tname, SEK_TYPEDEF, tty);
sym_set_decl(te, DECL_NONE, DS_TYPEDEF, DL_NONE, specs.flags,
DSTATE_DECLARED);
}
@@ -1587,9 +1634,11 @@ void parse_c(Compiler* c, Pool* pool, Pp* pp, DeclTable* decls, CG* cg,
/* Index storage for the scope/tag tables and the external-function table all
* comes from the arena via the pool's shared arena-heap facade. */
ExternalFuncMap_init(&p.external_funcs, &p.pool->arena_heap);
+ KwMap_init(&p.kw_map, &p.pool->arena_heap);
for (i = (CKw)1; i < KW_COUNT; ++i) {
p.kw_sym[i] = kit_sym_intern(p.pool->c, kit_slice_cstr(kw_names[i]));
+ (void)KwMap_set(&p.kw_map, p.kw_sym[i], (u8)i); /* canonical keyword */
}
p.sym_b_alloca = kit_sym_intern(p.pool->c, KIT_SLICE_LIT("__builtin_alloca"));
@@ -1674,6 +1723,16 @@ void parse_c(Compiler* c, Pool* pool, Pp* pp, DeclTable* decls, CG* cg,
p.sym_restrict_alias2 =
kit_sym_intern(p.pool->c, KIT_SLICE_LIT("__restrict__"));
p.sym_thread_alias = kit_sym_intern(p.pool->c, KIT_SLICE_LIT("__thread"));
+ /* GNU alias spellings -> their canonical CKw. Registered after the canonical
+ * keywords above so the canonical mapping always wins (matching the old
+ * ident_kw_inline fall-through, which scanned kw_sym[] before the aliases). */
+ (void)KwMap_set(&p.kw_map, p.sym_alignof_alias, (u8)KW_ALIGNOF);
+ (void)KwMap_set(&p.kw_map, p.sym_asm_alias, (u8)KW_BUILTIN_ASM);
+ (void)KwMap_set(&p.kw_map, p.sym_inline_alias, (u8)KW_INLINE);
+ (void)KwMap_set(&p.kw_map, p.sym_inline_alias2, (u8)KW_INLINE);
+ (void)KwMap_set(&p.kw_map, p.sym_restrict_alias, (u8)KW_RESTRICT);
+ (void)KwMap_set(&p.kw_map, p.sym_restrict_alias2, (u8)KW_RESTRICT);
+ (void)KwMap_set(&p.kw_map, p.sym_thread_alias, (u8)KW_THREAD_LOCAL);
p.sym_int128 = kit_sym_intern(p.pool->c, KIT_SLICE_LIT("__int128"));
p.sym_int128_t = kit_sym_intern(p.pool->c, KIT_SLICE_LIT("__int128_t"));
p.sym_uint128_t = kit_sym_intern(p.pool->c, KIT_SLICE_LIT("__uint128_t"));
diff --git a/lang/c/parse/parse_priv.h b/lang/c/parse/parse_priv.h
@@ -165,6 +165,13 @@ KIT_HASHMAP_DEFINE(SymEntryMap, Sym, SymEntry*, kit_hash_u32);
KIT_HASHMAP_DEFINE(TagEntryMap, Sym, TagEntry*, kit_hash_u32);
KIT_HASHMAP_DEFINE(ExternalFuncMap, Sym, SymEntry*, kit_hash_u32);
+/* Interned keyword/alias Sym -> CKw (stored as u8), built once in parse_c.
+ * Replaces the per-identifier linear scan of kw_sym[] (plus the alias compares)
+ * with a single O(1) probe: every non-keyword identifier (variable / typedef /
+ * field name) previously scanned all ~52 keyword syms before returning KW_NONE.
+ * Storage comes from the pool's shared arena-heap facade (Pool.arena_heap). */
+KIT_HASHMAP_DEFINE(KwMap, Sym, u8, kit_hash_u32);
+
typedef struct Scope Scope;
struct Scope {
SymEntry* entries; /* LIFO */
@@ -236,6 +243,7 @@ typedef struct Parser {
int has_pending;
Sym kw_sym[KW_COUNT];
+ KwMap kw_map; /* keyword/alias Sym -> CKw; built once, see ident_kw_inline */
Sym sym_b_alloca;
Sym sym_b_ctz;
@@ -434,18 +442,12 @@ static inline int is_punct(const Tok* t, u32 punct) {
static inline int is_pp_hash(const Tok* t) { return t->kind == TOK_PP_HASH; }
static inline CKw ident_kw_inline(const Parser* p, Sym name) {
- CKw i;
- for (i = (CKw)1; i < KW_COUNT; ++i) {
- if (p->kw_sym[i] == name) return i;
- }
- if (name == p->sym_alignof_alias) return KW_ALIGNOF;
- if (name == p->sym_asm_alias) return KW_BUILTIN_ASM;
- if (name == p->sym_inline_alias || name == p->sym_inline_alias2)
- return KW_INLINE;
- if (name == p->sym_restrict_alias || name == p->sym_restrict_alias2)
- return KW_RESTRICT;
- if (name == p->sym_thread_alias) return KW_THREAD_LOCAL;
- return KW_NONE;
+ /* O(1) classification via kw_map (canonical keywords + GNU alias spellings,
+ * populated once in parse_c with canonical-first ordering so a real keyword
+ * is never overwritten by an alias). Equivalent to the old linear scan of
+ * kw_sym[] followed by the explicit alias compares. */
+ const u8* v = name ? KwMap_get(&p->kw_map, name) : NULL;
+ return v ? (CKw)*v : KW_NONE;
}
static inline int is_kw(const Parser* p, const Tok* t, CKw k) {
diff --git a/lang/cpp/lex/lex.c b/lang/cpp/lex/lex.c
@@ -150,6 +150,34 @@ static int is_alpha(int c) {
}
static int is_alnum(int c) { return is_alpha(c) || is_digit(c); }
+/* Consume a maximal run of identifier-continuation bytes (is_alnum) from the
+ * cursor, advancing line/col exactly as a per-byte bump() would. No is_alnum
+ * byte is '\n', so within a run line changes only when a folded splice point is
+ * crossed. The common splice-free input (l->splices == NULL) collapses to a
+ * single pos/col advance with src/len held in registers — no per-byte struct
+ * reload and no per-byte splice load+branch in the hottest scan loop. Stops at
+ * the first non-alnum byte (a '\\' UCN lead, a punctuator, or end), which the
+ * caller re-examines (e.g. for a UCN continuation). */
+static void scan_ident_run(Lexer* l) {
+ const char* s = l->src;
+ size_t n = l->len;
+ if (l->splices == NULL) {
+ size_t p = l->pos;
+ while (p < n && is_alnum((unsigned char)s[p])) ++p;
+ l->col += (u32)(p - l->pos);
+ l->pos = p;
+ return;
+ }
+ /* Splice-present slow path: keep the per-byte cursor advance and the splice
+ * line/col sync bit-for-bit identical to bump() (is_alnum bytes are never
+ * '\n', so only lex_sync_splices can move the line). */
+ while (l->pos < n && is_alnum((unsigned char)s[l->pos])) {
+ l->pos++;
+ l->col++;
+ lex_sync_splices(l);
+ }
+}
+
/* Match a UCN at offset `off` from the current position. Returns the total
* length (6 for \uXXXX, 10 for \UXXXXXXXX), or 0 if no UCN matches. The
* range constraints from §6.4.3 (no UCN < 00A0 except $/@/`, and none in
@@ -357,7 +385,14 @@ Tok lex_next(Lexer* l) {
size_t start;
int ch;
- memset(&t, 0, sizeof(t));
+ /* No per-token memset: every content-token path assigns kind/loc/spelling/v,
+ * and flags is the only field accumulated via |=, so a deterministic zero
+ * base for flags (and the v union, which the number path leaves unwritten) is
+ * all the hot path needs. The cold EOF/NEWLINE early returns explicitly clear
+ * spelling so the returned token stays byte-identical to the old
+ * memset-then-fill token. */
+ t.flags = 0;
+ t.v.ident = 0;
/* Skip whitespace and comments. A newline token is emitted before any
* subsequent content tokens for the line that follows. */
@@ -366,6 +401,7 @@ Tok lex_next(Lexer* l) {
if (l->pos >= l->len) {
t.kind = TOK_EOF;
t.loc = lex_here(l);
+ t.spelling = 0;
return t;
}
if (peek(l, 0) == '\n') {
@@ -373,6 +409,7 @@ Tok lex_next(Lexer* l) {
bump(l);
t.kind = TOK_NEWLINE;
t.loc = tloc;
+ t.spelling = 0;
l->at_bol = 1;
l->had_space = 0;
l->dstate = 0;
@@ -486,10 +523,8 @@ Tok lex_next(Lexer* l) {
} else
bump(l);
for (;;) {
- int c = peek(l, 0);
- if (is_alnum(c)) {
- bump(l);
- } else if ((u = ucn_len(l, 0))) {
+ scan_ident_run(l); /* consume the maximal is_alnum run in one go */
+ if ((u = ucn_len(l, 0))) {
int i;
for (i = 0; i < u; ++i) bump(l);
} else {
diff --git a/lang/cpp/pp/pp.c b/lang/cpp/pp/pp.c
@@ -46,59 +46,80 @@ void src_pop(Pp* pp) {
--pp->nsources;
}
-/* Read next raw token from the top source. Returns TOK_EOF when stack is
- * empty. Pops empty buffer/lexer sources as it descends. `src_kind_out`,
- * if non-NULL, receives the kind of the source the token came from
- * (SRC_LEX vs SRC_BUF). Used by pp_next_raw to gate directive recognition
- * to lex-sourced tokens only — a `#` produced by macro expansion never
- * starts a directive (§6.10.3.4 ¶3, covered by `63_rescan_not_directive`). */
-Tok src_next_raw(Pp* pp, HidesetId* hs_out, u8* src_kind_out) {
- Tok t;
+/* Read next raw token from the top source, writing it through `out`. Sets
+ * *out to TOK_EOF when the stack is empty. Pops empty buffer/lexer sources
+ * as it descends. `src_kind_out`, if non-NULL, receives the kind of the
+ * source the token came from (SRC_LEX vs SRC_BUF). Used by pp_next_raw to
+ * gate directive recognition to lex-sourced tokens only — a `#` produced by
+ * macro expansion never starts a directive (§6.10.3.4 ¶3, covered by
+ * `63_rescan_not_directive`).
+ *
+ * This is the out-pointer form: lex_next already returns via sret, so it
+ * writes straight into the caller's slot with no inter-frame copy, and the
+ * EOF/empty-stack fallthrough is the only path that has to zero *out. */
+void src_next_raw_into(Pp* pp, Tok* out, HidesetId* hs_out, u8* src_kind_out) {
TokSrc* s;
while ((s = src_top(pp)) != NULL) {
if (s->kind == SRC_BUF) {
if (s->i < s->n) {
- t = s->toks[s->i];
+ *out = s->toks[s->i];
if (hs_out) *hs_out = s->hs ? s->hs[s->i] : HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_BUF;
++s->i;
- return t;
+ return;
}
if (s->scope_top) {
- memset(&t, 0, sizeof(t));
- t.kind = TOK_EOF;
+ /* Explicit field zeroing in lieu of memset: byte-identical to the
+ * old memset-then-kind EOF token, minus the per-EOF libcall. */
+ out->kind = TOK_EOF;
+ out->flags = 0;
+ out->loc = (SrcLoc){0};
+ out->spelling = 0;
+ out->v.ident = 0;
if (hs_out) *hs_out = HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_BUF;
- return t;
+ return;
}
src_pop(pp);
continue;
}
/* SRC_LEX */
- t = lex_next(s->lex);
- if (t.kind == TOK_EOF) {
+ *out = lex_next(s->lex);
+ if (out->kind == TOK_EOF) {
if (pp->nsources > 1) {
src_pop(pp);
continue;
}
if (hs_out) *hs_out = HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_LEX;
- return t;
+ return;
}
/* Apply #line line-number delta on the way out so the rest of
* the pipeline sees user-visible line numbers (matters for
* __LINE__ expansion and for line-tracking output cursors). */
if (s->line_delta) {
- t.loc.line = (u32)((i32)t.loc.line + s->line_delta);
+ out->loc.line = (u32)((i32)out->loc.line + s->line_delta);
}
if (hs_out) *hs_out = HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_LEX;
- return t;
+ return;
}
- memset(&t, 0, sizeof(t));
- t.kind = TOK_EOF;
+ /* Explicit field zeroing in lieu of memset: byte-identical to the old
+ * memset-then-kind EOF token, minus the per-EOF libcall. */
+ out->kind = TOK_EOF;
+ out->flags = 0;
+ out->loc = (SrcLoc){0};
+ out->spelling = 0;
+ out->v.ident = 0;
if (hs_out) *hs_out = HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_LEX;
+}
+
+/* Thin by-value shim for the cold/general callers (arg collection, paren
+ * peek) that pass a NULL src_kind and don't sit in the hot -E loop. */
+Tok src_next_raw(Pp* pp, HidesetId* hs_out, u8* src_kind_out) {
+ Tok t;
+ src_next_raw_into(pp, &t, hs_out, src_kind_out);
return t;
}
@@ -131,13 +152,16 @@ Tok pp_next(Pp* pp) {
* tokens as stray identifiers. When we see TOK_PP_HASH followed by
* `pragma`, swallow tokens through the next NEWLINE. */
for (;;) {
- Tok t = pp_next_raw(pp);
+ Tok t;
+ pp_next_raw_into(pp, &t);
if (t.kind == TOK_NEWLINE) continue;
if (t.kind == TOK_PP_HASH) {
- Tok t2 = pp_next_raw(pp);
+ Tok t2;
+ pp_next_raw_into(pp, &t2);
if (t2.kind == TOK_IDENT && t2.v.ident == pp->sym_pragma) {
for (;;) {
- Tok tt = pp_next_raw(pp);
+ Tok tt;
+ pp_next_raw_into(pp, &tt);
if (tt.kind == TOK_NEWLINE || tt.kind == TOK_EOF) break;
}
continue;
@@ -189,7 +213,8 @@ void pp_emit_text(Pp* pp, Writer* out) {
size_t on = 0;
int at_bol = 1;
for (;;) {
- Tok t = pp_next_raw(pp);
+ Tok t;
+ pp_next_raw_into(pp, &t);
if (t.kind == TOK_EOF) break;
if (t.kind == TOK_NEWLINE) {
pp_emit_stage(out, obuf, sizeof obuf, &on, "\n", 1);
diff --git a/lang/cpp/pp/pp_directive.c b/lang/cpp/pp/pp_directive.c
@@ -585,6 +585,45 @@ static void do_endif(Pp* pp, SrcLoc loc) {
* #include (§6.10.2)
* ============================================================ */
+/* Arena-heap adapter backing the IncCache slot table: alloc/realloc bump
+ * from pp->arena (immortal until pp_free), free is a no-op. A resize
+ * orphans the small old slot table in the arena — bounded by the final
+ * table size and negligible. Mirrors the parser's c_arena_heap. */
+static void* inc_cache_alloc(KitHeap* h, size_t n, size_t align) {
+ return kit_arena_alloc((KitArena*)h->user, n, align);
+}
+static void* inc_cache_realloc(KitHeap* h, void* old, size_t old_n,
+ size_t new_n, size_t align) {
+ void* q = kit_arena_alloc((KitArena*)h->user, new_n, align);
+ if (q && old && old_n) memcpy(q, old, old_n < new_n ? old_n : new_n);
+ return q;
+}
+static void inc_cache_free(KitHeap* h, void* p, size_t n) {
+ (void)h;
+ (void)p;
+ (void)n;
+}
+
+/* Lazily set up the header-content cache, resolution memo, and their
+ * shared arena-backed heap. */
+static void inc_cache_ensure(Pp* pp) {
+ if (pp->inc_cache_ready) return;
+ pp->inc_cache_heap.alloc = inc_cache_alloc;
+ pp->inc_cache_heap.realloc = inc_cache_realloc;
+ pp->inc_cache_heap.free = inc_cache_free;
+ pp->inc_cache_heap.user = pp->arena;
+ IncCache_init(&pp->inc_cache, &pp->inc_cache_heap);
+ IncResolveMap_init(&pp->inc_resolve, &pp->inc_cache_heap);
+ pp->inc_cache_ready = 1;
+}
+
+/* Intern a candidate/resolved path string into a Sym usable as a cache
+ * key. Returns 0 (an invalid key) only for the empty string, which never
+ * names a real header. */
+static Sym inc_cache_key(Pp* pp, const char* path) {
+ return kit_sym_intern(pp->pool->c, kit_slice_cstr(path));
+}
+
/* Read `path` via the host's file_io and copy its bytes into the pp
* arena so they outlive io->release. Returns 1 on success. */
static int try_open_include(Pp* pp, const char* path, const u8** data_out,
@@ -611,6 +650,36 @@ static int try_open_include(Pp* pp, const char* path, const u8** data_out,
return 1;
}
+/* Open `path` for #include, serving from the header-content cache when the
+ * same resolved path was already read this TU. A hit returns the cached
+ * arena bytes with NO syscalls; a miss reads via try_open_include and
+ * caches the result keyed on the path string. The returned (data,size) is
+ * byte-identical to a fresh read, so this never affects which file wins a
+ * search nor the bytes that feed the lexer. */
+static int open_include_cached(Pp* pp, const char* path, const u8** data_out,
+ size_t* size_out) {
+ Sym key;
+ IncEntry* hit;
+ inc_cache_ensure(pp);
+ key = inc_cache_key(pp, path);
+ if (key) {
+ hit = IncCache_get(&pp->inc_cache, key);
+ if (hit) {
+ *data_out = hit->data;
+ *size_out = hit->size;
+ return 1;
+ }
+ }
+ if (!try_open_include(pp, path, data_out, size_out)) return 0;
+ if (key) {
+ IncEntry e;
+ e.data = *data_out;
+ e.size = *size_out;
+ IncCache_set(&pp->inc_cache, key, e);
+ }
+ return 1;
+}
+
/* Return the includer's directory for resolving a quoted include, or "."
* for in-memory/builtin sources (where CWD is the natural fallback, like
* gcc treats stdin). `dir_out` must point to a buffer of size >= cap. */
@@ -656,10 +725,59 @@ static int includer_dir(Pp* pp, SrcLoc loc, char* dir_out, size_t cap) {
return 1;
}
+/* Build the resolution-memo key for a (spelling, system, includer-dir)
+ * request and intern it. The key folds the search-affecting inputs into
+ * one string: a leading '<' (system) or '"' (quoted) tag, and — for the
+ * quoted form, whose includer-relative step is includer-specific — the
+ * includer directory followed by a '\n' separator (which can appear in
+ * neither a path nor a header spelling) before the spelling. System form
+ * skips the includer step, so its key omits the dir. Returns 0 only if
+ * the key would overflow the scratch buffer (memo simply not used). */
+static Sym inc_resolve_key(Pp* pp, const char* path, size_t plen, int system,
+ SrcLoc loc) {
+ char key[4096 + 4096 + 8];
+ size_t pos = 0;
+ key[pos++] = system ? '<' : '"';
+ if (!system) {
+ char dir[4096];
+ if (!includer_dir(pp, loc, dir, sizeof(dir))) return 0;
+ {
+ size_t dlen = kit_slice_cstr(dir).len;
+ if (pos + dlen + 1 + plen >= sizeof(key)) return 0;
+ memcpy(key + pos, dir, dlen);
+ pos += dlen;
+ key[pos++] = '\n';
+ }
+ } else if (pos + plen >= sizeof(key)) {
+ return 0;
+ }
+ memcpy(key + pos, path, plen);
+ pos += plen;
+ return kit_sym_intern(pp->pool->c,
+ (KitSlice){.s = key, .len = pos});
+}
+
+/* Record a successful dir-search resolution under its spelling key, so the
+ * next request for the same spelling skips the search. No-op when the key
+ * was unavailable (rkey == 0). `resolved` is a NUL-terminated path. */
+static void inc_resolve_record(Pp* pp, Sym rkey, const char* resolved,
+ int resolved_system) {
+ IncResolved e;
+ if (!rkey) return;
+ e.path = kit_sym_intern(pp->pool->c, kit_slice_cstr(resolved));
+ e.system = (u8)(resolved_system ? 1 : 0);
+ IncResolveMap_set(&pp->inc_resolve, rkey, e);
+}
+
/* Search for a header. Absolute paths are opened verbatim. Quoted form
* ("...") additionally searches the directory of the file containing the
* #include first (per C §6.10.2); bracket form (<...>) skips that step.
- * Both forms then walk the configured -I / -isystem dirs in order. */
+ * Both forms then walk the configured -I / -isystem dirs in order.
+ *
+ * A resolution memo short-circuits the repeat case: when the same spelling
+ * (with the same system flag and, for quoted form, the same includer dir)
+ * was resolved before, we go straight to the winning path — skipping the
+ * dir-by-dir ENOENT storm — and the content cache serves its bytes. */
static int find_and_open_include(Pp* pp, const char* path, int system,
SrcLoc loc, const u8** data, size_t* size,
char* resolved, size_t resolved_cap,
@@ -667,13 +785,14 @@ static int find_and_open_include(Pp* pp, const char* path, int system,
char buf[4096];
u32 i;
size_t plen = kit_slice_cstr(path).len;
+ Sym rkey = 0;
/* Absolute paths and the includer-relative ("...") step are not system
* search dirs; only a configured -isystem dir flips this to 1 below. */
*resolved_system_out = 0;
if (plen > 0 && path[0] == '/') {
- if (try_open_include(pp, path, data, size)) {
+ if (open_include_cached(pp, path, data, size)) {
if (plen + 1 > resolved_cap) return 0;
memcpy(resolved, path, plen + 1);
return 1;
@@ -681,6 +800,24 @@ static int find_and_open_include(Pp* pp, const char* path, int system,
return 0;
}
+ /* Probe the resolution memo (covers only the dir-search winners below;
+ * absolute paths return above). On a hit, reopen the recorded resolved
+ * path — which the content cache already holds, so this is syscall-free
+ * — and reproduce the byte-identical resolved string + system flag. */
+ inc_cache_ensure(pp);
+ rkey = inc_resolve_key(pp, path, plen, system, loc);
+ if (rkey) {
+ IncResolved* hit = IncResolveMap_get(&pp->inc_resolve, rkey);
+ if (hit) {
+ KitSlice rp = kit_sym_str(pp->pool->c, hit->path);
+ if (rp.len + 1 > resolved_cap) return 0;
+ if (!open_include_cached(pp, rp.s, data, size)) return 0;
+ memcpy(resolved, rp.s, rp.len + 1);
+ *resolved_system_out = hit->system ? 1 : 0;
+ return 1;
+ }
+ }
+
if (!system) {
char dir[4096];
if (includer_dir(pp, loc, dir, sizeof(dir))) {
@@ -690,9 +827,10 @@ static int find_and_open_include(Pp* pp, const char* path, int system,
buf[dlen] = '/';
memcpy(buf + dlen + 1, path, plen);
buf[dlen + 1 + plen] = 0;
- if (try_open_include(pp, buf, data, size)) {
+ if (open_include_cached(pp, buf, data, size)) {
if (dlen + 1 + plen + 1 > resolved_cap) return 0;
memcpy(resolved, buf, dlen + 1 + plen + 1);
+ inc_resolve_record(pp, rkey, resolved, 0);
return 1;
}
}
@@ -706,10 +844,11 @@ static int find_and_open_include(Pp* pp, const char* path, int system,
buf[dlen] = '/';
memcpy(buf + dlen + 1, path, plen);
buf[dlen + 1 + plen] = 0;
- if (try_open_include(pp, buf, data, size)) {
+ if (open_include_cached(pp, buf, data, size)) {
if (dlen + 1 + plen + 1 > resolved_cap) return 0;
memcpy(resolved, buf, dlen + 1 + plen + 1);
*resolved_system_out = pp->inc_dirs[i].system ? 1 : 0;
+ inc_resolve_record(pp, rkey, resolved, *resolved_system_out);
return 1;
}
}
diff --git a/lang/cpp/pp/pp_expand.c b/lang/cpp/pp/pp_expand.c
@@ -427,7 +427,7 @@ void expand_arg_to_eof(Pp* pp, Tok* in, HidesetId* hs, u32 nin, TokVec* out) {
src_push(pp, src);
for (;;) {
- t = pp_next_raw(pp); /* drives macro expansion within this scope */
+ pp_next_raw_into(pp, &t); /* drives macro expansion within this scope */
if (t.kind == TOK_EOF) break;
if (t.kind == TOK_NEWLINE) {
/* Newlines inside an arg act as whitespace; convert to
@@ -793,24 +793,35 @@ static void subst_phase1(Pp* pp, const Macro* m, ArgList* a, const Tok* invoke,
static void subst_phase2(Pp* pp, const Tok* in, u32 nin, const Tok* invoke,
TokVec* out) {
u32 i;
+ /* Track whether any placemarker actually lands in `out`. The strip pass
+ * below is a full O(out->n) re-walk + element-wise compaction; the common
+ * macro body has no empty-arg/paste placemarkers, so when none was emitted
+ * we skip the second walk entirely. */
+ int had_pm = 0;
/* Phase-2 output is at most the input length (paste/placemarker only
* shrink); reserve it to avoid re-growing from cap 0. */
tv_grow(pp, out, nin);
for (i = 0; i < nin; ++i) {
Tok t = in[i];
if (t.kind == TOK_PP_PASTE) {
- Tok lhs, rhs;
+ Tok lhs, rhs, pasted;
if (out->n == 0 || i + 1 >= nin) {
compiler_panic(pp->c, invoke->loc,
"'##' at start or end of replacement list");
}
lhs = out->data[--out->n];
rhs = in[++i];
- tv_push(pp, out, paste_tokens(pp, lhs, rhs, invoke->loc));
+ /* paste_tokens collapses a placemarker operand to the other side, so
+ * the result is a placemarker only when both operands were. */
+ pasted = paste_tokens(pp, lhs, rhs, invoke->loc);
+ if (pasted.kind == TOK_PP_PLACEMARKER) had_pm = 1;
+ tv_push(pp, out, pasted);
continue;
}
+ if (t.kind == TOK_PP_PLACEMARKER) had_pm = 1;
tv_push(pp, out, t);
}
+ if (!had_pm) return; /* no placemarker to strip — skip the compaction walk */
/* Strip placemarkers, preserving leading-space flag on the next token. */
{
u32 r = 0, w = 0;
@@ -894,14 +905,21 @@ static int try_expand_func_macro(Pp* pp, const Macro* m, const Tok* invoke,
* Defined here; also declared in pp_priv.h so pp.c can call it.
* ============================================================ */
-/* pp_next_raw: reads from the top source, applies macro expansion when an
- * identifier names a macro that isn't blue-painted, and consumes
- * directives in-place. TOK_NEWLINE is preserved for pp_emit_text. */
-Tok pp_next_raw(Pp* pp) {
- Tok t;
+/* pp_next_raw_into: out-pointer form. Reads from the top source into *out,
+ * applies macro expansion when an identifier names a macro that isn't
+ * blue-painted, and consumes directives in-place. TOK_NEWLINE is preserved
+ * for pp_emit_text.
+ *
+ * Tok is 24B, so the by-value form (pp_next_raw, below) returns indirectly
+ * via sret and pays a 24B inter-frame copy at every `return t`. Writing
+ * through `out` lets the token — and lex_next's own sret — land in the
+ * caller's slot directly. The hot SRC_BUF case (macro replay) is inlined
+ * here so the common token never pays a call into src_next_raw_into. */
+void pp_next_raw_into(Pp* pp, Tok* out) {
HidesetId hs;
u8 src_kind;
for (;;) {
+ TokSrc* s;
/* Reclaim the transient expansion scratch whenever the source stack has
* drained back to a lexer. A SRC_BUF always sits above every SRC_LEX (a
* macro can't #include), so a lexer on top means no expansion buffer is
@@ -916,10 +934,25 @@ Tok pp_next_raw(Pp* pp) {
pp->sources[pp->nsources - 1].kind == SRC_LEX)) {
kit_arena_reset(pp->xarena);
}
- t = src_next_raw(pp, &hs, &src_kind);
- if (t.kind == TOK_EOF) return t;
- if (t.kind == TOK_PP_HASH && (t.flags & TF_AT_BOL) && src_kind == SRC_LEX) {
- process_directive(pp, t.loc);
+ /* Fast path: top source is a non-exhausted SRC_BUF (the dominant
+ * macro-replay case). Pull the token, hideset, and kind inline so the
+ * common token avoids the call + sret setup of src_next_raw_into. The
+ * scope_top-EOF and #line-delta cases never apply to a plain in-bounds
+ * SRC_BUF read, so they stay on the general (cold) path. */
+ if (pp->nsources != 0 &&
+ (s = &pp->sources[pp->nsources - 1])->kind == SRC_BUF &&
+ s->i < s->n) {
+ *out = s->toks[s->i];
+ hs = s->hs ? s->hs[s->i] : HS_EMPTY;
+ src_kind = SRC_BUF;
+ ++s->i;
+ } else {
+ src_next_raw_into(pp, out, &hs, &src_kind);
+ }
+ if (out->kind == TOK_EOF) return;
+ if (out->kind == TOK_PP_HASH && (out->flags & TF_AT_BOL) &&
+ src_kind == SRC_LEX) {
+ process_directive(pp, out->loc);
/* No synthesized newline: the comparator collapses
* whitespace, so blank-line replacement of consumed
* directives isn't observable here. Directives that produce
@@ -935,32 +968,32 @@ Tok pp_next_raw(Pp* pp) {
* the `defined_skip` field comment in pp_priv.h. */
if (pp->in_if_expansion) {
if (pp->defined_skip == 2) {
- if (t.kind == TOK_PUNCT && t.v.punct == '(') {
+ if (out->kind == TOK_PUNCT && out->v.punct == '(') {
pp->defined_skip = 3;
- } else if (t.kind == TOK_IDENT) {
+ } else if (out->kind == TOK_IDENT) {
/* `defined IDENT` (no parens): mark the operand and reset. */
- t.flags |= TF_NO_EXPAND;
+ out->flags |= TF_NO_EXPAND;
pp->defined_skip = 0;
} else {
pp->defined_skip = 0;
}
} else if (pp->defined_skip == 3) {
- if (t.kind == TOK_IDENT) {
- t.flags |= TF_NO_EXPAND;
+ if (out->kind == TOK_IDENT) {
+ out->flags |= TF_NO_EXPAND;
pp->defined_skip = 4;
- } else if (t.kind == TOK_PUNCT && t.v.punct == ')') {
+ } else if (out->kind == TOK_PUNCT && out->v.punct == ')') {
pp->defined_skip = 0;
}
} else if (pp->defined_skip == 4) {
- if (t.kind == TOK_PUNCT && t.v.punct == ')') {
+ if (out->kind == TOK_PUNCT && out->v.punct == ')') {
pp->defined_skip = 0;
}
- } else if (t.kind == TOK_IDENT && t.v.ident == pp->sym_defined) {
+ } else if (out->kind == TOK_IDENT && out->v.ident == pp->sym_defined) {
pp->defined_skip = 2;
}
}
- if (t.kind == TOK_IDENT && (t.flags & TF_NO_EXPAND) == 0) {
- Sym id = t.v.ident;
+ if (out->kind == TOK_IDENT && (out->flags & TF_NO_EXPAND) == 0) {
+ Sym id = out->v.ident;
/* Dynamic predefined macros: __LINE__ / __FILE__ /
* __DATE__ / __TIME__. Always expand, ignoring the macro
@@ -968,7 +1001,7 @@ Tok pp_next_raw(Pp* pp) {
if (id == pp->sym_line__) {
char tmp[16], buf[16];
int k = 0, j = 0;
- u32 ln = t.loc.line;
+ u32 ln = out->loc.line;
if (ln == 0)
buf[k++] = '0';
else {
@@ -978,10 +1011,10 @@ Tok pp_next_raw(Pp* pp) {
}
while (j > 0) buf[k++] = tmp[--j];
}
- t.kind = TOK_NUM;
- t.spelling =
+ out->kind = TOK_NUM;
+ out->spelling =
kit_sym_intern(pp->pool->c, (KitSlice){.s = buf, .len = (size_t)k});
- return t;
+ return;
}
if (id == pp->sym_file__) {
TokSrc* ls = current_lex_src(pp);
@@ -1020,27 +1053,27 @@ Tok pp_next_raw(Pp* pp) {
buf[bn++] = ch;
}
buf[bn++] = '"';
- t.kind = TOK_STR;
- t.spelling =
+ out->kind = TOK_STR;
+ out->spelling =
kit_sym_intern(pp->pool->c, (KitSlice){.s = buf, .len = bn});
- t.v.str = t.spelling;
+ out->v.str = out->spelling;
}
- return t;
+ return;
}
if (id == pp->sym_date__) {
- t.kind = TOK_STR;
- t.spelling = pp->val_date_str;
- t.v.str = t.spelling;
- return t;
+ out->kind = TOK_STR;
+ out->spelling = pp->val_date_str;
+ out->v.str = out->spelling;
+ return;
}
if (id == pp->sym_time__) {
- t.kind = TOK_STR;
- t.spelling = pp->val_time_str;
- t.v.str = t.spelling;
- return t;
+ out->kind = TOK_STR;
+ out->spelling = pp->val_time_str;
+ out->v.str = out->spelling;
+ return;
}
if (id == pp->sym__pragma) {
- if (try_expand_pragma_op(pp, &t)) continue;
+ if (try_expand_pragma_op(pp, out)) continue;
/* No '(' — fall through and emit as plain ident. */
}
@@ -1048,16 +1081,24 @@ Tok pp_next_raw(Pp* pp) {
Macro* m = mt_get(pp, id);
if (m && !hs_contains(pp, hs, m->name)) {
if (!m->is_func) {
- expand_object_macro(pp, m, &t, hs);
+ expand_object_macro(pp, m, out, hs);
continue;
}
- if (try_expand_func_macro(pp, m, &t, hs)) {
+ if (try_expand_func_macro(pp, m, out, hs)) {
continue;
}
/* No '(' followed; emit as plain identifier. */
}
}
}
- return t;
+ return;
}
}
+
+/* Thin by-value shim: the mutual-recursion entry (expand_arg_to_eof) and any
+ * external caller use this; the hot -E loops call pp_next_raw_into directly. */
+Tok pp_next_raw(Pp* pp) {
+ Tok t;
+ pp_next_raw_into(pp, &t);
+ return t;
+}
diff --git a/lang/cpp/pp/pp_priv.h b/lang/cpp/pp/pp_priv.h
@@ -87,6 +87,34 @@ typedef struct IfFrame {
static inline u32 macro_hash_(Sym s) { return kit_hash_u32((u32)s); }
KIT_HASHMAP_DEFINE(MacroMap, Sym, Macro*, macro_hash_);
+/* IncCache = Sym(resolved-path) -> cached header bytes. The same guarded
+ * header included many times across a TU is read from disk once; later
+ * inclusions reuse the arena-resident bytes (lex_open_mem only borrows
+ * them and the pp arena keeps them alive until pp_free), saving the
+ * open+fstat+read+close + malloc/memcpy/free per repeat. Keyed strictly
+ * on the resolved-path STRING so distinct spellings/symlinks never
+ * over-merge; the cached bytes are byte-identical to a fresh read. */
+typedef struct IncEntry {
+ const u8* data;
+ size_t size;
+} IncEntry;
+static inline u32 inc_hash_(Sym s) { return kit_hash_u32((u32)s); }
+KIT_HASHMAP_DEFINE(IncCache, Sym, IncEntry, inc_hash_);
+
+/* IncResolveMap = Sym(spelling key) -> resolved (path Sym, system flag).
+ * Memoizes the winner of the -I/-isystem dir search so a repeated request
+ * for the same header spelling skips the ENOENT storm of failed opens on
+ * the earlier dirs. The key folds the system flag (and, for quoted form,
+ * the includer directory) into the spelling so the resolved result is the
+ * exact one a fresh search would produce — byte-identical resolved path
+ * and resolved_system flag feeding source_add_include / DWARF / -M. */
+typedef struct IncResolved {
+ Sym path; /* interned resolved path string */
+ u8 system; /* the resolved_system flag (winning -isystem) */
+} IncResolved;
+static inline u32 incres_hash_(Sym s) { return kit_hash_u32((u32)s); }
+KIT_HASHMAP_DEFINE(IncResolveMap, Sym, IncResolved, incres_hash_);
+
/* ============================================================
* Pp struct (definition shared across all three TUs)
* ============================================================ */
@@ -103,6 +131,19 @@ struct Pp {
/* Macro table (open-addressed; key = Sym, value = Macro*). */
MacroMap mtab;
+ /* Header-content cache (key = interned resolved-path Sym, value =
+ * arena-resident bytes). Collapses N re-includes of the same header to
+ * one read + N-1 cache hits. Lazily initialized on first #include; its
+ * slot table is backed by inc_cache_heap (an arena adapter over
+ * pp->arena), so nothing needs freeing at pp_free — the arena is. */
+ IncCache inc_cache;
+ KitHeap inc_cache_heap;
+ u8 inc_cache_ready;
+
+ /* Include-resolution memo (spelling -> winning resolved path + system
+ * flag). Shares inc_cache_heap (gated by inc_cache_ready). */
+ IncResolveMap inc_resolve;
+
/* Conditional inclusion stack (#if / #ifdef / #ifndef → #endif). */
IfFrame* ifstk;
u32 ifstk_n;
@@ -300,6 +341,9 @@ static inline void cb_putc(Pp* pp, CharBuf* b, char c) {
* ============================================================ */
/* --- pp.c (source stack) → pp_expand.c, pp_directive.c --- */
+/* Out-pointer form (hot path, no 24B sret round-trip); src_next_raw is the
+ * by-value shim over it for the cold/general callers. */
+void src_next_raw_into(Pp* pp, Tok* out, HidesetId* hs_out, u8* src_kind_out);
Tok src_next_raw(Pp* pp, HidesetId* hs_out, u8* src_kind_out);
void src_push(Pp* pp, TokSrc s);
void src_pop(Pp* pp);
@@ -307,7 +351,10 @@ void push_buf(Pp* pp, Tok* toks, HidesetId* hs, u32 n);
/* pp_next_raw is the mutual-recursion entry: expand_arg_to_eof calls it,
* and pp_next_raw drives directives and expansion. Declared non-static so
- * pp_expand.c can call it without a forward decl each time. */
+ * pp_expand.c can call it without a forward decl each time. pp_next_raw_into
+ * is the out-pointer form the hot -E loops call directly; pp_next_raw is the
+ * by-value shim. */
+void pp_next_raw_into(Pp* pp, Tok* out);
Tok pp_next_raw(Pp* pp);
/* --- pp_expand.c → pp.c, pp_directive.c --- */
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -2839,8 +2839,14 @@ static void aa_plan_call(NativeTarget* t, const NativeCallDesc* desc,
NativeCallPlan* plan) {
NativeCallPlanRet* rets;
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
+ /* Right-size the result scratch to the exact number of entries the ret loops
+ * below write: nparts on a DIRECT register return, 1 on the !abi fallback,
+ * 0 (NULL) otherwise (IGNORE / sret / no results). */
+ u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
+ ? abi->ret.nparts
+ : ((!abi && desc->nresults) ? 1u : 0u);
memset(plan, 0, sizeof *plan);
- rets = desc->nresults ? arena_zarray(t->c->tu, NativeCallPlanRet, 4) : NULL;
+ rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPlanRet, nrets_cap) : NULL;
plan->callee = desc->callee;
plan->rets = rets;
plan->flags = desc->flags;
@@ -3058,7 +3064,6 @@ static void aa_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
NativeCallPlanRet* rets = NULL;
u32 nr = 0;
- if (value) rets = arena_zarray(t->c->tu, NativeCallPlanRet, 4);
if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
AANativeTarget* a = aa_of(t);
/* Hold the sret destination pointer in x8, not AA_TMP1: aa_copy_bytes
@@ -3088,6 +3093,7 @@ static void aa_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
}
if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
u32 ni = 0, nf = 0;
+ rets = arena_zarray(t->c->tu, NativeCallPlanRet, abi->ret.nparts);
for (u32 p = 0; p < abi->ret.nparts; ++p) {
const ABIArgPart* part = &abi->ret.parts[p];
NativeAllocClass cls =
@@ -3110,6 +3116,7 @@ static void aa_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
nr++;
}
} else if (value) {
+ rets = arena_zarray(t->c->tu, NativeCallPlanRet, 1);
rets[0].src = *value;
rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, 0);
rets[0].mem = aa_mem_for_type(t, value->type, 0);
diff --git a/src/arch/mc.c b/src/arch/mc.c
@@ -144,7 +144,10 @@ static void labels_grow(MCImpl* mc, u32 want) {
while (ncap < want) ncap *= 2;
MCLabelInfo* nbuf = arena_array(mc->arena, MCLabelInfo, ncap);
if (mc->labels) memcpy(nbuf, mc->labels, sizeof(MCLabelInfo) * mc->nlabels);
- memset(nbuf + mc->nlabels, 0, sizeof(MCLabelInfo) * (ncap - mc->nlabels));
+ /* The grown tail is left uninitialized: m_label_new fully assigns every
+ * field of the one slot it hands out before any consumer indexes it, and
+ * nothing ever reads labels[i] for i >= nlabels (every access guards on
+ * id < nlabels and rejects MC_LABEL_NONE). */
mc->labels = nbuf;
mc->cap = ncap;
}
@@ -233,6 +236,10 @@ ObjSymId mc_label_symbol(MCEmitter* m, MCLabel id) {
static void m_set_section(MCEmitter* m, u32 section_id) {
m->section_id = section_id;
+ /* Cache the active section's byte buffer so the hot emit path avoids the
+ * per-instruction Sections_at deref + nobits branch. NULL for NOBITS/.bss
+ * (or none): emit then falls back to obj_write for bss_size accounting. */
+ m->cur_bytes = obj_section_bytes(m->obj, section_id);
}
static u32 m_pos(MCEmitter* m) { return obj_pos(m->obj, m->section_id); }
@@ -289,7 +296,13 @@ static void m_label_place(MCEmitter* m, MCLabel id) {
}
static void m_emit_bytes(MCEmitter* m, const u8* data, size_t n) {
- obj_write(m->obj, m->section_id, data, n);
+ /* Fast path: append straight to the cached section buffer (inlined
+ * buf_write). cur_bytes is NULL for NOBITS/.bss/none, where obj_write does
+ * the bss_size accounting instead. */
+ if (m->cur_bytes)
+ buf_write(m->cur_bytes, data, n);
+ else
+ obj_write(m->obj, m->section_id, data, n);
}
static void m_emit_fill(MCEmitter* m, size_t n, u8 byte) {
diff --git a/src/arch/mc.h b/src/arch/mc.h
@@ -47,6 +47,13 @@ struct MCEmitter {
Compiler* c;
ObjBuilder* obj;
u32 section_id;
+ /* Byte buffer of the active section, resolved once per set_section so the
+ * per-instruction emit_bytes path skips obj_write's Sections_at deref +
+ * nobits branch. NULL when the active section is NOBITS/.bss (or none): emit
+ * then falls back to obj_write so bss_size accounting still runs. The pointer
+ * is stable across emits (SegVec elements don't move); only set_section
+ * re-points it. */
+ Buf* cur_bytes;
/* Pending source location, updated by set_loc. Promoted to the base so
* arch backends' emit-bytes choke point can read it without reaching
diff --git a/src/arch/riscv/native.c b/src/arch/riscv/native.c
@@ -2240,8 +2240,14 @@ static void rv_plan_call(NativeTarget* t, const NativeCallDesc* desc,
const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
NativeCallPlanRet* rets;
KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
+ /* Right-size the result scratch to the exact number of entries the ret loops
+ * below write: nparts on a DIRECT register return, 1 on the !abi fallback,
+ * 0 (NULL) otherwise (IGNORE / sret / no results). */
+ u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
+ ? abi->ret.nparts
+ : ((!abi && desc->nresults) ? 1u : 0u);
memset(plan, 0, sizeof *plan);
- rets = desc->nresults ? arena_zarray(t->c->tu, NativeCallPlanRet, 4) : NULL;
+ rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPlanRet, nrets_cap) : NULL;
plan->callee = desc->callee;
plan->rets = rets;
plan->flags = desc->flags;
diff --git a/src/arch/x64/native.c b/src/arch/x64/native.c
@@ -2504,8 +2504,14 @@ static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
NativeCallPlanRet* rets;
KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
u32 c;
+ /* Right-size the result scratch to the exact number of entries the ret loops
+ * below write: nparts on a DIRECT register return, 1 on the !abi fallback,
+ * 0 (NULL) otherwise (IGNORE / sret / no results). */
+ u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
+ ? abi->ret.nparts
+ : ((!abi && desc->nresults) ? 1u : 0u);
memset(plan, 0, sizeof *plan);
- rets = desc->nresults ? arena_zarray(t->c->tu, NativeCallPlanRet, 4) : NULL;
+ rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPlanRet, nrets_cap) : NULL;
plan->callee = desc->callee;
plan->rets = rets;
plan->flags = desc->flags;
diff --git a/src/cg/arith.c b/src/cg/arith.c
@@ -35,11 +35,17 @@ void api_cg_binop(KitCg* g, BinOp iop, u32 flags) {
Operand dst;
ApiSValue folded_sv;
i64 folded;
+ int can_delay;
if (!g) return;
T = g->target;
b = api_pop(g);
a = api_pop(g);
ty = a.type ? a.type : b.type;
+ /* Delayability is a pure function of (ty, flags), neither of which changes
+ * below (strength-reduce rewrites the op and operands, not the type), so
+ * classify the foldable int once instead of re-deriving it at each of the
+ * three delay gates. */
+ can_delay = api_can_delay_int_arith(g, ty, flags);
if (!flags && api_sv_op_is(&a, OPK_IMM) && api_sv_op_is(&b, OPK_IMM) &&
api_try_fold_int_binop(g, iop, ty, a.op.v.imm, b.op.v.imm, &folded)) {
@@ -54,7 +60,7 @@ void api_cg_binop(KitCg* g, BinOp iop, u32 flags) {
* identity / fallback machinery as any other shift or and. */
if (!flags) api_try_strength_reduce(g, &iop, ty, &a, &b);
- if (api_can_delay_int_arith(g, ty, flags) &&
+ if (can_delay &&
api_try_fold_arith_chain(g, iop, ty, &a, &b, &folded_sv)) {
api_release(g, &a);
api_release(g, &b);
@@ -70,7 +76,7 @@ void api_cg_binop(KitCg* g, BinOp iop, u32 flags) {
rb = api_force_local_unless_imm(g, &b, ty);
}
- if (api_can_delay_int_arith(g, ty, flags) &&
+ if (can_delay &&
api_try_collapse_binop_identity(g, iop, ty, &a, &b, &folded_sv)) {
api_release(g, &a);
api_release(g, &b);
@@ -78,7 +84,7 @@ void api_cg_binop(KitCg* g, BinOp iop, u32 flags) {
return;
}
- if (api_can_delay_int_arith(g, ty, flags) &&
+ if (can_delay &&
(ra.kind == OPK_LOCAL || rb.kind == OPK_LOCAL) &&
(ra.kind == OPK_LOCAL || ra.kind == OPK_IMM) &&
(rb.kind == OPK_LOCAL || rb.kind == OPK_IMM)) {
@@ -109,10 +115,14 @@ void api_cg_unop(KitCg* g, UnOp iop, u32 flags) {
Operand dst;
ApiSValue folded_sv;
i64 folded;
+ int can_delay;
if (!g) return;
T = g->target;
a = api_pop(g);
ty = a.type ? a.type : a.op.type;
+ /* Pure function of (ty, flags); classify the foldable int once for both delay
+ * gates below. */
+ can_delay = api_can_delay_int_arith(g, ty, flags);
if (iop == UO_FNEG) {
if (!api_type_is_float(g->c, ty)) {
@@ -145,15 +155,14 @@ void api_cg_unop(KitCg* g, UnOp iop, u32 flags) {
return;
}
- if (api_can_delay_int_arith(g, ty, flags) &&
- api_try_fold_unary_chain(&a, iop, ty, &folded_sv)) {
+ if (can_delay && api_try_fold_unary_chain(&a, iop, ty, &folded_sv)) {
api_release(g, &a);
api_push(g, folded_sv);
return;
}
ra = api_force_local_unless_imm(g, &a, ty);
- if (api_can_delay_int_arith(g, ty, flags) && ra.kind == OPK_LOCAL) {
+ if (can_delay && ra.kind == OPK_LOCAL) {
int a_owned = api_sv_owns_operand_local(&a, &ra);
api_push(g, api_make_arith_unop(iop, ra, ty, a_owned));
if (a_owned) a.res = RES_INHERENT;
@@ -307,26 +316,28 @@ void api_cg_convert_kind(KitCg* g, KitCgTypeId dst_type, ConvKind ck) {
* mirrors the f128 dispatch in kit_cg_fp_*.
* ============================================================ */
+/* The wide / soft-float dispatch class cached on the value-stack node at
+ * `depth` below TOS (api_push computes it via api_wide_kind_for). The five
+ * stack-top predicates below are now a bounds check plus a tag compare; the
+ * type-get + alias-chase gauntlet that used to run per operand happens once at
+ * push instead. WK_NARROW for an out-of-range depth keeps every predicate false
+ * exactly as the old bounds guards did. */
+static u8 api_stack_wide_kind(KitCg* g, u32 depth) {
+ if (!g || g->sp <= depth) return WK_NARROW;
+ return g->stack[g->sp - 1u - depth].bitfield.wide_kind;
+}
+
int api_i128_stack_top(KitCg* g, u32 depth) {
- if (!g || g->sp <= depth) return 0;
- return api_is_i128_type(g->c, api_sv_type(&g->stack[g->sp - 1u - depth]));
+ return api_stack_wide_kind(g, depth) == WK_I128;
}
/* 64-bit integer split into two 32-bit lanes by the selected ABI. The native
* backend handles add/sub/and/or/xor on such values as register pairs, but
* mul/div/shift must be lowered to a __*di3 runtime call (see
* api_wideint64_binop). i128 routes through its own ti3 path (api_i128_*), so
- * it is explicitly excluded here. */
-static int api_int_is_wide64(KitCg* g, KitCgTypeId ty) {
- if (!g) return 0;
- if (api_is_i128_type(g->c, ty)) return 0;
- if (kit_cg_type_int_width((KitCompiler*)g->c, ty) == 0) return 0;
- return api_is_wide8_scalar_type(g->c, ty);
-}
-
+ * the WK_WIDE8 class (computed in api_wide_kind_for) excludes it. */
static int api_wide64_stack_top(KitCg* g, u32 depth) {
- if (!g || g->sp <= depth) return 0;
- return api_int_is_wide64(g, api_sv_type(&g->stack[g->sp - 1u - depth]));
+ return api_stack_wide_kind(g, depth) == WK_WIDE8;
}
static int api_binop_is_shift(BinOp iop) {
@@ -1196,8 +1207,7 @@ static const char* api_softdf_binop_helper(KitCgFpBinOp op) {
}
int api_f128_stack_top(KitCg* g, u32 depth) {
- if (!g || g->sp <= depth) return 0;
- return api_is_f128_type(g->c, api_sv_type(&g->stack[g->sp - 1u - depth]));
+ return api_stack_wide_kind(g, depth) == WK_F128;
}
/* True when the target has no hardware double: float_abi is SOFT (ilp32/lp64,
@@ -1220,8 +1230,7 @@ static int api_type_is_soft_double(KitCg* g, KitCgTypeId ty) {
}
static int api_soft_double_stack_top(KitCg* g, u32 depth) {
- if (!g || g->sp <= depth) return 0;
- return api_type_is_soft_double(g, api_sv_type(&g->stack[g->sp - 1u - depth]));
+ return api_stack_wide_kind(g, depth) == WK_SOFT_DOUBLE;
}
/* f32 under pure-soft ilp32/lp64 (float_abi SOFT, no FP unit): single-precision
@@ -1233,8 +1242,7 @@ static int api_type_is_soft_single(KitCg* g, KitCgTypeId ty) {
}
static int api_soft_single_stack_top(KitCg* g, u32 depth) {
- if (!g || g->sp <= depth) return 0;
- return api_type_is_soft_single(g, api_sv_type(&g->stack[g->sp - 1u - depth]));
+ return api_stack_wide_kind(g, depth) == WK_SOFT_SINGLE;
}
/* Runtime helper for f32 arithmetic on a soft-float target (mirrors
diff --git a/src/cg/internal.h b/src/cg/internal.h
@@ -59,18 +59,43 @@ typedef struct ApiDelayedArith {
u8 pad;
} ApiDelayedArith;
+/* Scalar wide-class tag cached on every value-stack node (api_push). The wide /
+ * soft-float dispatch in arith.c re-derived this on every operand with a string
+ * of type-get + alias-chase round trips; it is a pure function of the value type
+ * (and the target's fixed float-ABI / split-lane policy), and the type never
+ * changes after a value is pushed, so it is computed once at push time and the
+ * stack-top predicates become a load and compare. The tags are mutually
+ * exclusive (a type matches at most one; see api_wide_kind_for). WK_I128 is kept
+ * distinct from WK_WIDE8 because the int wide64 path explicitly excludes i128
+ * (i128 routes through its own ti3 lowering). WK_NARROW == 0 so a zeroed node
+ * (api_make_sv memsets) defaults to the common narrow case. */
+typedef enum ApiWideKind {
+ WK_NARROW = 0, /* not wide / not soft-float: the common fast path */
+ WK_I128, /* 128-bit integer (api_is_i128_type) */
+ WK_F128, /* 128-bit float / long double (api_is_f128_type) */
+ WK_WIDE8, /* int split into two 32-bit lanes (api_int_is_wide64) */
+ WK_SOFT_DOUBLE, /* f64 on a target without hardware double */
+ WK_SOFT_SINGLE, /* f32 on a pure-soft target (no FP unit) */
+} ApiWideKind;
+
/* Bit-field geometry carried by a bit-field PLACE. `kit_cg_field` fills this
* from the record layout when it projects to a bit-field; a plain load/store on
* the carrying place then performs the extract/insert. The storage MemAccess is
* rebuilt from the place operand + field type at load/store time, so only the
* bit geometry needs to ride on the place. width == 0 means "not a bit-field".
- */
+ *
+ * `wide_kind` (ApiWideKind) rides in this struct's trailing slack: it is the
+ * only one-byte hole in the value-stack node, and keeping it here leaves the
+ * node size unchanged (static_assert in value.c). It is node state, not bit
+ * geometry, so kit_cg_field_bits writes only the bit_* members and leaves it
+ * intact, and node copies (dup/swap/rot) carry it along with the type. */
typedef struct ApiBitField {
u16 bit_offset; /* target-endian bit offset within the storage unit */
u16 bit_width; /* 0 => the place is not a bit-field place */
u32 bit_storage_size; /* storage-unit size in bytes */
u8 bit_signed; /* signed extraction on load */
- u8 pad[3];
+ u8 wide_kind; /* ApiWideKind, cached at api_push (node slack) */
+ u8 pad[2];
} ApiBitField;
typedef struct ApiSValue {
@@ -397,6 +422,7 @@ int api_is_f128_type(Compiler* c, KitCgTypeId ty);
int api_is_i128_type(Compiler* c, KitCgTypeId ty);
int api_is_wide16_scalar_type(Compiler* c, KitCgTypeId ty);
int api_is_wide8_scalar_type(Compiler* c, KitCgTypeId ty);
+u8 api_wide_kind_for(KitCg* g, KitCgTypeId ty);
Operand api_op_imm(i64 v, KitCgTypeId ty);
Operand api_op_local(CGLocal r, KitCgTypeId ty);
Operand api_op_global(ObjSymId sym, i64 addend, KitCgTypeId ty);
diff --git a/src/cg/native_direct_target.c b/src/cg/native_direct_target.c
@@ -500,9 +500,12 @@ static int nd_local_cacheable(NativeDirectTarget* d,
* a possibly-stale frame home. Base/index reads are always of the local's own
* type, so the width check is trivially met for that use; the value-read width
* hazard is handled separately in nd_materialize_operand. */
-/* Stamp a cache touch (def/read/addressing use) for LRU victim selection. */
+/* Stamp a cache touch (def/read/addressing use) for LRU victim selection. Every
+ * caller holds l->reg set (l is currently cached), so the flat reg_last_use
+ * mirror the victim scan reads stays in lockstep with l->last_use. */
static void nd_touch_local(NativeDirectTarget* d, NativeDirectLocal* l) {
l->last_use = ++d->use_tick;
+ d->reg_last_use[l->cls][l->reg] = l->last_use;
}
static Reg nd_cache_reg_for(NativeDirectTarget* d, CGLocal local,
@@ -526,14 +529,15 @@ static Reg nd_pick_cache_victim(NativeDirectTarget* d, NativeAllocClass cls) {
u32 best_use = 0;
for (u32 i = 0; i < ci->nallocable; ++i) {
Reg r = ci->allocable[i];
- CGLocal owner;
+ u32 use;
if (r >= 32u) continue;
- owner = d->reg_owner[cls][r];
- if (owner == CG_LOCAL_NONE) continue;
+ if (d->reg_owner[cls][r] == CG_LOCAL_NONE) continue;
if (d->scratch_used[cls] & (1u << r)) continue; /* pinned: never a victim */
- if (best == REG_NONE || nd_local(d, owner)->last_use < best_use) {
+ /* Flat mirror: equals nd_local(owner)->last_use for the owning local. */
+ use = d->reg_last_use[cls][r];
+ if (best == REG_NONE || use < best_use) {
best = r;
- best_use = nd_local(d, owner)->last_use;
+ best_use = use;
}
}
return best;
@@ -543,13 +547,17 @@ static Reg nd_pick_cache_victim(NativeDirectTarget* d, NativeAllocClass cls) {
* evict the LRU non-pinned cached local. REG_NONE means use the frame-only
* path. */
static Reg nd_cache_alloc(NativeDirectTarget* d, NativeAllocClass cls) {
- const NativeAllocClassInfo* ci = nd_class_info(d, cls);
- u32 caller = nd_caller_saved_mask(d, cls);
+ u32 caller = d->caller_saved[cls];
+ const Reg* sub = d->caller_alloc[cls];
+ u32 nsub = d->ncaller_alloc[cls];
Reg victim;
- for (u32 i = 0; i < ci->nallocable; ++i) {
- Reg r = ci->allocable[i];
- if (r >= 32u) continue;
- if ((caller & (1u << r)) && d->reg_owner[cls][r] == CG_LOCAL_NONE &&
+ /* The cache only ever allocates caller-saved registers, so scan the
+ * precomputed caller-saved-allocable sublist (allocable order preserved) and
+ * skip the always-failing callee-saved candidates. First match is identical
+ * to the former full-allocable scan. */
+ for (u32 i = 0; i < nsub; ++i) {
+ Reg r = sub[i];
+ if (d->reg_owner[cls][r] == CG_LOCAL_NONE &&
(d->scratch_used[cls] & (1u << r)) == 0) {
nd_note_reg_used(d, cls, r);
return r;
@@ -1919,6 +1927,23 @@ CgTarget* native_direct_target_new(Compiler* c, ObjBuilder* obj,
}
}
+ /* Precompute, per class, the caller-saved-allocable sublist (in allocable
+ * order) and the live-ABI caller-saved mask, both constant for the program.
+ * The local register cache only allocates caller-saved registers, so the
+ * free-reg scan iterates this short list and avoids re-querying the mask. */
+ for (u32 cls = 0; cls < 3u; ++cls) {
+ const NativeAllocClassInfo* ci = d->class_info[cls];
+ u32 mask, n = 0;
+ if (!ci) continue;
+ mask = nd_caller_saved_mask(d, (NativeAllocClass)cls);
+ d->caller_saved[cls] = mask;
+ for (u32 i = 0; i < ci->nallocable; ++i) {
+ Reg r = ci->allocable[i];
+ if (r < 32u && (mask & (1u << r))) d->caller_alloc[cls][n++] = r;
+ }
+ d->ncaller_alloc[cls] = n;
+ }
+
d->base.func_begin = nd_func_begin;
d->base.func_end = nd_func_end;
d->base.alias = nd_alias;
diff --git a/src/cg/native_direct_target.h b/src/cg/native_direct_target.h
@@ -120,6 +120,15 @@ struct NativeDirectTarget {
const NativeRegInfo* reg_info;
const NativeAllocClassInfo* class_info[3];
+ /* Per-class caller-saved-allocable sublist, precomputed at construction in
+ * `allocable[]` order (the cache only ever allocates caller-saved registers,
+ * so the free-reg scan iterates this short list instead of all allocable
+ * registers, and avoids re-querying the live-ABI caller-saved mask per call).
+ * caller_saved[cls] is that same mask cached. */
+ Reg caller_alloc[3][32];
+ u32 ncaller_alloc[3];
+ u32 caller_saved[3];
+
const CGFuncDesc* func;
SrcLoc loc;
@@ -146,6 +155,13 @@ struct NativeDirectTarget {
* NativeDirectLocal. See doc/CODEGEN.md "local register cache". */
CGLocal reg_owner[3][32];
u32 use_tick; /* monotonic counter stamped onto NativeDirectLocal.last_use */
+ /* Flat mirror of the currently-owning local's last_use, keyed by physical
+ * register, so the LRU victim scan reads one contiguous array instead of
+ * chasing a NativeDirectLocal per candidate. Written by nd_touch_local
+ * whenever an owned register's last_use changes; only read for registers with
+ * reg_owner[cls][reg] != CG_LOCAL_NONE, where it equals that local's
+ * last_use. */
+ u32 reg_last_use[3][32];
/* Head/tail of the intrusive cached-locals list (in caching order), -1 when
* empty; ncached is its length. Lets nd_flush_all run in O(cached) instead of
* scanning all nlocals on every control-flow / barrier op; cache_tail makes
diff --git a/src/cg/value.c b/src/cg/value.c
@@ -1,5 +1,15 @@
#include "cg/internal.h"
+/* The cached wide-class tag (ApiBitField.wide_kind) reuses the bit-field
+ * member's trailing pad: ApiBitField stays at its original 12-byte footprint
+ * (u16 + u16 + u32 + u8 bit_signed + u8 wide_kind + u8 pad[2]), and the node has
+ * no slack past it, so the value-stack node size is unchanged. */
+_Static_assert(sizeof(ApiBitField) == 12,
+ "wide_kind must reuse ApiBitField pad, not grow it");
+_Static_assert(offsetof(ApiSValue, bitfield) + sizeof(ApiBitField) ==
+ sizeof(ApiSValue),
+ "ApiSValue size must be unchanged by the cached wide_kind tag");
+
int api_type_is_float(Compiler* c, KitCgTypeId ty) {
const CgType* cg;
ty = api_unalias_type(c, ty);
@@ -38,6 +48,55 @@ int api_is_wide8_scalar_type(Compiler* c, KitCgTypeId ty) {
(ti.scalar_kind == ABI_SC_INT || ti.scalar_kind == ABI_SC_FLOAT);
}
+/* Classify a value type into its wide / soft-float dispatch class once, so the
+ * stack-top predicates in arith.c become a cached-byte compare instead of a
+ * fresh gauntlet of type-get + alias-chase round trips per operand. Called from
+ * api_push (the sole stack writer); the result rides on the node (the bitfield
+ * slack) for the value's lifetime, which is sound because the class is a pure
+ * function of the type plus the target's fixed float-ABI / split-lane policy,
+ * and the type never changes after a push.
+ *
+ * The classes are mutually exclusive and reproduce the original predicates
+ * exactly:
+ * WK_I128 == api_is_i128_type
+ * WK_F128 == api_is_f128_type
+ * WK_WIDE8 == api_int_is_wide64 (int-width-bearing wide8, excluding i128)
+ * WK_SOFT_DOUBLE == api_type_is_soft_double (f64 under SOFT/SINGLE float-abi)
+ * WK_SOFT_SINGLE == api_type_is_soft_single (f32 under SOFT float-abi)
+ * They cannot overlap: i128/wide8 are integer, f128/soft-double/soft-single are
+ * float, and the three float classes have distinct widths (128/64/32). A wide8
+ * float (e.g. double on a 32-bit ABI) bears int width 0 so it is not WK_WIDE8 —
+ * it falls through to WK_SOFT_DOUBLE, exactly as the original int-width guard in
+ * api_int_is_wide64 intended. */
+u8 api_wide_kind_for(KitCg* g, KitCgTypeId ty) {
+ Compiler* c;
+ u8 fa;
+ if (!g || !ty) return WK_NARROW;
+ c = g->c;
+ if (api_is_i128_type(c, ty)) return WK_I128;
+ if (api_is_f128_type(c, ty)) return WK_F128;
+ /* Int split into two 32-bit lanes by the ABI. The original api_int_is_wide64
+ * excludes i128 (handled above) and requires a nonzero int-like width, so a
+ * split-lane *float* never matches here. */
+ if (kit_cg_type_int_width((KitCompiler*)c, ty) != 0 &&
+ api_is_wide8_scalar_type(c, ty)) {
+ return WK_WIDE8;
+ }
+ fa = c->target.float_abi;
+ /* Soft double: f64 where the target has no hardware double (SOFT or SINGLE;
+ * under SINGLE only single-precision is in FP regs, double is always soft). */
+ if ((fa == KIT_FLOAT_ABI_SOFT || fa == KIT_FLOAT_ABI_SINGLE) &&
+ kit_cg_type_float_width((KitCompiler*)c, ty) == 64) {
+ return WK_SOFT_DOUBLE;
+ }
+ /* Soft single: f32 under pure-soft float-abi (no FP unit at all). */
+ if (fa == KIT_FLOAT_ABI_SOFT &&
+ kit_cg_type_float_width((KitCompiler*)c, ty) == 32) {
+ return WK_SOFT_SINGLE;
+ }
+ return WK_NARROW;
+}
+
Operand api_op_imm(i64 v, KitCgTypeId ty) {
Operand o;
memset(&o, 0, sizeof o);
@@ -194,16 +253,22 @@ void api_stack_grow(KitCg* g, u32 want) {
}
void api_push(KitCg* g, ApiSValue v) {
+ KitCgTypeId ty = api_sv_type(&v);
/* An aggregate (record) can only ever be a PLACE: it is addressed, loaded,
* and passed by SRET/BYVAL/BYREF, never materialized as a scalar VALUE. Catch
* any aggregate VALUE at the point it would enter the stack. i128/f128 are
* scalars (cg_type_is_aggregate is false for them), so they remain valid
* VALUEs and are unaffected. */
- if (cg_type_is_aggregate(g->c, api_sv_type(&v)) && !api_is_lvalue_sv(&v)) {
+ if (cg_type_is_aggregate(g->c, ty) && !api_is_lvalue_sv(&v)) {
compiler_panic(g->c, g->cur_loc,
"KitCg: aggregate must be a place, not a value; load the "
"place or pass it by reference");
}
+ /* Cache the wide/soft-float dispatch class once, here at the sole stack
+ * writer; the wide and soft-float predicates in arith.c read it back instead
+ * of re-deriving it per operand. dup/swap/rot copy the whole node (type and
+ * tag together), so it stays valid for every later reader. */
+ v.bitfield.wide_kind = api_wide_kind_for(g, ty);
api_stack_grow(g, g->sp + 1);
g->stack[g->sp++] = v;
}
diff --git a/src/core/buf.c b/src/core/buf.c
@@ -79,7 +79,10 @@ static int buf_ensure_tail(Buf* b, size_t need) {
return 0;
}
-void buf_write(Buf* b, const void* data, size_t n) {
+/* Out-of-line tail of buf_write: handles the empty-tail and chunk-spanning
+ * cases. The common single-chunk append is inlined in buf.h; only writes that
+ * don't fit the current tail (or hit an empty/NULL tail) land here. */
+void buf_write_slow(Buf* b, const void* data, size_t n) {
const u8* p = (const u8*)data;
while (n) {
size_t avail;
diff --git a/src/core/buf.h b/src/core/buf.h
@@ -1,6 +1,8 @@
#ifndef KIT_BUF_H
#define KIT_BUF_H
+#include <string.h>
+
#include "core/core.h"
#include "core/heap.h"
@@ -39,7 +41,25 @@ typedef struct Buf {
void buf_init(Buf*, Heap*);
void buf_fini(Buf*);
-void buf_write(Buf*, const void* data, size_t n);
+/* Out-of-line body for the empty-tail / chunk-spanning case. */
+void buf_write_slow(Buf*, const void* data, size_t n);
+
+/* Append n bytes. Inlined fast path: when the bytes fit the current tail chunk
+ * (the overwhelmingly common code-emit case — chunks hold 64KB), this is a bare
+ * memcpy + two field bumps with no call. Empty-tail / spanning writes fall to
+ * buf_write_slow. Behaviour is byte-for-byte identical to the old out-of-line
+ * loop; only the call/loop scaffolding is removed from the hot case. */
+static inline void buf_write(Buf* b, const void* data, size_t n) {
+ BufChunk* t = b->tail;
+ if (t && (size_t)(t->cap - t->used) >= n) {
+ memcpy(t->data + t->used, data, n);
+ t->used += (u32)n;
+ b->total += (u32)n;
+ return;
+ }
+ buf_write_slow(b, data, n);
+}
+
u8* buf_reserve(Buf*,
size_t n); /* contiguous; spills to a fresh chunk if needed */
u32 buf_pos(const Buf*);
diff --git a/src/core/pool.c b/src/core/pool.c
@@ -29,7 +29,19 @@ static u32 fnv1a(const char* s, size_t len) {
}
static int sym_eq(const PoolEntry* e, const char* s, size_t len, u32 h) {
- return e->hash == h && e->len == (u32)len && memcmp(e->data, s, len) == 0;
+ /* Inline byte loop instead of memcmp: the library is built -ffreestanding,
+ * under which clang emits a real memcmp libcall for a runtime length rather
+ * than inlining it. For the short identifiers the frontend interns, the call
+ * dispatch dominates the few-byte compare. The hash+len guards reject almost
+ * all non-matches before any byte is read. No word-at-a-time read: `s` points
+ * into the borrowed lexer buffer with no trailing-NUL/padding guarantee, so an
+ * over-wide load could overread at EOF; the byte loop is the safe form (and
+ * matches include/kit/core.h kit_slice_eq). */
+ size_t i;
+ if (e->hash != h || e->len != (u32)len) return 0;
+ for (i = 0; i < len; ++i)
+ if (e->data[i] != s[i]) return 0;
+ return 1;
}
static void table_rehash(Pool* p, u32 new_cap) {
@@ -50,6 +62,13 @@ static void table_rehash(Pool* p, u32 new_cap) {
if (p->table) p->heap->free(p->heap, p->table, sizeof(Sym) * p->cap);
p->table = new_table;
p->cap = new_cap;
+ /* Precompute the grow trigger so the per-intern load-factor test is a single
+ * compare (no multiplies). cap is always a power of two >= INITIAL_TABLE_CAP,
+ * so cap*LOAD_NUM is divisible by LOAD_DEN and this is exact: the trigger
+ * `used >= grow_threshold` fires at the same point as `used*DEN >= cap*NUM`.
+ * u64 in the product avoids overflow when cap is large. */
+ p->grow_threshold =
+ (u32)((u64)new_cap * POOL_TABLE_LOAD_NUM / POOL_TABLE_LOAD_DEN);
}
static int entries_grow(Pool* p) {
@@ -105,7 +124,7 @@ Sym pool_intern_slice(Pool* p, Slice in) {
Sym sym;
if (!s || len == 0) return 0;
- if (p->used * POOL_TABLE_LOAD_DEN >= p->cap * POOL_TABLE_LOAD_NUM) {
+ if (p->used >= p->grow_threshold) {
table_rehash(p, p->cap * 2);
}
h = fnv1a(s, len);
@@ -131,7 +150,16 @@ Sym pool_intern_slice(Pool* p, Slice in) {
{
char* dst = (char*)arena_alloc(&p->arena, len + 1, 1);
if (!dst) return 0;
- memcpy(dst, s, len);
+ /* Inline copy for the short-string common case: under -ffreestanding the
+ * memcpy is a libcall, and interned spellings are short (a few bytes), so
+ * the inline loop wins the call dispatch. Fall back to memcpy for the rare
+ * long spelling (e.g. a big string-literal token routed through intern). */
+ if (len <= 32) {
+ size_t i;
+ for (i = 0; i < len; ++i) dst[i] = s[i];
+ } else {
+ memcpy(dst, s, len);
+ }
dst[len] = '\0';
sym = (Sym)p->nentries++;
p->entries[sym].data = dst;
diff --git a/src/core/pool.h b/src/core/pool.h
@@ -20,6 +20,7 @@ struct Pool {
Sym* table;
u32 cap; /* always a power of two */
u32 used;
+ u32 grow_threshold; /* precomputed cap*LOAD_NUM/LOAD_DEN; grow when used>= it */
/* Sym → string mapping. Index 0 reserved as Sym = 0 ("none"). */
PoolEntry* entries;
diff --git a/src/obj/obj.c b/src/obj/obj.c
@@ -123,7 +123,12 @@ ObjBuilder* obj_new(Compiler* c) {
Relocs_init(&ob->relocs, h);
Groups_init(&ob->groups, h);
Atoms_init(&ob->atoms, h);
- SymNameIndex_init(&ob->sym_by_name, h);
+ /* Pre-size the symbol-name index: a typical TU has hundreds of symbols, and
+ * the default 16-slot map would otherwise resize/rehash ~4 times (16->256)
+ * while declarations stream in. 256 (a power of two, holds 192 at the 3/4
+ * load factor) skips that early cascade. Order-preserving — resize reinserts
+ * and the first-wins set at obj_symbol_make is unchanged. */
+ SymNameIndex_init_cap(&ob->sym_by_name, h, 256u);
SecKeyIndex_init(&ob->sec_by_key, h);
WeakAliases_init(&ob->weak_aliases, h);
@@ -620,6 +625,18 @@ void obj_write(ObjBuilder* ob, ObjSecId id, const void* data, size_t n) {
buf_write(&s->bytes, data, n);
}
+/* See obj.h: the byte buffer for a PROGBITS section, NULL for NOBITS/.bss (so
+ * the caller falls back to obj_write's bss_size path) or invalid ids. Lets the
+ * MCEmitter hoist the per-emit Sections_at deref + nobits branch out of the hot
+ * loop, re-resolving only at set_section. */
+Buf* obj_section_bytes(ObjBuilder* ob, ObjSecId id) {
+ Section* s;
+ if (id == OBJ_SEC_NONE) return NULL;
+ s = Sections_at(&ob->sections, id);
+ if (!s || sec_is_nobits(s)) return NULL;
+ return &s->bytes;
+}
+
u8* obj_reserve(ObjBuilder* ob, ObjSecId id, size_t n) {
Section* s;
if (id == OBJ_SEC_NONE) return NULL;
diff --git a/src/obj/obj.h b/src/obj/obj.h
@@ -433,6 +433,12 @@ void obj_section_set_addr(ObjBuilder*, ObjSecId, u64 addr);
void obj_section_set_ext(ObjBuilder*, ObjSecId, ObjExtKind, u32 ext_type,
u32 ext_flags);
void obj_write(ObjBuilder*, ObjSecId section_id, const void* data, size_t n);
+/* Resolve a section's byte buffer for direct emit. Returns NULL for the
+ * OBJ_SEC_NONE sentinel, an unknown id, or a NOBITS/.bss section (which stores
+ * only a size, not bytes — callers must route those through obj_write so the
+ * bss_size accounting runs). The returned pointer is stable: SegVec elements do
+ * not move, so callers may cache it until the next set_section. */
+Buf* obj_section_bytes(ObjBuilder*, ObjSecId section_id);
u8* obj_reserve(ObjBuilder*, ObjSecId section_id, size_t n);
void obj_reserve_bss(ObjBuilder*, ObjSecId section_id, u32 size, u32 align);
/* Pad `section_id` to `align`, returning the resulting offset. For
