commit a753147bfd561abf78e6b010eeaaad3d3950b114
parent 503c2d59c817f84c69b267fa9747cdc6f2170b74
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Fri, 12 Jun 2026 14:26:06 -0700
perf(pp): multiple-include optimization (controlling #ifndef guard + #pragma once)
Record each header's controlling #ifndef macro (and #pragma once) as it is
first lexed, and skip re-lexing it entirely on a later #include while that
macro is defined -- the classic multiple-include optimization tcc/gcc/clang
have and kit lacked. kit's IncCache already served the bytes syscall-free, but
re-lexing them through skip_until_active still cost a full lex+intern of every
token of the guarded body, producing nothing.
Detection reuses the real lexer/pp as the file is first lexed (no second scan,
no divergence risk): a per-file 4-state machine (START->IN on the opening
'#ifndef GUARD' at the file's base if-depth; IN->AFTER on the matching #endif;
any code before the opening directive or after the controlling #endif ->
FAILED). Committed to the IncCache at src_pop; consulted at the next #include of
the same resolved path. A reading_directive flag keeps the directive's own
tokens (the 'ifndef'/macro name) from counting as file content.
sqlite amalgamation (arm64-macos, instructions, best-of-7): of 806 include
events, 502 now skipped (sys/cdefs.h was re-lexed 77x, now once). -E 1.42B ->
1.06B (-25.5%), -fsyntax-only 2.35B -> 1.98B (-15.5%), -c 2.83B -> 2.45B
(-13.2%). tcc gap 4.2x -> 3.66x.
Parser token stream is byte-identical (non-blank -E matches exactly; sqlite
compiles+runs 84|2; test-pp 110/0). The only -E text change is fewer blank
lines for skipped re-includes, matching gcc/clang (the compile path ignores
blank lines).
Diffstat:
3 files changed, 120 insertions(+), 0 deletions(-)
diff --git a/lang/cpp/pp/pp.c b/lang/cpp/pp/pp.c
@@ -40,6 +40,18 @@ void src_pop(Pp* pp) {
if (!pp->nsources) return;
t = &pp->sources[pp->nsources - 1];
if (t->kind == SRC_LEX && t->lex) {
+ /* Commit the multiple-include memo for this file before closing it: a
+ * fully #ifndef-wrapped file records its controlling macro; a #pragma once
+ * file records the once flag. A later #include of the same resolved path
+ * then skips re-lexing it entirely (see do_include). */
+ if (t->path_key && pp->inc_cache_ready) {
+ IncEntry* e = IncCache_get(&pp->inc_cache, t->path_key);
+ if (e) {
+ if (t->once) e->once = 1;
+ if (t->guard_state == GUARD_AFTER && t->guard_macro)
+ e->guard = t->guard_macro;
+ }
+ }
lex_close(t->lex);
t->lex = NULL;
}
@@ -94,6 +106,16 @@ void src_next_raw_into(Pp* pp, Tok* out, HidesetId* hs_out, u8* src_kind_out) {
if (src_kind_out) *src_kind_out = SRC_LEX;
return;
}
+ /* Multiple-include-guard detection: real code before the opening guard
+ * directive, or after the controlling #endif, disqualifies the whole-file
+ * #ifndef guard. A newline is whitespace and a beginning-of-line `#` is a
+ * directive introducer — both transparent; anything else is content. */
+ if (!pp->reading_directive &&
+ (s->guard_state == GUARD_START || s->guard_state == GUARD_AFTER) &&
+ out->kind != TOK_NEWLINE &&
+ !(out->kind == TOK_PP_HASH && (out->flags & TF_AT_BOL))) {
+ s->guard_state = GUARD_FAILED;
+ }
/* Apply #line line-number delta on the way out so the rest of
* the pipeline sees user-visible line numbers (matters for
* __LINE__ expansion and for line-tracking output cursors). */
@@ -273,6 +295,7 @@ static void pp_intern_keywords(Pp* pp) {
pp->sym_endif = kit_sym_intern(p->c, KIT_SLICE_LIT("endif"));
pp->sym_line = kit_sym_intern(p->c, KIT_SLICE_LIT("line"));
pp->sym_pragma = kit_sym_intern(p->c, KIT_SLICE_LIT("pragma"));
+ pp->sym_once = kit_sym_intern(p->c, KIT_SLICE_LIT("once"));
pp->sym_pragma_kw = pp->sym_pragma;
pp->sym_error = kit_sym_intern(p->c, KIT_SLICE_LIT("error"));
pp->sym_warning = kit_sym_intern(p->c, KIT_SLICE_LIT("warning"));
diff --git a/lang/cpp/pp/pp_directive.c b/lang/cpp/pp/pp_directive.c
@@ -41,6 +41,8 @@ void read_directive_line(Pp* pp, Tok** out_toks, u32* out_n) {
u32 cap = 0, n = 0;
Tok t;
HidesetId hs;
+ u8 saved_rd = pp->reading_directive;
+ pp->reading_directive = 1;
for (;;) {
t = src_next_raw(pp, &hs, NULL);
if (t.kind == TOK_NEWLINE || t.kind == TOK_EOF) break;
@@ -53,6 +55,7 @@ void read_directive_line(Pp* pp, Tok** out_toks, u32* out_n) {
}
buf[n++] = t;
}
+ pp->reading_directive = saved_rd;
*out_toks = buf;
*out_n = n;
}
@@ -1107,6 +1110,7 @@ static void do_include(Pp* pp, const Tok* line, u32 n, SrcLoc loc,
u32 i;
int found;
TokSrc s;
+ Sym pkey;
parse_include_path(pp, line, n, loc, path, sizeof(path), &system_form);
@@ -1136,6 +1140,18 @@ static void do_include(Pp* pp, const Tok* line, u32 n, SrcLoc loc,
KIT_SLICE_ARG(kit_slice_cstr(path)));
}
+ /* Multiple-include optimization: if this resolved file was already lexed and
+ * is either #pragma once or still wrapped by a defined #ifndef guard, skip
+ * re-lexing it entirely — it would emit no tokens anyway. This is the classic
+ * controlling-macro optimization (tcc/gcc/clang have it); kit's IncCache
+ * already serves the bytes syscall-free, but re-lexing them through
+ * skip_until_active still costs the full lex+intern of every token. */
+ pkey = inc_cache_key(pp, resolved);
+ if (pkey && pp->inc_cache_ready) {
+ IncEntry* e = IncCache_get(&pp->inc_cache, pkey);
+ if (e && (e->once || (e->guard && mt_get(pp, e->guard) != NULL))) return;
+ }
+
lex = lex_open_mem(pp->c, resolved, (const char*)data, size);
included_id = lex_file_id(lex);
@@ -1143,6 +1159,8 @@ static void do_include(Pp* pp, const Tok* line, u32 n, SrcLoc loc,
s.kind = SRC_LEX;
s.lex = lex;
s.inc_next_start = next_start;
+ s.path_key = pkey;
+ s.guard_if_base = pp->ifstk_n;
src_push(pp, s);
kit_source_add_include(pp->c, includer_id, included_id, loc, system_form,
@@ -1330,6 +1348,12 @@ static void handle_pragma_pack(Pp* pp, const Tok* line, u32 n, SrcLoc loc) {
}
static void do_pragma(Pp* pp, const Tok* line, u32 n, SrcLoc loc) {
+ /* #pragma once: mark the current file include-once for the multiple-include
+ * optimization (see do_include). Still forwarded to output like any pragma. */
+ if (n >= 1 && line[0].kind == TOK_IDENT && line[0].v.ident == pp->sym_once) {
+ TokSrc* gls = current_lex_src(pp);
+ if (gls) gls->once = 1;
+ }
/* Forward unrecognised pragmas to the output. STDC pragmas pass
* through too; we don't act on them yet. */
handle_pragma_pack(pp, line, n, loc);
@@ -1654,6 +1678,30 @@ void process_directive(Pp* pp, SrcLoc hash_loc) {
compiler_panic(pp->c, line[0].loc, "expected directive name after '#'");
}
name = line[0].v.ident;
+
+ /* Multiple-include-guard state machine, run on the current file source
+ * before dispatch. The only directive that may open a whole-file guard is
+ * `#ifndef MACRO` as the very first directive (nothing but whitespace before
+ * it, at the file's base if-depth); any other directive seen at START, or any
+ * directive at all after the controlling #endif (AFTER), disqualifies it.
+ * #pragma is exempt — #pragma once is its own (orthogonal) mechanism. */
+ {
+ TokSrc* gls = current_lex_src(pp);
+ if (gls && name != pp->sym_pragma) {
+ if (gls->guard_state == GUARD_START) {
+ if (name == pp->sym_ifndef && n >= 2 && line[1].kind == TOK_IDENT &&
+ pp->ifstk_n == gls->guard_if_base) {
+ gls->guard_macro = line[1].v.ident;
+ gls->guard_state = GUARD_IN;
+ } else {
+ gls->guard_state = GUARD_FAILED;
+ }
+ } else if (gls->guard_state == GUARD_AFTER) {
+ gls->guard_state = GUARD_FAILED;
+ }
+ }
+ }
+
if (name == pp->sym_define)
do_define(pp, line + 1, n - 1);
else if (name == pp->sym_undef)
@@ -1687,4 +1735,18 @@ void process_directive(Pp* pp, SrcLoc hash_loc) {
else {
compiler_panic(pp->c, line[0].loc, "unsupported directive");
}
+
+ /* The controlling #endif (the one returning the if-stack to the file's base
+ * depth) closes a candidate guard. After this only whitespace/newlines may
+ * follow for the guard to hold; any further token/directive flips it to
+ * FAILED (see the START/AFTER handling above and the content check in
+ * src_next_raw_into). do_include/do_undef etc. do not change current_lex_src
+ * out from under us here because this only fires for #endif. */
+ if (name == pp->sym_endif) {
+ TokSrc* gls = current_lex_src(pp);
+ if (gls && gls->guard_state == GUARD_IN &&
+ pp->ifstk_n == gls->guard_if_base) {
+ gls->guard_state = GUARD_AFTER;
+ }
+ }
}
diff --git a/lang/cpp/pp/pp_priv.h b/lang/cpp/pp/pp_priv.h
@@ -53,6 +53,21 @@ typedef struct Hideset {
typedef enum { SRC_LEX = 1, SRC_BUF = 2 } SrcKind;
+/* Multiple-include-guard detection state for a SRC_LEX file (the classic
+ * "controlling macro" optimization). A file whose entire body is wrapped in
+ * `#ifndef GUARD ... #endif` (with nothing but whitespace/comments outside)
+ * can be skipped on a later #include while GUARD is defined — it would emit no
+ * tokens anyway. Detected live as the file is first lexed (no second scan):
+ * START -> IN on the opening `#ifndef GUARD`; IN -> AFTER on the matching
+ * `#endif`; any code before the opening directive or after the controlling
+ * `#endif` -> FAILED. Committed to the IncCache at src_pop (see pp.c). */
+typedef enum GuardState {
+ GUARD_START = 0, /* only whitespace/newlines seen so far */
+ GUARD_IN = 1, /* inside the controlling #ifndef ... #endif */
+ GUARD_AFTER = 2, /* controlling #endif closed; only ws may follow */
+ GUARD_FAILED = 3 /* not a whole-file guard */
+} GuardState;
+
typedef struct TokSrc {
u8 kind;
/* When set on a SRC_BUF: src_next_raw returns TOK_EOF when this is
@@ -88,6 +103,15 @@ typedef struct TokSrc {
* ("...") step or an absolute path, so an `#include_next` there scans the
* whole search path (matching GCC). */
u32 inc_next_start;
+ /* Multiple-include-guard detection (SRC_LEX only). guard_if_base is the
+ * if-stack depth at file entry, used to recognize the controlling #endif;
+ * path_key is the IncCache key (interned resolved path, 0 for the top-level
+ * TU and other non-cached sources) used to commit the memo at src_pop. */
+ u8 guard_state; /* GuardState */
+ u8 once; /* #pragma once seen in this file */
+ Sym guard_macro;
+ u32 guard_if_base;
+ Sym path_key;
} TokSrc;
typedef enum IfState {
@@ -116,6 +140,11 @@ typedef struct IfFrame {
typedef struct IncEntry {
const u8* data;
size_t size;
+ /* Multiple-include optimization (committed at src_pop, consulted at the next
+ * #include of the same resolved path). guard = the file's controlling
+ * #ifndef macro (0 = none); once = the file carried #pragma once. */
+ Sym guard;
+ u8 once;
} IncEntry;
static inline u32 inc_hash_(Sym s) { return kit_hash_u32((u32)s); }
KIT_HASHMAP_DEFINE(IncCache, Sym, IncEntry, inc_hash_);
@@ -223,6 +252,7 @@ struct Pp {
Sym sym_endif;
Sym sym_line;
Sym sym_pragma;
+ Sym sym_once; /* "once" — #pragma once (multiple-include opt) */
Sym sym_error;
Sym sym_warning;
Sym sym_embed;
@@ -267,6 +297,11 @@ struct Pp {
* before the macro-expansion check at the head of pp_next_raw. */
u8 in_if_expansion;
u8 defined_skip;
+ /* Set while read_directive_line is pulling a directive's own tokens from the
+ * lexer, so the multiple-include-guard content check in src_next_raw_into
+ * doesn't count directive-internal tokens (e.g. the `ifndef` / macro name of
+ * the guard itself) as file content. */
+ u8 reading_directive;
};
/* ============================================================
