commit 11fddf5b8aa3a6634ba00d053ca73d291f8b1e29
parent 6e8a65407c77b86bff1ee2de56fe11b6b4ad119e
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Fri, 12 Jun 2026 12:06:36 -0700
perf(lex): cache punctuator spelling Sym per lexer instead of re-interning
Punctuators are ~half of all tokens but there are only ~60 distinct
forms, so interning each one's spelling (as identifiers are) is millions
of pool hash-probes to rediscover the same handful of entries — visible
as a large share of pool_intern_slice/lex_next in compile profiles.
Cache the interned spelling Sym per Punct value on the Lexer, lazily on
first sight, and reuse it for every later occurrence. The first occurrence
interns the exact source bytes, so the Sym is bit-identical to the prior
unconditional lex_intern; subsequent same-punct tokens have identical
source bytes and would intern to the same content-addressed Sym anyway.
Lazy (not eager-at-open) on purpose: lex_open_mem runs per ## token-paste,
so eager interning of ~60 syms per open would regress macro-heavy code;
lazy caps interns at distinct-puncts-seen, strictly <= the old per-token cost.
The six digraph spellings (<: <% %> :> %: %:%:) share a Punct value with
their canonical form but spell differently, so they bypass the cache and
intern verbatim — preserving their spelling under #/##/-E without poisoning
the canonical slot.
Byte-identical: test-pp-ok (golden -E), test-parse-ok 3920/0, test-toy
1392/0, test-asm all lanes green; digraph stringize/-E spot-checked.
Diffstat:
1 file changed, 39 insertions(+), 1 deletion(-)
diff --git a/lang/cpp/lex/lex.c b/lang/cpp/lex/lex.c
@@ -43,6 +43,17 @@ struct Lexer {
u32* splices;
u32 nsplices;
u32 next_splice;
+ /* Lazily-interned spelling Sym per punctuator value (0 = not yet interned).
+ * Punctuators are about half of all tokens but there are only ~60 distinct
+ * forms, so interning the spelling of every one (as identifiers are) is
+ * millions of pool hash-probes to rediscover the same handful of entries.
+ * Cache the Sym the first time each punctuator is seen and reuse it for every
+ * later occurrence. Indexed directly by the Punct value (single-char puncts
+ * use their ASCII code, multi-char use P_ARROW..P_HASH_HASH). The digraph
+ * spellings (<: <% %> :> %: %:%:) are NOT cached here: they share a Punct
+ * value with their canonical form but spell differently, so they intern
+ * their source bytes verbatim each time (see the punct scanner). */
+ Sym punct_sym[P_HASH_HASH + 1];
};
/* §5.1.1.2 translation phase 2: fold every `\<newline>` line splice out of the
@@ -265,6 +276,22 @@ static Sym lex_intern(Lexer* l, size_t start, size_t end) {
(KitSlice){.s = l->src + start, .len = end - start});
}
+/* Spelling Sym for a non-digraph punctuator, interned once per lexer and then
+ * reused for every later occurrence of the same punctuator. The Punct value
+ * uniquely identifies the source spelling for every non-digraph form (all '+'
+ * tokens are "+", all P_ARROW are "->", ...), so caching by it is exact. The
+ * first occurrence interns the exact source bytes [start,end), making the Sym
+ * bit-identical to what an unconditional lex_intern would have produced. Pool
+ * entry 0 is the reserved "none" sym, so a cached spelling is always nonzero
+ * and 0 is an unambiguous not-yet-interned sentinel. Digraphs are excluded by
+ * the caller: they share a Punct value with their canonical form but spell
+ * differently, so they must intern their source bytes verbatim. */
+static Sym punct_spelling(Lexer* l, u32 punct, size_t start, size_t end) {
+ Sym s = l->punct_sym[punct];
+ if (!s) s = l->punct_sym[punct] = lex_intern(l, start, end);
+ return s;
+}
+
/* §6.4.7 header-name lookahead: in include-directive context, a `<` or `"`
* starts a header-name that runs to the matching `>` or `"`. The lexer
* recognizes only header-name forms (whose contents are implementation
@@ -613,6 +640,10 @@ Tok lex_next(Lexer* l) {
u32 punct = P_NONE;
u16 kind = TOK_PUNCT;
int i;
+ /* Set in the six digraph branches (<: <% %> :> %: %:%:): their source
+ * spelling differs from the canonical punct, so they bypass the punct_sym
+ * cache and intern verbatim rather than aliasing the canonical spelling. */
+ int digraph = 0;
switch (n0) {
case '#':
@@ -675,10 +706,12 @@ Tok lex_next(Lexer* l) {
} else if (n1 == ':') {
adv = 2;
punct = '[';
+ digraph = 1;
} /* digraph */
else if (n1 == '%') {
adv = 2;
punct = '{';
+ digraph = 1;
} /* digraph */
else {
adv = 1;
@@ -774,16 +807,19 @@ Tok lex_next(Lexer* l) {
adv = 4;
kind = TOK_PP_PASTE;
punct = P_HASH_HASH;
+ digraph = 1;
} else if (n1 == ':') {
adv = 2;
kind = TOK_PP_HASH;
punct = '#';
+ digraph = 1;
} else if (n1 == '=') {
adv = 2;
punct = P_MOD_ASSIGN;
} else if (n1 == '>') {
adv = 2;
punct = '}';
+ digraph = 1;
} /* digraph */
else {
adv = 1;
@@ -794,6 +830,7 @@ Tok lex_next(Lexer* l) {
if (n1 == '>') {
adv = 2;
punct = ']';
+ digraph = 1;
} /* digraph */
else {
adv = 1;
@@ -824,7 +861,8 @@ Tok lex_next(Lexer* l) {
for (i = 0; i < adv; ++i) bump(l);
t.kind = kind;
t.v.punct = punct;
- t.spelling = lex_intern(l, start, l->pos);
+ t.spelling = digraph ? lex_intern(l, start, l->pos)
+ : punct_spelling(l, punct, start, l->pos);
if (kind == TOK_PP_HASH)
l->dstate = 1;
else
