commit 811351c2b5e379813b0838812cab66d2425663ec
parent d59dff083787fc4850bd4cf12191d01d0b773f33
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 21:55:17 -0700
perf(lex): suppress non-directive newline tokens on cc path (PERF §4.1 #3B)
lex_next constructed a TOK_NEWLINE for every physical line, but ~51% of lexer
outputs are newlines and the C parser drops them all downstream. Suppress them
at the source on the cc parser-feed path.
Option B2 (no cross-frame coupling): the lexer tracks directive_line (set when
a TF_AT_BOL TOK_PP_HASH is emitted, cleared by the terminating newline) and
ALWAYS emits the newline that terminates a directive line; all other newlines
are suppressed when emit_newlines is off. Suppression still runs the full
bookkeeping (line++/line_start/at_bol/dstate/lex_catchup_splices) and continues
scanning, so line numbers and BOL detection (TF_AT_BOL via l->at_bol) are
unaffected — only the token is dropped.
Threading: new lex_set_emit_newlines setter (default ON, so -E/cpp/paste/
_Pragma sublexers are unchanged); Pp.suppress_lexer_newlines +
pp_set_suppress_lexer_newlines; c.c sets it before pp_push_input so the primary
source and every #include inherit OFF; the -E/cpp path leaves it ON. The
PP-layer skip_nl drain (pp_pull_into) becomes a near no-op but stays correct.
read_directive_line / #if-skip still see their directive-terminating newline;
the cc pragma-swallow consumes a SRC_BUF-synthesized newline, unaffected.
Gate (all golden-vs-candidate, contention-immune corpus diffs):
- perf-gate byte-identical 60/60 (incl -E / -c / -S / -g / diagnostics +
line-splice battery)
- sqlite amalg (263K lines): -c object, -E text, AND -c -g DWARF object all
byte-identical
- parse corpus 490 cases: 0 -fsyntax-only diag diffs, 0 exit mismatches, 0 -c
object diffs, 0 -c -g DWARF object diffs
- pp corpus 87 cases: 0 -E (text+diag) diffs
- test-pp 110/0
Measured: sqlite -c -32,278,680 Ir (1,926,022,213 -> 1,893,743,533, -1.68%);
-fsyntax-only -31,877,138 Ir (-1.92%, confirming the win is frontend).
Diffstat:
7 files changed, 94 insertions(+), 5 deletions(-)
diff --git a/lang/c/c.c b/lang/c/c.c
@@ -100,6 +100,12 @@ static KitStatus c_frontend_compile_cg(KitFrontendState* frontend,
fe_opts->preprocess.nundefines);
c_apply_pp_options(pp, &fe_opts->preprocess);
kit_frontend_metrics_scope_end(c, "compile.c.pp_options");
+ /* The C parser drops preprocessor newlines, so suppress non-directive
+ * newline tokens at the lexer (primary source + #includes) — ~51% of lexer
+ * outputs. Directive-terminating newlines are still emitted, so the PP's
+ * directive handling is unaffected. Must be set before pp_push_input so the
+ * primary lexer inherits the mode. */
+ pp_set_suppress_lexer_newlines(pp, 1);
kit_frontend_metrics_scope_begin(c, "compile.c.pp_push_input");
pp_push_input(pp, lex);
kit_frontend_metrics_scope_end(c, "compile.c.pp_push_input");
diff --git a/lang/cpp/lex/lex.c b/lang/cpp/lex/lex.c
@@ -148,6 +148,20 @@ struct Lexer {
* token may be a header-name. */
u8 dstate;
u8 owns_src; /* src is a heap-allocated folded copy to free at close */
+ /* Newline-token policy. When emit_newlines (the default) is set, every
+ * physical newline surfaces as a TOK_NEWLINE (the -E / cpp path needs them
+ * for line reconstruction). When cleared (the cc parser-feed path), a
+ * NON-directive newline is consumed silently — the line/at_bol/dstate
+ * bookkeeping still runs, but no token is returned; the parser never wanted
+ * newlines and PP's downstream skip_nl drain becomes a no-op. The newline
+ * that TERMINATES a directive line is ALWAYS emitted regardless, so
+ * read_directive_line / #if-skip still see their terminator (Option B2). */
+ u8 emit_newlines;
+ /* Set when a directive-introducing `#` (TOK_PP_HASH at BOL) was emitted on
+ * the current physical line; persists across the directive's tokens and is
+ * cleared by the line-terminating newline. Used to keep emitting that one
+ * newline even when emit_newlines is off. */
+ u8 directive_line;
/* Sorted logical offsets at which a `\<newline>` splice was folded out, so
* line tracking can still advance the physical line as the cursor passes
* each. Empty (and NULL) for splice-free input — the common case pays
@@ -332,6 +346,7 @@ static void lex_point_at(Lexer* l, const char* src, size_t len) {
l->line = 1;
l->line_start = l->src;
l->dstate = 0;
+ l->directive_line = 0;
if (l->splices) lex_catchup_splices(l); /* a splice folded at offset 0 -> line 2 */
l->at_bol = 1;
l->had_space = 0;
@@ -364,6 +379,7 @@ static void lex_point_at_nosplice(Lexer* l, const char* src, size_t len) {
l->line = 1;
l->line_start = l->src;
l->dstate = 0;
+ l->directive_line = 0;
l->at_bol = 1;
l->had_space = 0;
}
@@ -383,6 +399,10 @@ Lexer* lex_open_mem_sym(Compiler* c, Sym name, const char* src, size_t len) {
l->c = c;
l->heap = h;
l->file_id = 0;
+ /* Default: surface newline tokens (the -E / cpp / paste / _Pragma callers
+ * rely on them). The cc parser-feed path opts out via lex_set_emit_newlines.
+ * directive_line is (re)initialized by lex_point_at below. */
+ l->emit_newlines = 1;
(void)kit_source_add_memory_sym(c, name, &l->file_id);
lex_point_at(l, src, len);
return l;
@@ -445,6 +465,13 @@ void lex_skip_shebang(Lexer* l) {
SrcLoc lex_loc(const Lexer* l) { return lex_here(l); }
u32 lex_file_id(const Lexer* l) { return l->file_id; }
+/* Set the newline-token policy (see the emit_newlines field). Off = suppress
+ * non-directive newlines (cc parser feed); on (default) = surface every
+ * newline (-E / cpp). Directive-terminating newlines are emitted either way. */
+void lex_set_emit_newlines(Lexer* l, int on) {
+ if (l) l->emit_newlines = on ? 1u : 0u;
+}
+
/* Intern a token's spelling [a, b) straight from the folded buffer (it is
* already post-phase-2 logical text — splices were removed at open). */
static Sym lex_intern(Lexer* l, const char* a, const char* b) {
@@ -743,18 +770,31 @@ Tok lex_next(Lexer* l) {
return t;
}
if (*l->cur == '\n') {
+ /* Whether this newline must surface: always under emit_newlines (-E /
+ * cpp), and on the cc path only when it terminates a directive line
+ * (Option B2 — read_directive_line / #if-skip rely on that one). */
+ int emit_nl = l->emit_newlines || l->directive_line;
tloc = lex_here(l);
/* consume the newline */
l->cur++;
l->line++;
l->line_start = l->cur;
if (l->splices) lex_catchup_splices(l);
- t.kind = TOK_NEWLINE;
- t.loc = tloc;
- t.spelling = 0;
l->at_bol = 1;
l->had_space = 0;
l->dstate = 0;
+ /* The directive line (if any) ends at its terminating newline. */
+ l->directive_line = 0;
+ if (!emit_nl) {
+ /* Suppressed non-directive newline: bookkeeping done above (line /
+ * line_start / at_bol / dstate / splices), drop the token and keep
+ * scanning. The next content token carries TF_AT_BOL via l->at_bol, so
+ * BOL detection is unaffected by not materializing the newline. */
+ continue;
+ }
+ t.kind = TOK_NEWLINE;
+ t.loc = tloc;
+ t.spelling = 0;
return t;
}
break;
@@ -1142,10 +1182,17 @@ Tok lex_next(Lexer* l) {
t.v.punct = punct;
t.spelling = digraph ? lex_intern(l, tok_start, l->cur)
: punct_spelling(l, punct, tok_start, l->cur);
- if (kind == TOK_PP_HASH)
+ if (kind == TOK_PP_HASH) {
l->dstate = 1;
- else
+ /* A `#` at beginning-of-line introduces a directive (the PP only treats
+ * a TF_AT_BOL, lex-sourced `#` as a directive). Mark the line so its
+ * terminating newline is emitted even when emit_newlines is off, keeping
+ * read_directive_line's contract intact. A non-BOL `#` (e.g. `a # b`) is
+ * not a directive and leaves directive_line untouched. */
+ if (t.flags & TF_AT_BOL) l->directive_line = 1;
+ } else {
l->dstate = 0;
+ }
return t;
}
}
diff --git a/lang/cpp/lex/lex.h b/lang/cpp/lex/lex.h
@@ -103,6 +103,15 @@ void lex_close(Lexer*);
* primary source lexer, before any token is pulled; no-op otherwise. */
void lex_skip_shebang(Lexer*);
+/* Newline-token policy. Default ON: every physical newline surfaces as a
+ * TOK_NEWLINE (required by the -E / cpp text reconstruction and the paste /
+ * _Pragma sub-lexers). Set OFF on the cc parser-feed lexers (primary source +
+ * includes) to suppress NON-directive newlines at the source — the parser
+ * never wanted them, so the ~51% of lexer outputs that are newlines are never
+ * materialized. The newline that TERMINATES a directive line is emitted in
+ * either mode so read_directive_line / #if-skip keep their terminator. */
+void lex_set_emit_newlines(Lexer*, int on);
+
/* Streaming. Returns TOK_EOF repeatedly at end of input. */
Tok lex_next(Lexer*);
SrcLoc lex_loc(const Lexer*);
diff --git a/lang/cpp/pp/pp.c b/lang/cpp/pp/pp.c
@@ -947,11 +947,20 @@ void pp_free(Pp* pp) {
h->free(h, pp, sizeof(*pp));
}
+void pp_set_suppress_lexer_newlines(Pp* pp, int on) {
+ if (pp) pp->suppress_lexer_newlines = on ? 1u : 0u;
+}
+
void pp_push_input(Pp* pp, Lexer* lex) {
TokSrc s;
memset(&s, 0, sizeof(s));
s.kind = SRC_LEX;
s.lex = lex;
+ /* cc parser-feed mode: drop non-directive newlines at the lexer (see
+ * pp_set_suppress_lexer_newlines). pp_push_input is the entry for the primary
+ * source; #include'd lexers apply the same mode at their open site in
+ * do_include. */
+ if (pp->suppress_lexer_newlines) lex_set_emit_newlines(lex, 0);
src_push(pp, s);
}
diff --git a/lang/cpp/pp/pp.h b/lang/cpp/pp/pp.h
@@ -26,6 +26,14 @@ void pp_push_input(Pp*, Lexer*);
void pp_add_include_edge(Pp*, u32 includer_file_id, u32 included_file_id,
SrcLoc include_loc, int system);
+/* cc parser-feed mode: suppress non-directive newline tokens at every
+ * SRC_LEX lexer (primary source + #includes). The C parser drops newlines, so
+ * materializing them is pure overhead (~51% of lexer outputs). The -E / cpp
+ * path must NOT call this (it reconstructs text from newline tokens). Set once
+ * after pp_new, before pushing the primary input. Directive-terminating
+ * newlines are still emitted, so directive parsing is unaffected. */
+void pp_set_suppress_lexer_newlines(Pp*, int on);
+
/* Streaming. Yields preprocessed tokens (macro-expanded, directives consumed).
*/
Tok pp_next(Pp*);
diff --git a/lang/cpp/pp/pp_directive.c b/lang/cpp/pp/pp_directive.c
@@ -1154,6 +1154,10 @@ static void do_include(Pp* pp, const Tok* line, u32 n, SrcLoc loc,
lex = lex_open_mem(pp->c, resolved, (const char*)data, size);
included_id = lex_file_id(lex);
+ /* Inherit the parser-feed newline policy from the primary source: in cc mode
+ * an #include'd file's non-directive newlines are dropped at the lexer too
+ * (the parser never sees them); -E / cpp leaves them on. */
+ if (pp->suppress_lexer_newlines) lex_set_emit_newlines(lex, 0);
memset(&s, 0, sizeof(s));
s.kind = SRC_LEX;
diff --git a/lang/cpp/pp/pp_priv.h b/lang/cpp/pp/pp_priv.h
@@ -325,6 +325,12 @@ struct Pp {
* doesn't count directive-internal tokens (e.g. the `ifndef` / macro name of
* the guard itself) as file content. */
u8 reading_directive;
+ /* cc parser-feed mode: when set, every SRC_LEX lexer (the primary source and
+ * each #include'd file) is opened with lex_set_emit_newlines(lex, 0) so
+ * non-directive newlines are suppressed at the lexer (the parser drops them
+ * anyway). The -E / cpp path leaves this 0 so newlines surface for text
+ * reconstruction. Set via pp_set_suppress_lexer_newlines after pp_new. */
+ u8 suppress_lexer_newlines;
};
/* ============================================================
