commit f368c7f86254e694ed3e3f4eafdc04e8512f8f67
parent c8ee91332d1285e7e3dcd070e08e108a648e6bb2
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 21:25:11 -0700
perf(lex): paste-buffer no-splice fast path (PERF §4.1 #4 Cut A)
Paste buffers (~2,796 opens compiling sqlite) paid a full lex_fold_splices
memchr scan that PROVABLY finds zero splices: paste content is interned,
already-folded token spellings + one appended '\n', so a lone backslash can
never sit immediately before that newline.
Add lex_point_at_nosplice — the nspl==0 branch of lex_fold_splices inlined
(borrow src verbatim, no splice table) WITHOUT the memchr scan, keeping the
cursor/line resets. Route lex_reset_mem (the sole paste-path entry) through
it; the precondition is guarded structurally (a separate entry point used
only by the paste path) and documented at the paste call site. File / -D /
_Pragma opens stay on the safe full lex_point_at path.
Gate: perf-gate byte-identical (60/60); test-pp/parse/asm/toy 4260/0; sqlite
-c object + -E text byte-identical golden-vs-candidate; -1,329,077 Ir
(1,915,270,714 -> 1,913,941,637, best-of-7).
Diffstat:
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/lang/cpp/lex/lex.c b/lang/cpp/lex/lex.c
@@ -337,6 +337,37 @@ static void lex_point_at(Lexer* l, const char* src, size_t len) {
l->had_space = 0;
}
+/* Splice-free variant of lex_point_at for buffers PROVABLY free of `\<newline>`
+ * splices. Does exactly what lex_point_at does, but takes the nspl==0 branch of
+ * lex_fold_splices unconditionally (borrow src verbatim, no splice table)
+ * WITHOUT the per-byte memchr scan that, for such a buffer, would only ever
+ * conclude "no splices". The cursor/line resets are identical to lex_point_at's
+ * (and to the nspl==0 path it inlines: owns_src=0, splices=NULL, no
+ * lex_catchup_splices since l->splices stays NULL).
+ *
+ * PRECONDITION (caller-guaranteed, NOT checked here): (src, len) contains no
+ * `\<newline>` line-splice. The ONLY caller is the token-paste path
+ * (lex_reset_mem ← paste_tokens): paste content is interned, already-folded
+ * token spellings concatenated with one appended '\n', so a lone `\` can never
+ * sit immediately before a '\n'. Do NOT route <command-line>/<_Pragma>/file
+ * opens through here — those stay on the safe lex_point_at path (a -D value
+ * could in theory carry a splice). */
+static void lex_point_at_nosplice(Lexer* l, const char* src, size_t len) {
+ l->src = src ? src : "";
+ l->len = src ? len : 0;
+ l->owns_src = 0;
+ l->splices = NULL;
+ l->nsplices = 0;
+ l->next_splice = 0;
+ l->cur = l->src;
+ l->end = l->src + l->len;
+ l->line = 1;
+ l->line_start = l->src;
+ l->dstate = 0;
+ l->at_bol = 1;
+ l->had_space = 0;
+}
+
Lexer* lex_open_mem_sym(Compiler* c, Sym name, const char* src, size_t len) {
Heap* h = (Heap*)kit_compiler_context(c)->heap;
Lexer* l = (Lexer*)h->alloc(h, sizeof(*l), _Alignof(Lexer));
@@ -372,7 +403,10 @@ void lex_reset_mem(Lexer* l, Sym name, const char* src, size_t len) {
l->nsplices = 0;
l->file_id = 0;
(void)kit_source_add_memory_sym(l->c, name, &l->file_id);
- lex_point_at(l, src, len);
+ /* The paste path is the sole caller, and its buffer (concatenated interned
+ * token spellings + one trailing '\n') provably has no `\<newline>` splice,
+ * so skip the memchr splice scan entirely (see lex_point_at_nosplice). */
+ lex_point_at_nosplice(l, src, len);
}
void lex_close(Lexer* l) {
diff --git a/lang/cpp/pp/pp_expand.c b/lang/cpp/pp/pp_expand.c
@@ -729,6 +729,12 @@ static Tok paste_tokens(Pp* pp, Tok lhs, Tok rhs, SrcLoc loc) {
if (blen) memcpy(buf + alen, b, blen);
buf[alen + blen] = '\n';
buf[alen + blen + 1] = 0;
+ /* INVARIANT for lex_reset_mem's no-splice fast path: this buffer is two
+ * already-folded interned token spellings (a, b) followed by exactly one
+ * '\n'. A token spelling never ends in a lone '\\' immediately before that
+ * '\n' (a backslash-newline would have been folded out at the original lex),
+ * so the buffer is free of `\<newline>` line-splices and the paste lexer
+ * (lex_reset_mem → lex_point_at_nosplice) can skip the splice scan. */
/* Intern the "<paste>" name once and reuse a single lexer across pastes: the
* lexer is re-pointed (not reallocated) per paste, but still draws a fresh
