commit 39c384b15f4d072000fa24e54ed52c85de2d4c38
parent 7ada3f622cc608f4a05db5a87ea9d5e88623f39d
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 11:53:37 -0700
perf(pp): reuse paste lexer + cache <paste> Sym, keep file_id ordering (A2)
(cherry picked from commit 4e255d5f78e6e4ee9988f7a3cb8845633879bb4c)
Diffstat:
9 files changed, 109 insertions(+), 19 deletions(-)
diff --git a/include/kit/source.h b/include/kit/source.h
@@ -15,6 +15,12 @@ KIT_API KitStatus kit_source_add_file(KitCompiler*, const char* path,
int system_header, uint32_t* file_id_out);
KIT_API KitStatus kit_source_add_memory(KitCompiler*, KitSlice name,
uint32_t* file_id_out);
+/* As kit_source_add_memory but takes a pre-interned name Sym: the caller has
+ * already interned the name (e.g. a fixed "<paste>" reused across many opens),
+ * so this skips the redundant intern + strlen while drawing the same
+ * sequential file_id. */
+KIT_API KitStatus kit_source_add_memory_sym(KitCompiler*, KitSym name,
+ uint32_t* file_id_out);
KIT_API KitStatus kit_source_add_builtin(KitCompiler*, KitSlice name,
uint32_t* file_id_out);
/* `system` is the spelling form (<...> vs "..."); `resolved_system` records
diff --git a/lang/cpp/lex/lex.c b/lang/cpp/lex/lex.c
@@ -320,27 +320,61 @@ static SrcLoc lex_here(const Lexer* l) {
return loc;
}
-Lexer* lex_open_mem(Compiler* c, const char* name, const char* src,
- size_t len) {
- Heap* h = (Heap*)kit_compiler_context(c)->heap;
- Lexer* l = (Lexer*)h->alloc(h, sizeof(*l), _Alignof(Lexer));
- if (!l) return NULL;
- memset(l, 0, sizeof(*l));
- l->c = c;
- l->heap = h;
+/* Fold splices on (src, len) and reset the scanner cursor/line state to the
+ * start of the buffer. Shared by lex_open_mem_sym (fresh lexer) and
+ * lex_reset_mem (re-pointed lexer). Does NOT touch file_id, which the caller
+ * sets, nor the punct_sym cache, which stays valid across resets (the interned
+ * Sym for a spelling is stable). */
+static void lex_point_at(Lexer* l, const char* src, size_t len) {
lex_fold_splices(l, src ? src : "", src ? len : 0);
l->cur = l->src;
l->end = l->src + l->len;
l->line = 1;
l->line_start = l->src;
- l->file_id = 0;
- (void)kit_source_add_memory(c, kit_slice_cstr(name), &l->file_id);
+ l->dstate = 0;
if (l->splices) lex_catchup_splices(l); /* a splice folded at offset 0 -> line 2 */
l->at_bol = 1;
l->had_space = 0;
+}
+
+Lexer* lex_open_mem_sym(Compiler* c, Sym name, const char* src, size_t len) {
+ Heap* h = (Heap*)kit_compiler_context(c)->heap;
+ Lexer* l = (Lexer*)h->alloc(h, sizeof(*l), _Alignof(Lexer));
+ if (!l) return NULL;
+ memset(l, 0, sizeof(*l));
+ l->c = c;
+ l->heap = h;
+ l->file_id = 0;
+ (void)kit_source_add_memory_sym(c, name, &l->file_id);
+ lex_point_at(l, src, len);
return l;
}
+Lexer* lex_open_mem(Compiler* c, const char* name, const char* src,
+ size_t len) {
+ Sym sym = kit_sym_intern(c, kit_slice_cstr(name));
+ return lex_open_mem_sym(c, sym, src, len);
+}
+
+/* Re-point an existing memory lexer at a fresh buffer for reuse across many
+ * tiny opens (e.g. `<paste>` buffers), avoiding a fresh alloc(sizeof Lexer) +
+ * lex_close free pair per open. Frees the previous folded buffer/splices (if
+ * any), re-registers a fresh sequential file_id (the count is byte-observable
+ * via DWARF, so it must still be bumped per open), then re-folds + resets the
+ * scanner. The borrowed (src, len) must outlive the next use of the lexer. */
+void lex_reset_mem(Lexer* l, Sym name, const char* src, size_t len) {
+ if (l->owns_src) l->heap->free(l->heap, (char*)l->src, l->len);
+ if (l->splices)
+ l->heap->free(l->heap, l->splices, l->nsplices * sizeof(u32));
+ l->owns_src = 0;
+ l->src = NULL;
+ l->splices = NULL;
+ l->nsplices = 0;
+ l->file_id = 0;
+ (void)kit_source_add_memory_sym(l->c, name, &l->file_id);
+ lex_point_at(l, src, len);
+}
+
void lex_close(Lexer* l) {
if (!l) return;
if (l->owns_src) l->heap->free(l->heap, (char*)l->src, l->len);
diff --git a/lang/cpp/lex/lex.h b/lang/cpp/lex/lex.h
@@ -89,6 +89,13 @@ typedef struct Lexer Lexer;
* The borrowed (src, len) buffer must outlive the Lexer, which for a pushed
* Lexer means outliving pp_free. */
Lexer* lex_open_mem(Compiler*, const char* name, const char* src, size_t len);
+/* As lex_open_mem but takes a pre-interned name Sym, so a fixed name reused
+ * across many opens (e.g. "<paste>") is interned just once by the caller. */
+Lexer* lex_open_mem_sym(Compiler*, Sym name, const char* src, size_t len);
+/* Re-point an existing memory lexer at a fresh buffer, reusing its allocation
+ * (no per-open alloc/free of the Lexer). Still registers a fresh sequential
+ * file_id per call, so the file_id order is identical to lex_open_mem_sym. */
+void lex_reset_mem(Lexer*, Sym name, const char* src, size_t len);
void lex_close(Lexer*);
/* Skip a leading `#!` script-interpreter ("shebang") line so an executable
diff --git a/lang/cpp/pp/pp.c b/lang/cpp/pp/pp.c
@@ -932,6 +932,9 @@ void pp_free(Pp* pp) {
h = pp_heap(pp);
/* Pop / close any remaining lex sources. */
while (pp->nsources) src_pop(pp);
+ /* The reused token-paste lexer is never pushed as a source, so close it
+ * directly here (it is lazily opened on the first `##`). */
+ if (pp->paste_lex) lex_close(pp->paste_lex);
pp_xfree(pp, pp->sources, sizeof(TokSrc) * pp->sources_cap);
MacroTab_fini(&pp->macros);
pp_xfree(pp, pp->hsets, sizeof(Hideset*) * pp->hsets_cap);
diff --git a/lang/cpp/pp/pp_expand.c b/lang/cpp/pp/pp_expand.c
@@ -730,19 +730,29 @@ static Tok paste_tokens(Pp* pp, Tok lhs, Tok rhs, SrcLoc loc) {
buf[alen + blen] = '\n';
buf[alen + blen + 1] = 0;
- lex = lex_open_mem(pp->c, "<paste>", buf, alen + blen + 1);
+ /* Intern the "<paste>" name once and reuse a single lexer across pastes: the
+ * lexer is re-pointed (not reallocated) per paste, but still draws a fresh
+ * sequential file_id so the file_id order matches a per-paste open exactly.
+ * t1.loc is overwritten below, so the paste file_id is dead for output. */
+ if (!pp->paste_name_sym) {
+ pp->paste_name_sym = kit_sym_intern(pp->c, kit_slice_cstr("<paste>"));
+ }
+ if (pp->paste_lex) {
+ lex_reset_mem(pp->paste_lex, pp->paste_name_sym, buf, alen + blen + 1);
+ } else {
+ pp->paste_lex =
+ lex_open_mem_sym(pp->c, pp->paste_name_sym, buf, alen + blen + 1);
+ }
+ lex = pp->paste_lex;
t1 = lex_next(lex);
t2 = lex_next(lex);
if (t1.kind == TOK_EOF) {
/* Both empty (shouldn't reach here since we handled placemarkers). */
- lex_close(lex);
return lhs;
}
if (t2.kind != TOK_NEWLINE && t2.kind != TOK_EOF) {
- lex_close(lex);
compiler_panic(pp->c, loc, "token pasting yields multiple tokens, invalid");
}
- lex_close(lex);
/* Inherit positional flags from LHS (it sat in the same slot). */
t1.flags = (u16)((t1.flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
diff --git a/lang/cpp/pp/pp_priv.h b/lang/cpp/pp/pp_priv.h
@@ -289,6 +289,14 @@ struct Pp {
Sym val_date_str;
Sym val_time_str;
+ /* Token-paste (`##`) scratch. paste_name_sym is the interned "<paste>" name,
+ * cached so each paste open doesn't re-intern the literal. paste_lex is a
+ * single reused memory lexer (re-pointed via lex_reset_mem per paste) so the
+ * thousands of tiny paste buffers don't each alloc/free a 288B Lexer. Both
+ * are lazily initialized on the first paste and torn down in pp_free. */
+ Sym paste_name_sym;
+ Lexer* paste_lex;
+
/* Defined-operator handling during #if expansion.
*
* The first prepass in eval_if_expr replaces `defined X` / `defined
diff --git a/src/api/source.c b/src/api/source.c
@@ -18,6 +18,12 @@ KitStatus kit_source_add_memory(KitCompiler* c, KitSlice name,
return source_add_memory(c->sources, name, file_id_out);
}
+KitStatus kit_source_add_memory_sym(KitCompiler* c, KitSym name,
+ uint32_t* file_id_out) {
+ if (!c || !file_id_out) return KIT_INVALID;
+ return source_add_memory_sym(c->sources, name, file_id_out);
+}
+
KitStatus kit_source_add_builtin(KitCompiler* c, KitSlice name,
uint32_t* file_id_out) {
if (!c || !file_id_out) return KIT_INVALID;
diff --git a/src/core/core.h b/src/core/core.h
@@ -101,6 +101,9 @@ void source_free(SourceManager*);
KitStatus source_add_file(SourceManager*, const char* path, int system_header,
u32* id_out);
KitStatus source_add_memory(SourceManager*, KitSlice name, u32* id_out);
+/* Register an in-memory file from a pre-interned name Sym (skips the per-call
+ * intern + strlen; the file_id sequence is identical to source_add_memory). */
+KitStatus source_add_memory_sym(SourceManager*, Sym name, u32* id_out);
KitStatus source_add_builtin(SourceManager*, KitSlice name, u32* id_out);
KitStatus source_add_include(SourceManager*, u32 includer_file_id,
u32 included_file_id, SrcLoc include_loc,
diff --git a/src/core/source.c b/src/core/source.c
@@ -91,13 +91,15 @@ void source_free(SourceManager* sm) {
sm->heap->free(sm->heap, sm, sizeof(*sm));
}
-static KitStatus file_register(SourceManager* sm, Slice name,
- SourceFileKind kind, int system_header,
- u32* id_out) {
- Sym sym;
+/* Register a file slot from a pre-interned name Sym (skips the intern). Draws
+ * the next sequential file_id from sm->nfiles++ exactly like file_register, so
+ * the file_id sequence -- which is byte-observable through DWARF line tables and
+ * diagnostics -- is unchanged. */
+static KitStatus file_register_sym(SourceManager* sm, Sym sym,
+ SourceFileKind kind, int system_header,
+ u32* id_out) {
u32 id;
if (files_grow(sm, sm->nfiles + 1)) return KIT_NOMEM;
- sym = pool_intern_slice(sm->c->global, name);
id = sm->nfiles++;
memset(&sm->files[id], 0, sizeof(sm->files[id]));
sm->files[id].info.id = id;
@@ -109,6 +111,13 @@ static KitStatus file_register(SourceManager* sm, Slice name,
return KIT_OK;
}
+static KitStatus file_register(SourceManager* sm, Slice name,
+ SourceFileKind kind, int system_header,
+ u32* id_out) {
+ Sym sym = pool_intern_slice(sm->c->global, name);
+ return file_register_sym(sm, sym, kind, system_header, id_out);
+}
+
KitStatus source_add_file(SourceManager* sm, const char* path,
int system_header, u32* id_out) {
return file_register(sm, slice_from_cstr(path ? path : ""), SRC_FILE_REAL,
@@ -119,6 +128,10 @@ KitStatus source_add_memory(SourceManager* sm, KitSlice name, u32* id_out) {
return file_register(sm, name, SRC_FILE_MEMORY, 0, id_out);
}
+KitStatus source_add_memory_sym(SourceManager* sm, Sym name, u32* id_out) {
+ return file_register_sym(sm, name, SRC_FILE_MEMORY, 0, id_out);
+}
+
KitStatus source_add_builtin(SourceManager* sm, KitSlice name, u32* id_out) {
return file_register(sm, name, SRC_FILE_BUILTIN, 0, id_out);
}
