commit 0e5337e33c91ef60bf270d726103e970d665e1d5
parent 7387376e6968085c62386dbfb8a2c26ed6fa54d2
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Mon, 15 Jun 2026 17:09:58 -0700
Merge Phase 1: macro-availability rewrite (hideset → disabled_depth + TF_NO_EXPAND)
Diffstat:
11 files changed, 139 insertions(+), 309 deletions(-)
diff --git a/lang/cpp/pp/pp.c b/lang/cpp/pp/pp.c
@@ -5,9 +5,11 @@
* line structure of the source.
*
* The token-source stack carries either a Lexer (file or #include'd file) or
- * a pre-built Tok[] buffer (macro expansion). Each buffer token carries a
- * hideset (Prosser, the standard's "nested-replacement" rule) recording
- * which macro names it must not be re-expanded by during rescan.
+ * a pre-built Tok[] buffer (macro expansion). A macro replacement-list buffer
+ * is a "disabled frame" (cpplib model): while it is live on the source stack
+ * its owning macro's disabled_depth is non-zero, so a recursive occurrence of
+ * that macro during rescan is returned permanently un-expanded (TF_NO_EXPAND)
+ * rather than re-replaced. This replaces the per-token Prosser hideset.
*
* Residual module: source stack, pp_next / pp_next_raw (public streaming),
* pp_new/free, predefined macros, lifecycle, keyword interning. */
@@ -40,6 +42,10 @@ void src_pop(Pp* pp) {
TokSrc* t;
if (!pp->nsources) return;
t = &pp->sources[pp->nsources - 1];
+ /* Re-enable the macro whose replacement-list frame this is (cpplib
+ * disabled-frame model): popping the frame ends the rescan during which the
+ * macro was unavailable. Balances the increment in push_buf_uniform/replay. */
+ if (t->disabled_owner) --t->disabled_owner->disabled_depth;
if (t->kind == SRC_LEX && t->lex) {
/* Commit the multiple-include memo for this file before closing it: a
* fully #ifndef-wrapped file records its controlling macro; a #pragma once
@@ -74,13 +80,12 @@ static void tok_eof(Tok* out) {
* source the token came from (SRC_LEX vs SRC_BUF). Used by pp_next_raw to
* gate directive recognition to lex-sourced tokens only — a `#` produced by
* macro expansion never starts a directive (§6.10.3.4 ¶3). */
-void src_next_raw_into(Pp* pp, Tok* out, HidesetId* hs_out, u8* src_kind_out) {
+void src_next_raw_into(Pp* pp, Tok* out, u8* src_kind_out) {
TokSrc* s;
while ((s = src_top(pp)) != NULL) {
if (s->kind == SRC_BUF) {
if (s->i < s->n) {
*out = s->toks[s->i];
- if (hs_out) *hs_out = s->hs ? s->hs[s->i] : s->hs_uniform;
if (src_kind_out) *src_kind_out = SRC_BUF;
/* Pointer-replayed body: apply the per-invocation loc and (first token
* only) BOL/SPACE flags the old fresh-copy path baked in. */
@@ -96,7 +101,6 @@ void src_next_raw_into(Pp* pp, Tok* out, HidesetId* hs_out, u8* src_kind_out) {
}
if (s->scope_top) {
tok_eof(out);
- if (hs_out) *hs_out = HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_BUF;
return;
}
@@ -110,7 +114,6 @@ void src_next_raw_into(Pp* pp, Tok* out, HidesetId* hs_out, u8* src_kind_out) {
src_pop(pp);
continue;
}
- if (hs_out) *hs_out = HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_LEX;
return;
}
@@ -127,20 +130,18 @@ void src_next_raw_into(Pp* pp, Tok* out, HidesetId* hs_out, u8* src_kind_out) {
/* #line numbering is no longer applied here: a lean token carries only a
* byte offset, and the #line delta is recorded as a positional overlay
* segment (see pp_add_line_seg) applied lazily by pp_materialize_loc. */
- if (hs_out) *hs_out = HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_LEX;
return;
}
tok_eof(out);
- if (hs_out) *hs_out = HS_EMPTY;
if (src_kind_out) *src_kind_out = SRC_LEX;
}
/* Thin by-value shim for the cold/general callers (arg collection, paren
* peek) that pass a NULL src_kind and don't sit in the hot -E loop. */
-Tok src_next_raw(Pp* pp, HidesetId* hs_out, u8* src_kind_out) {
+Tok src_next_raw(Pp* pp, u8* src_kind_out) {
Tok t;
- src_next_raw_into(pp, &t, hs_out, src_kind_out);
+ src_next_raw_into(pp, &t, src_kind_out);
return t;
}
@@ -148,29 +149,34 @@ Tok src_next_raw(Pp* pp, HidesetId* hs_out, u8* src_kind_out) {
* Buffer source push helpers
* ============================================================ */
-void push_buf(Pp* pp, Tok* toks, HidesetId* hs, u32 n) {
+/* Push a non-macro buffer source (pushed-back peek tokens, a #pragma / #embed
+ * payload, an arg-prescan scope). No disabled frame: these payloads do not
+ * disable any macro during their rescan. */
+void push_buf(Pp* pp, Tok* toks, u32 n) {
TokSrc s;
memset(&s, 0, sizeof(s));
s.kind = SRC_BUF;
s.toks = toks;
- s.hs = hs;
s.i = 0;
s.n = n;
src_push(pp, s);
}
-/* Push a buffer whose tokens all share one hideset (the common macro-body
- * case): no per-token array, just the scalar in hs_uniform (read when
- * TokSrc.hs is NULL). */
-void push_buf_uniform(Pp* pp, Tok* toks, u32 n, HidesetId hs_uniform) {
+/* Push a macro replacement-list buffer (function-like substituted body, paste
+ * result). `owner` is the macro being expanded: marking the frame with it and
+ * incrementing owner->disabled_depth makes the macro unavailable for the whole
+ * rescan of this replacement (cpplib disabled-frame model); src_pop decrements
+ * it. Set the owner on `s` BEFORE src_push (which copies by value); the
+ * increment lands on the macro itself, which is fine. */
+void push_buf_uniform(Pp* pp, Tok* toks, u32 n, Macro* owner) {
TokSrc s;
memset(&s, 0, sizeof(s));
s.kind = SRC_BUF;
s.toks = toks;
- s.hs = NULL;
- s.hs_uniform = hs_uniform;
s.i = 0;
s.n = n;
+ s.disabled_owner = owner;
+ if (owner) ++owner->disabled_depth;
src_push(pp, s);
}
@@ -180,19 +186,19 @@ void push_buf_uniform(Pp* pp, Tok* toks, u32 n, HidesetId hs_uniform) {
* value and never write back through s->toks. The per-read loc/flags override
* (has_loc_override) reproduces the mutations the old fresh-copy path baked in,
* so the shared body is surfaced byte-for-byte without copying it. */
-void push_buf_replay(Pp* pp, const Tok* toks, u32 n, HidesetId hs_uniform,
+void push_buf_replay(Pp* pp, const Tok* toks, u32 n, Macro* owner,
LocRef loc_override, u16 first_flags_or) {
TokSrc s;
memset(&s, 0, sizeof(s));
s.kind = SRC_BUF;
s.toks = (Tok*)toks;
- s.hs = NULL;
- s.hs_uniform = hs_uniform;
s.has_loc_override = 1;
s.loc_override = loc_override;
s.first_flags_or = first_flags_or;
s.i = 0;
s.n = n;
+ s.disabled_owner = owner;
+ if (owner) ++owner->disabled_depth;
src_push(pp, s);
}
@@ -231,10 +237,8 @@ void pp_next_parse(Pp* pp, Tok* out) {
/* Not a pragma — push the peeked token back as a 1-element buffer
* so the next pp_next_raw returns it, and surface the hash now. */
Tok* keep = arena_array(pp->xarena, Tok, 1);
- HidesetId* hs = arena_array(pp->xarena, HidesetId, 1);
keep[0] = t2;
- hs[0] = HS_EMPTY;
- push_buf(pp, keep, hs, 1);
+ push_buf(pp, keep, 1);
*out = t;
return;
}
@@ -1125,18 +1129,6 @@ Pp* pp_new(Compiler* c) {
h->free(h, pp, sizeof(*pp));
return NULL;
}
- /* Reserve hideset slot 0 for HS_EMPTY. The slot is unused but the
- * indexing convention costs only a pointer. */
- pp->hsets_cap = 8;
- pp->hsets = (Hideset**)pp_xrealloc(
- pp, NULL, 0, sizeof(Hideset*) * pp->hsets_cap, _Alignof(Hideset*));
- pp->hsets[0] = NULL;
- pp->hsets_n = 1;
- pp->hs_index_cap = 64;
- pp->hs_index_used = 0;
- pp->hs_index = (HidesetId*)pp_xrealloc(
- pp, NULL, 0, sizeof(HidesetId) * pp->hs_index_cap, _Alignof(HidesetId));
- memset(pp->hs_index, 0, sizeof(HidesetId) * pp->hs_index_cap);
MacroTab_init(&pp->macros, h);
pp_intern_keywords(pp);
compute_date_time(pp);
@@ -1169,8 +1161,6 @@ void pp_free(Pp* pp) {
}
pp_xfree(pp, pp->sources, sizeof(TokSrc) * pp->sources_cap);
MacroTab_fini(&pp->macros);
- pp_xfree(pp, pp->hsets, sizeof(Hideset*) * pp->hsets_cap);
- pp_xfree(pp, pp->hs_index, sizeof(HidesetId) * pp->hs_index_cap);
pp_xfree(pp, pp->ifstk, sizeof(IfFrame) * pp->ifstk_cap);
pp_xfree(pp, pp->inc_dirs, sizeof(*pp->inc_dirs) * pp->inc_dirs_cap);
c_pool_free(pp->pool);
diff --git a/lang/cpp/pp/pp_directive.c b/lang/cpp/pp/pp_directive.c
@@ -40,11 +40,10 @@ void read_directive_line(Pp* pp, Tok** out_toks, u32* out_n) {
Tok* buf = NULL;
u32 cap = 0, n = 0;
Tok t;
- HidesetId hs;
u8 saved_rd = pp->reading_directive;
pp->reading_directive = 1;
for (;;) {
- t = src_next_raw(pp, &hs, NULL);
+ t = src_next_raw(pp, NULL);
if (t.kind == TOK_NEWLINE || t.kind == TOK_EOF) break;
if (n == cap) {
u32 nc = cap ? cap * 2 : 8;
@@ -166,7 +165,7 @@ static void expand_for_if(Pp* pp, const Tok* in, u32 nin, TokVec* out) {
saved = pp->in_if_expansion;
pp->in_if_expansion = 1;
pp->defined_skip = 0;
- expand_arg_to_eof(pp, slice, NULL, nin, out);
+ expand_arg_to_eof(pp, slice, nin, out);
pp->in_if_expansion = saved;
pp->defined_skip = 0;
}
@@ -537,7 +536,7 @@ i64 eval_if_expr(Pp* pp, const Tok* line, u32 n, LocRef loc) {
static void consume_to_newline(Pp* pp) {
Tok t;
do {
- t = src_next_raw(pp, NULL, NULL);
+ t = src_next_raw(pp, NULL);
} while (t.kind != TOK_NEWLINE && t.kind != TOK_EOF);
}
@@ -554,7 +553,7 @@ static void skip_until_active(Pp* pp) {
IfFrame* top = if_top(pp);
Tok t;
if (top->state == IF_INCLUDE && local_depth == 0) return;
- t = src_next_raw(pp, NULL, NULL);
+ t = src_next_raw(pp, NULL);
if (t.kind == TOK_EOF) {
compiler_panic(pp->c, pp_materialize_loc(pp, top->loc),
"unterminated #if / #ifdef");
@@ -563,7 +562,7 @@ static void skip_until_active(Pp* pp) {
/* Read directive name (or null directive). */
{
- Tok nt = src_next_raw(pp, NULL, NULL);
+ Tok nt = src_next_raw(pp, NULL);
Sym name;
if (nt.kind == TOK_NEWLINE || nt.kind == TOK_EOF) continue;
if (nt.kind != TOK_IDENT) {
@@ -1064,7 +1063,7 @@ static void parse_include_path(Pp* pp, const Tok* line, u32 n, LocRef loc,
TokVec exp = {0};
Tok* slice = arena_array(pp->arena, Tok, n);
memcpy(slice, line, sizeof(Tok) * n);
- expand_arg_to_eof(pp, slice, NULL, n, &exp);
+ expand_arg_to_eof(pp, slice, n, &exp);
if (exp.n == 0) {
compiler_panic(pp->c, pp_materialize_loc(pp, loc),
@@ -1236,7 +1235,7 @@ static void do_line(Pp* pp, const Tok* line, u32 n, LocRef loc) {
"#line: missing arguments");
slice = arena_array(pp->arena, Tok, n);
memcpy(slice, line, sizeof(Tok) * n);
- expand_arg_to_eof(pp, slice, NULL, n, &exp);
+ expand_arg_to_eof(pp, slice, n, &exp);
if (exp.n == 0 || exp.data[0].kind != TOK_NUM) {
compiler_panic(pp->c, pp_materialize_loc(pp, loc),
@@ -1291,7 +1290,6 @@ static void do_line(Pp* pp, const Tok* line, u32 n, LocRef loc) {
* recognition off, so this won't recurse. */
void emit_pragma_line(Pp* pp, const Tok* line, u32 n, LocRef loc) {
TokVec out = {0};
- HidesetId* hids;
u32 i;
Tok hash, ident, nl;
@@ -1326,9 +1324,7 @@ void emit_pragma_line(Pp* pp, const Tok* line, u32 n, LocRef loc) {
nl.loc = loc;
tv_push(pp, &out, nl);
- hids = arena_array(pp->arena, HidesetId, out.n ? out.n : 1);
- for (i = 0; i < out.n; ++i) hids[i] = HS_EMPTY;
- push_buf(pp, out.data, hids, out.n);
+ push_buf(pp, out.data, out.n);
}
static int pragma_num_u32(Pp* pp, const Tok* t, u32* out) {
@@ -1453,18 +1449,12 @@ int try_expand_pragma_op(Pp* pp, const Tok* invoke) {
(void)saw_ws;
}
/* Read the string literal arg. */
- {
- HidesetId hs;
- str = src_next_raw(pp, &hs, NULL);
- }
+ str = src_next_raw(pp, NULL);
if (str.kind != TOK_STR) {
compiler_panic(pp->c, pp_materialize_loc(pp, invoke->loc),
"_Pragma: expected string literal");
}
- {
- HidesetId hs;
- rp = src_next_raw(pp, &hs, NULL);
- }
+ rp = src_next_raw(pp, NULL);
if (rp.kind != TOK_PUNCT || tok_punct(&rp) != ')') {
compiler_panic(pp->c, pp_materialize_loc(pp, invoke->loc),
"_Pragma: expected ')'");
@@ -1663,17 +1653,13 @@ static void do_embed(Pp* pp, const Tok* line, u32 n, LocRef loc) {
if (emit_n == 0) {
/* Empty: emit if_empty payload (or nothing). */
if (if_empty_toks && if_empty_n) {
- HidesetId* hids = arena_array(pp->arena, HidesetId, if_empty_n);
- u32 i;
- for (i = 0; i < if_empty_n; ++i) hids[i] = HS_EMPTY;
- push_buf(pp, if_empty_toks, hids, if_empty_n);
+ push_buf(pp, if_empty_toks, if_empty_n);
}
return;
}
/* Build a buffer of pp-numbers separated by ',' punctuators. */
{
TokVec out = {0};
- HidesetId* hids;
size_t i;
for (i = 0; i < emit_n; ++i) {
char numbuf[8];
@@ -1716,12 +1702,7 @@ static void do_embed(Pp* pp, const Tok* line, u32 n, LocRef loc) {
tv_push(pp, &out, comma);
}
}
- hids = arena_array(pp->arena, HidesetId, out.n ? out.n : 1);
- {
- u32 k;
- for (k = 0; k < out.n; ++k) hids[k] = HS_EMPTY;
- }
- push_buf(pp, out.data, hids, out.n);
+ push_buf(pp, out.data, out.n);
}
}
}
diff --git a/lang/cpp/pp/pp_expand.c b/lang/cpp/pp/pp_expand.c
@@ -1,5 +1,8 @@
-/* pp_expand.c — hideset table, macro hashmap, #define/#undef, substitution,
- * paste, stringize, argument prescan, func/object macro expansion. */
+/* pp_expand.c — macro table, #define/#undef, substitution, paste, stringize,
+ * argument prescan, func/object macro expansion. Macro-expansion availability
+ * uses the cpplib disabled-frame model (Macro.disabled_depth + the per-token
+ * TF_NO_EXPAND bit), not a per-token Prosser hideset; see pp.c / the identifier
+ * test in pp_pull_into below. */
#include "pp/pp_priv.h"
@@ -8,127 +11,6 @@ static int body_tokens_equal(Pp* pp, const Tok* a, u32 na, const Tok* b,
static int macros_equal(Pp* pp, const Macro* a, const Macro* b);
/* ============================================================
- * Hideset table
- * ============================================================ */
-
-static int sym_in_array(const Sym* a, u32 n, Sym s) {
- u32 i;
- for (i = 0; i < n; ++i)
- if (a[i] == s) return 1;
- return 0;
-}
-
-/* FNV-1a over the (sorted) Sym array — the dedup index key. */
-static u32 hs_hash(const Sym* names, u32 n) {
- u32 h = 0x811C9DC5u, i;
- for (i = 0; i < n; ++i) {
- h ^= names[i];
- h *= 0x01000193u;
- }
- return h;
-}
-
-/* Re-place every live hideset (ids 1..hsets_n-1) into a freshly zeroed index
- * of capacity nc (a power of two). Called when the load factor crosses 3/4. */
-static void hs_index_rebuild(Pp* pp, u32 nc) {
- u32 mask, k;
- pp->hs_index = (HidesetId*)pp_xrealloc(
- pp, pp->hs_index, sizeof(HidesetId) * pp->hs_index_cap,
- sizeof(HidesetId) * nc, _Alignof(HidesetId));
- for (k = 0; k < nc; ++k) pp->hs_index[k] = 0;
- pp->hs_index_cap = nc;
- mask = nc - 1;
- for (k = 1; k < pp->hsets_n; ++k) {
- u32 i = pp->hsets[k]->hash & mask;
- while (pp->hs_index[i] != 0) i = (i + 1) & mask;
- pp->hs_index[i] = (HidesetId)k;
- }
-}
-
-/* Intern a hideset: return the canonical id for `names` (content-deduplicated),
- * allocating a new one on first sight. Ids are assigned in first-appearance
- * order (identical to the historical linear scan — the id is internal-only, but
- * the order match keeps the dedup behavior bit-for-bit). */
-static HidesetId hs_register(Pp* pp, const Sym* names, u32 n) {
- Hideset* h;
- u32 hash, mask, i, j;
- HidesetId id;
- if (n == 0) return HS_EMPTY;
-
- hash = hs_hash(names, n);
- mask = pp->hs_index_cap - 1;
- i = hash & mask;
- /* Probe for an existing identical hideset; the loop also lands `i` on the
- * first empty slot for this hash, which is where a miss inserts. */
- while ((id = pp->hs_index[i]) != 0) {
- Hideset* e = pp->hsets[id];
- if (e->hash == hash && e->n == n) {
- for (j = 0; j < n; ++j)
- if (e->names[j] != names[j]) break;
- if (j == n) return id;
- }
- i = (i + 1) & mask;
- }
-
- if (pp->hsets_n == pp->hsets_cap) {
- u32 nc = pp->hsets_cap ? pp->hsets_cap * 2 : 8;
- pp->hsets =
- (Hideset**)pp_xrealloc(pp, pp->hsets, sizeof(Hideset*) * pp->hsets_cap,
- sizeof(Hideset*) * nc, _Alignof(Hideset*));
- pp->hsets_cap = nc;
- }
- h = (Hideset*)arena_alloc(pp->arena,
- sizeof(Hideset) + sizeof(Sym) * (n ? n - 1 : 0),
- _Alignof(Hideset));
- h->n = n;
- h->hash = hash;
- for (j = 0; j < n; ++j) h->names[j] = names[j];
- id = (HidesetId)pp->hsets_n;
- pp->hsets[pp->hsets_n++] = h;
- /* Insert at the empty slot the probe found, then grow+rehash if loaded. */
- pp->hs_index[i] = id;
- if (++pp->hs_index_used * 4u >= pp->hs_index_cap * 3u)
- hs_index_rebuild(pp, pp->hs_index_cap * 2u);
- return id;
-}
-
-int hs_contains(Pp* pp, HidesetId id, Sym s) {
- Hideset* h;
- if (id == HS_EMPTY || s == 0) return 0;
- h = pp->hsets[id];
- return sym_in_array(h->names, h->n, s);
-}
-
-HidesetId hs_add(Pp* pp, HidesetId id, Sym s) {
- Sym buf[64];
- Hideset* h;
- u32 n;
- u32 i;
-
- if (s == 0) return id;
- if (hs_contains(pp, id, s)) return id;
-
- n = (id == HS_EMPTY) ? 0 : pp->hsets[id]->n;
- if (n + 1 > sizeof(buf) / sizeof(buf[0])) {
- compiler_panic(pp->c, (SrcLoc){0, 0, 0}, "pp: hideset overflow");
- }
- if (id != HS_EMPTY) {
- h = pp->hsets[id];
- for (i = 0; i < h->n; ++i) buf[i] = h->names[i];
- }
- /* Keep sorted (numerically) for canonical hideset identity. */
- {
- u32 pos = n;
- while (pos > 0 && buf[pos - 1] > s) {
- buf[pos] = buf[pos - 1];
- --pos;
- }
- buf[pos] = s;
- }
- return hs_register(pp, buf, n + 1);
-}
-
-/* ============================================================
* #define / #undef
* ============================================================ */
/* mt_get / mt_put / mt_del are inlined Sym-indexed loads in pp_priv.h
@@ -379,11 +261,9 @@ static void subst_phase2(Pp* pp, const Tok* in, u32 nin, const Tok* invoke,
/* Build a buffer of the macro's body (with hidesets) and push it. The
* first expanded token inherits the invocation token's TF_AT_BOL /
* TF_HAS_SPACE so output formatting matches the invocation site. */
-static void expand_object_macro(Pp* pp, const Macro* m, const Tok* invoke,
- HidesetId invoke_hs) {
+static void expand_object_macro(Pp* pp, Macro* m, const Tok* invoke) {
TokVec body = {0};
Tok* tmp;
- HidesetId hs;
u32 i;
if (m->body_len == 0) {
@@ -394,12 +274,12 @@ static void expand_object_macro(Pp* pp, const Macro* m, const Tok* invoke,
* definition-time m->body by pointer, applying the per-invocation loc and
* first-token flag transfer at read time (push_buf_replay). Byte-identical
* to the slow path (same tokens, same loc, same first-token flags), with
- * zero per-invocation token copies or body allocs. */
+ * zero per-invocation token copies or body allocs. The frame disables `m`
+ * for the whole rescan of its replacement (cpplib model). */
if (!m->has_paste) {
u16 ff = (u16)((m->body[0].flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
(invoke->flags & (TF_AT_BOL | TF_HAS_SPACE)));
- hs = hs_add(pp, invoke_hs, m->name);
- push_buf_replay(pp, m->body, m->body_len, hs, invoke->loc, ff);
+ push_buf_replay(pp, m->body, m->body_len, m, invoke->loc, ff);
return;
}
@@ -417,8 +297,7 @@ static void expand_object_macro(Pp* pp, const Macro* m, const Tok* invoke,
(invoke->flags & (TF_AT_BOL | TF_HAS_SPACE)));
for (i = 0; i < body.n; ++i) body.data[i].loc = invoke->loc;
- hs = hs_add(pp, invoke_hs, m->name);
- push_buf_uniform(pp, body.data, body.n, hs);
+ push_buf_uniform(pp, body.data, body.n, m);
}
/* ============================================================
@@ -433,22 +312,19 @@ static void expand_object_macro(Pp* pp, const Macro* m, const Tok* invoke,
* are restored as a buffer source so subsequent reads still see them. */
int peek_for_invoke_paren(Pp* pp, int* ws_has_space_out) {
TokVec saved = {0};
- HsVec saved_hs = {0};
int saw_ws = 0;
Tok t;
- HidesetId hs;
for (;;) {
- t = src_next_raw(pp, &hs, NULL);
+ t = src_next_raw(pp, NULL);
if (t.kind == TOK_NEWLINE) {
saw_ws = 1;
tv_push(pp, &saved, t);
- hsv_push(pp, &saved_hs, hs);
continue;
}
if (t.kind == TOK_EOF) {
/* No '(' — push back saved tokens, leave EOF for next read. */
- if (saved.n) push_buf(pp, saved.data, saved_hs.data, saved.n);
+ if (saved.n) push_buf(pp, saved.data, saved.n);
*ws_has_space_out = saw_ws;
return 0;
}
@@ -461,8 +337,7 @@ int peek_for_invoke_paren(Pp* pp, int* ws_has_space_out) {
}
/* Save this non-`(` token too and push back. */
tv_push(pp, &saved, t);
- hsv_push(pp, &saved_hs, hs);
- push_buf(pp, saved.data, saved_hs.data, saved.n);
+ push_buf(pp, saved.data, saved.n);
*ws_has_space_out = saw_ws;
return 0;
}
@@ -471,7 +346,7 @@ int peek_for_invoke_paren(Pp* pp, int* ws_has_space_out) {
/* Run macro expansion on a fixed token sequence to completion, yielding the
* fully-expanded token sequence. Used to pre-expand each function-macro
* argument before substitution (§6.10.3.1 ¶1). */
-void expand_arg_to_eof(Pp* pp, Tok* in, HidesetId* hs, u32 nin, TokVec* out) {
+void expand_arg_to_eof(Pp* pp, Tok* in, u32 nin, TokVec* out) {
TokSrc src;
Tok t;
@@ -479,7 +354,6 @@ void expand_arg_to_eof(Pp* pp, Tok* in, HidesetId* hs, u32 nin, TokVec* out) {
src.kind = SRC_BUF;
src.scope_top = 1;
src.toks = in;
- src.hs = hs;
src.n = nin;
src_push(pp, src);
@@ -493,7 +367,11 @@ void expand_arg_to_eof(Pp* pp, Tok* in, HidesetId* hs, u32 nin, TokVec* out) {
}
tv_push(pp, out, t);
}
- /* Pop our scope source. */
+ /* Pop our scope source directly (not src_pop). This is the one place the
+ * disabled-frame decrement in src_pop is intentionally bypassed: the
+ * arg-prescan scope frame is pushed above with src.disabled_owner == NULL (it
+ * disables no macro — arguments are pre-expanded with the invoking macro NOT
+ * yet disabled), so there is no disabled_depth to decrement here. */
--pp->nsources;
}
@@ -503,7 +381,6 @@ void expand_arg_to_eof(Pp* pp, Tok* in, HidesetId* hs, u32 nin, TokVec* out) {
typedef struct ArgList {
/* Unexpanded arg tokens (raw as collected from invocation). */
Tok* raw;
- HidesetId* raw_hs;
u32 raw_n;
u32* raw_start; /* size n_args + 1 (sentinel = raw_n) */
/* Pre-expanded tokens. */
@@ -518,14 +395,12 @@ typedef struct ArgList {
static Tok read_invocation_args(Pp* pp, const Macro* m, LocRef invoke_loc,
ArgList* out) {
TokVec raw = {0};
- HsVec raw_hs = {0};
u32* starts;
u32 starts_cap = 0;
u32 n_args = 0;
u32 cur_start = 0;
int depth = 0;
Tok t;
- HidesetId hs;
int first_token_of_arg = 1;
Tok close_tok;
@@ -535,7 +410,7 @@ static Tok read_invocation_args(Pp* pp, const Macro* m, LocRef invoke_loc,
starts[0] = 0;
for (;;) {
- t = src_next_raw(pp, &hs, NULL);
+ t = src_next_raw(pp, NULL);
if (t.kind == TOK_EOF) {
compiler_panic(pp->c, pp_materialize_loc(pp, invoke_loc),
"unterminated function-like macro invocation");
@@ -583,7 +458,6 @@ static Tok read_invocation_args(Pp* pp, const Macro* m, LocRef invoke_loc,
if (m->is_variadic && n_args + 1 >= m->n_params) {
/* This comma is part of __VA_ARGS__. Push it. */
tv_push(pp, &raw, t);
- hsv_push(pp, &raw_hs, hs);
first_token_of_arg = 0;
continue;
}
@@ -603,7 +477,6 @@ static Tok read_invocation_args(Pp* pp, const Macro* m, LocRef invoke_loc,
}
}
tv_push(pp, &raw, t);
- hsv_push(pp, &raw_hs, hs);
first_token_of_arg = 0;
}
done:
@@ -643,7 +516,6 @@ done:
}
}
out->raw = raw.data;
- out->raw_hs = raw_hs.data;
out->raw_n = raw.n;
out->raw_start = starts;
out->n_args = n_args;
@@ -665,8 +537,7 @@ static void preexpand_args(Pp* pp, ArgList* a) {
* own it without aliasing. */
Tok* slice = arena_array(pp->xarena, Tok, hi - lo);
memcpy(slice, &a->raw[lo], sizeof(Tok) * (hi - lo));
- expand_arg_to_eof(pp, slice, a->raw_hs ? &a->raw_hs[lo] : NULL, hi - lo,
- &exp);
+ expand_arg_to_eof(pp, slice, hi - lo, &exp);
}
exp_start[i + 1] = exp.n;
}
@@ -1005,12 +876,10 @@ static void substitute_body(Pp* pp, const Macro* m, ArgList* a,
* pre-expand them, substitute the body, push the result. Returns 1 if
* the invocation was performed, 0 if there was no `(` (the caller should
* emit the identifier as-is). */
-static int try_expand_func_macro(Pp* pp, const Macro* m, const Tok* invoke,
- HidesetId invoke_hs) {
+static int try_expand_func_macro(Pp* pp, Macro* m, const Tok* invoke) {
int saw_ws;
ArgList args;
TokVec body = {0};
- HidesetId result_hs;
Tok close_tok;
if (!peek_for_invoke_paren(pp, &saw_ws)) {
@@ -1023,16 +892,13 @@ static int try_expand_func_macro(Pp* pp, const Macro* m, const Tok* invoke,
(void)close_tok;
preexpand_args(pp, &args);
- /* Hideset of result = invocation hideset ∪ {macro_name}. The standard
- * intersects with the closing `)`'s hideset for blue-paint purity, but
- * for the freshly-collected `)` from the lex source that's the empty
- * set, so the union form suffices here. */
- result_hs = hs_add(pp, invoke_hs, m->name);
substitute_body(pp, m, &args, invoke, &body);
- /* result_hs is uniform across the entire substituted body, so push it as a
- * single-hideset buffer — no per-token parallel array. */
- push_buf_uniform(pp, body.data, body.n, result_hs);
+ /* Push the substituted body as a disabled frame owned by `m`: the macro is
+ * unavailable for the whole rescan of this replacement (cpplib model), so a
+ * recursive occurrence of `m` is returned permanently un-expanded. The
+ * arguments were already pre-expanded above with `m` NOT disabled. */
+ push_buf_uniform(pp, body.data, body.n, m);
return 1;
}
@@ -1066,7 +932,6 @@ static int try_expand_func_macro(Pp* pp, const Macro* m, const Tok* invoke,
* directly. The hot SRC_BUF case (macro replay) is inlined here so the common
* token never pays a call into src_next_raw_into. */
static void pp_pull_into(Pp* pp, Tok* out, int skip_nl) {
- HidesetId hs;
u8 src_kind;
for (;;) {
TokSrc* s;
@@ -1088,14 +953,13 @@ static void pp_pull_into(Pp* pp, Tok* out, int skip_nl) {
if (!kit_arena_is_empty(pp->xarena)) kit_arena_reset(pp->xarena);
}
/* Fast path: top source is a non-exhausted SRC_BUF (the dominant
- * macro-replay case). Pull the token, hideset, and kind inline so the
- * common token avoids the call + sret setup of src_next_raw_into. The
- * scope_top-EOF and #line-delta cases never apply to a plain in-bounds
- * SRC_BUF read, so they stay on the general (cold) path. */
+ * macro-replay case). Pull the token and kind inline so the common token
+ * avoids the call + sret setup of src_next_raw_into. The scope_top-EOF and
+ * #line-delta cases never apply to a plain in-bounds SRC_BUF read, so they
+ * stay on the general (cold) path. */
if (pp->nsources != 0 &&
(s = &pp->sources[pp->nsources - 1])->kind == SRC_BUF && s->i < s->n) {
*out = s->toks[s->i];
- hs = s->hs ? s->hs[s->i] : s->hs_uniform;
src_kind = SRC_BUF;
/* Pointer-replayed body: apply the per-invocation loc and (first token
* only) BOL/SPACE flags the old fresh-copy path baked in. Identical to
@@ -1109,7 +973,7 @@ static void pp_pull_into(Pp* pp, Tok* out, int skip_nl) {
}
++s->i;
} else {
- src_next_raw_into(pp, out, &hs, &src_kind);
+ src_next_raw_into(pp, out, &src_kind);
}
if (out->kind == TOK_EOF) return;
/* §C.3a/§C.3b: absorb the parser-facing non-directive newline drop here so
@@ -1131,7 +995,7 @@ static void pp_pull_into(Pp* pp, Tok* out, int skip_nl) {
* newline is handled exactly as before. */
if (skip_nl && out->kind == TOK_NEWLINE) {
do {
- src_next_raw_into(pp, out, &hs, &src_kind);
+ src_next_raw_into(pp, out, &src_kind);
} while (out->kind == TOK_NEWLINE);
if (out->kind == TOK_EOF) return;
}
@@ -1255,15 +1119,28 @@ static void pp_pull_into(Pp* pp, Tok* out, int skip_nl) {
{
Macro* m = mt_get(pp, id);
- if (m && !hs_contains(pp, hs, m->name)) {
+ if (m && m->disabled_depth != 0) {
+ /* cpplib: the macro is disabled because one of its own
+ * replacement-list frames is still being rescanned. Mark this
+ * occurrence permanently un-expandable, so the bit survives if this
+ * token is later captured as an argument, substituted into another
+ * macro, or replayed (this is what keeps `foo(foo)(1)` and
+ * `id(foo(foo)(1))` as `foo(1)`). */
+ out->flags |= TF_NO_EXPAND;
+ return;
+ }
+ if (m) {
if (!m->is_func) {
- expand_object_macro(pp, m, out, hs);
+ expand_object_macro(pp, m, out);
continue;
}
- if (try_expand_func_macro(pp, m, out, hs)) {
+ if (try_expand_func_macro(pp, m, out)) {
continue;
}
- /* No '(' followed; emit as plain identifier. */
+ /* Function-like macro name NOT followed by '(': emit as a plain
+ * identifier WITHOUT setting TF_NO_EXPAND, so a later rescan that
+ * sees a '(' supplied by another macro can still expand it (keeps
+ * `id(f L 1))` with `#define L (` able to become `1`). */
}
}
}
diff --git a/lang/cpp/pp/pp_priv.h b/lang/cpp/pp/pp_priv.h
@@ -34,6 +34,15 @@ typedef struct Macro {
* replayed by pointer without the per-invocation copy + paste pass. */
u8 has_paste;
u8 pad[1];
+ /* cpplib-style macro-disabled count (replaces the Prosser hideset): the
+ * number of this macro's replacement-list frames currently live on the
+ * source stack. Incremented when a replacement frame for this macro is
+ * pushed, decremented when it pops (see push_buf_uniform/replay + src_pop).
+ * While > 0 the macro is unavailable for expansion, so a recursive
+ * occurrence in its own (or a nested) replacement is returned with the
+ * permanent TF_NO_EXPAND bit set instead of re-expanding. Auto-zeros: do_define
+ * mints Macro via arena_znew. */
+ u16 disabled_depth;
u32 n_params;
Sym* params; /* parameter names */
Tok* body; /* body tokens; TOK_PP_PARAM kind + aux=param idx */
@@ -45,16 +54,6 @@ typedef struct Macro {
* the expander's hot loop, replacing a per-identifier hashmap probe. */
KIT_SYMTAB_DEFINE(MacroTab, Macro*);
-typedef u32 HidesetId;
-#define HS_EMPTY 0u
-
-typedef struct Hideset {
- u32 n;
- u32 hash; /* content hash of names[0..n) — dedup index key (see hs_register)
- */
- Sym names[1]; /* flexible; allocated with extra trailing slots */
-} Hideset;
-
typedef enum { SRC_LEX = 1, SRC_BUF = 2 } SrcKind;
/* Multiple-include-guard detection state for a SRC_LEX file (the classic
@@ -93,14 +92,15 @@ typedef struct TokSrc {
Lexer* lex;
/* SRC_BUF */
Tok* toks;
- /* Per-token hidesets. When `hs` is non-NULL it is a parallel array (one id
- * per token) — used only by the argument-prescan path, where tokens can
- * carry differing hidesets. When `hs` is NULL the whole buffer shares the
- * single `hs_uniform` id: macro-body expansions (object- and function-like)
- * are uniform, so they take this path and skip the per-token array entirely
- * (no alloc, no fill, one scalar instead of n copies). */
- HidesetId* hs;
- HidesetId hs_uniform;
+ /* cpplib macro-disabled frame owner. Non-NULL only on a macro
+ * replacement-list frame (object-like replay, function-like substituted body,
+ * paste result); pushing such a frame increments owner->disabled_depth and
+ * src_pop decrements it, so the macro stays unavailable for the whole rescan
+ * of its replacement (and any nested replacement above it). NULL on every
+ * non-macro buffer (pushed-back peek tokens, #pragma/#embed payloads,
+ * arg-prescan scopes); every push_buf* memsets the whole TokSrc, so NULL is
+ * the default. */
+ Macro* disabled_owner;
/* Per-token loc applied when has_loc_override (see scope_top block). */
LocRef loc_override;
u32 i;
@@ -254,18 +254,6 @@ struct Pp {
u32 ifstk_n;
u32 ifstk_cap;
- /* Hideset table. Element 0 reserved as HS_EMPTY. */
- Hideset** hsets;
- u32 hsets_n;
- u32 hsets_cap;
- /* Content-addressed dedup index over hsets: open-addressed, power-of-two,
- * slot holds a HidesetId (0 = empty). Replaces the O(n) linear scan in
- * hs_register with an O(1) probe — the per-macro-invocation dedup was the
- * superlinear cost on macro-heavy input. */
- HidesetId* hs_index;
- u32 hs_index_cap;
- u32 hs_index_used;
-
/* Include directories (stage 9). */
struct {
const char* path;
@@ -279,12 +267,12 @@ struct Pp {
u32 pack_stack[16];
u32 pack_stack_n;
- /* Permanent arena: macro bodies, params, hidesets, #include file data.
+ /* Permanent arena: macro bodies, params, #include file data.
* Lives until pp_free. */
KitArena* arena;
- /* Transient expansion scratch: the macro-substitution token/hideset buffers
- * (tv_grow/hsv_grow), arg slices, paste/stringize buffers -- everything that
+ /* Transient expansion scratch: the macro-substitution token buffers
+ * (tv_grow), arg slices, paste/stringize buffers -- everything that
* backs a SRC_BUF or is consumed within one expansion. Reset to its
* high-water mark by pp_next_raw whenever the source stack drains back to a
* lexer (no SRC_BUF live), so memory is O(expansion depth), not
@@ -413,12 +401,6 @@ typedef struct TokVec {
u32 cap;
} TokVec;
-typedef struct HsVec {
- HidesetId* data;
- u32 n;
- u32 cap;
-} HsVec;
-
static inline void tv_grow(Pp* pp, TokVec* v, u32 want) {
u32 nc;
if (v->cap >= want) return;
@@ -437,24 +419,6 @@ static inline void tv_push(Pp* pp, TokVec* v, Tok t) {
v->data[v->n++] = t;
}
-static inline void hsv_grow(Pp* pp, HsVec* v, u32 want) {
- u32 nc;
- if (v->cap >= want) return;
- nc = v->cap ? v->cap * 2 : 8;
- while (nc < want) nc *= 2;
- {
- HidesetId* nb = arena_array(pp->xarena, HidesetId, nc);
- if (v->n) memcpy(nb, v->data, sizeof(HidesetId) * v->n);
- v->data = nb;
- v->cap = nc;
- }
-}
-
-static inline void hsv_push(Pp* pp, HsVec* v, HidesetId hs) {
- hsv_grow(pp, v, v->n + 1);
- v->data[v->n++] = hs;
-}
-
/* Growable char buffer (arena-backed). */
typedef struct CharBuf {
char* data;
@@ -488,17 +452,22 @@ static inline void cb_putc(Pp* pp, CharBuf* b, char c) {
/* --- pp.c (source stack) → pp_expand.c, pp_directive.c --- */
/* Out-pointer form (hot path); src_next_raw is the by-value shim over it for
* the cold/general callers. */
-void src_next_raw_into(Pp* pp, Tok* out, HidesetId* hs_out, u8* src_kind_out);
-Tok src_next_raw(Pp* pp, HidesetId* hs_out, u8* src_kind_out);
+void src_next_raw_into(Pp* pp, Tok* out, u8* src_kind_out);
+Tok src_next_raw(Pp* pp, u8* src_kind_out);
void src_push(Pp* pp, TokSrc s);
void src_pop(Pp* pp);
-void push_buf(Pp* pp, Tok* toks, HidesetId* hs, u32 n);
-void push_buf_uniform(Pp* pp, Tok* toks, u32 n, HidesetId hs_uniform);
+void push_buf(Pp* pp, Tok* toks, u32 n);
+/* Push a macro replacement-list buffer. `owner` is the macro whose replacement
+ * this is (NULL for a non-macro payload): a non-NULL owner increments
+ * owner->disabled_depth at push and src_pop decrements it, so the macro stays
+ * unavailable for the whole rescan of its replacement (cpplib disabled-frame
+ * model). */
+void push_buf_uniform(Pp* pp, Tok* toks, u32 n, Macro* owner);
/* Push an immutable, read-only token buffer (e.g. a no-`##` macro body) for
* pointer-replay. The buffer is never written through s->toks; per-read
* loc/flags overrides reproduce the mutations the old body-copy path baked in.
- */
-void push_buf_replay(Pp* pp, const Tok* toks, u32 n, HidesetId hs_uniform,
+ * `owner` follows the same disabled-frame rule as push_buf_uniform. */
+void push_buf_replay(Pp* pp, const Tok* toks, u32 n, Macro* owner,
LocRef loc_override, u16 first_flags_or);
/* pp_next_raw (public, in pp.h) is the mutual-recursion entry:
@@ -522,9 +491,7 @@ u32 pp_phys_line(Pp* pp, LocRef loc);
Sym pp_materialize_file(Pp* pp, LocRef loc);
/* --- pp_expand.c → pp.c, pp_directive.c --- */
-HidesetId hs_add(Pp* pp, HidesetId id, Sym s);
-int hs_contains(Pp* pp, HidesetId id, Sym s);
-void expand_arg_to_eof(Pp* pp, Tok* in, HidesetId* hs, u32 nin, TokVec* out);
+void expand_arg_to_eof(Pp* pp, Tok* in, u32 nin, TokVec* out);
/* Macro binding lookup/define/undef. Inlined Sym-indexed loads (no hashmap
* probe): mt_get runs on every identifier the expander sees. NULL/absent =
diff --git a/test/pp/CORPUS.md b/test/pp/CORPUS.md
@@ -104,6 +104,9 @@ Both runners pin the values of the otherwise-environmental predefined macros:
| 63_rescan_not_directive | 6.10.3.4 ¶3 | tokens produced by expansion are not a directive |
| 64_rescan_self_in_func | 6.10.3.4 ¶2 | function-like macro does not re-invoke itself in rescan |
| 65_rescan_arg_hideset_prescan | 6.10.3.4 ¶2 | argument prescan keeps no-reexpand state from caller |
+| 66_rescan_self_apply_paren | 6.10.3.4 ¶2 | disabled macro name kept unavailable when re-applied |
+| 67_rescan_arg_self_apply | 6.10.3.4 ¶2 | unavailable bit survives argument capture into id() |
+| 68_rescan_func_name_no_paren_then_paren | 6.10.3.4 ¶1 | func-like name with no `(` may expand when one is supplied later |
### 70–79 Variadic macros (§6.10.3 ¶12, §6.10.3.1 ¶2)
diff --git a/test/pp/cases/66_rescan_self_apply_paren.c b/test/pp/cases/66_rescan_self_apply_paren.c
@@ -0,0 +1,2 @@
+#define foo(x) foo
+foo(foo)(1)
diff --git a/test/pp/cases/66_rescan_self_apply_paren.expected b/test/pp/cases/66_rescan_self_apply_paren.expected
@@ -0,0 +1 @@
+foo(1)
diff --git a/test/pp/cases/67_rescan_arg_self_apply.c b/test/pp/cases/67_rescan_arg_self_apply.c
@@ -0,0 +1,3 @@
+#define id(x) x
+#define foo(x) foo
+id(foo(foo)(1))
diff --git a/test/pp/cases/67_rescan_arg_self_apply.expected b/test/pp/cases/67_rescan_arg_self_apply.expected
@@ -0,0 +1 @@
+foo(1)
diff --git a/test/pp/cases/68_rescan_func_name_no_paren_then_paren.c b/test/pp/cases/68_rescan_func_name_no_paren_then_paren.c
@@ -0,0 +1,4 @@
+#define L (
+#define f(a) a
+#define id(x) x
+id(f L 1))
diff --git a/test/pp/cases/68_rescan_func_name_no_paren_then_paren.expected b/test/pp/cases/68_rescan_func_name_no_paren_then_paren.expected
@@ -0,0 +1 @@
+1
