kit

pp_expand.c (45435B)

---

 /* pp_expand.c — macro table, #define/#undef, substitution, paste, stringize,
  * argument prescan, func/object macro expansion. Macro-expansion availability
  * uses the cpplib disabled-frame model (Macro.disabled_depth + the per-token
  * TF_NO_EXPAND bit), not a per-token Prosser hideset; see pp.c / the identifier
  * test in pp_pull_into below. */
 
 #include "pp/pp_priv.h"
 
 static int body_tokens_equal(Pp* pp, const Tok* a, u32 na, const Tok* b,
                              u32 nb);
 static int macros_equal(Pp* pp, const Macro* a, const Macro* b);
 
 /* ============================================================
  * #define / #undef
  * ============================================================ */
 /* mt_get / mt_put / mt_del are inlined Sym-indexed loads in pp_priv.h
  * (MacroTab, a SYMTAB mapping Sym -> Macro*). */
 
 void do_define(Pp* pp, const Tok* line, u32 n) {
   Macro* m;
   u32 i = 0;
   Sym name;
   LocRef def_loc;
   Macro* existing;
 
   if (i >= n || line[i].kind != TOK_IDENT) {
     compiler_panic(pp->c,
                    n ? pp_materialize_loc(pp, line[0].loc) : (SrcLoc){0, 0, 0},
                    "#define: expected macro name");
   }
   name = tok_ident(&line[i]);
   def_loc = line[i].loc;
   ++i;
 
   m = arena_znew(pp->arena, Macro);
   m->name = name;
   m->def_loc = def_loc;
 
   /* Function-like vs object-like: '(' immediately after the name with no
    * intervening whitespace. */
   if (i < n && line[i].kind == TOK_PUNCT && tok_punct(&line[i]) == '(' &&
       (line[i].flags & TF_HAS_SPACE) == 0) {
     Sym* params = NULL;
     u32 pcap = 0, pn = 0;
     ++i;
     m->is_func = 1;
     if (i < n && line[i].kind == TOK_PUNCT && tok_punct(&line[i]) == ')') {
       ++i;
     } else {
       for (;;) {
         if (i >= n) {
           compiler_panic(pp->c, pp_materialize_loc(pp, def_loc),
                          "#define: unterminated parameter list");
         }
         if (line[i].kind == TOK_PUNCT && tok_punct(&line[i]) == P_ELLIPSIS) {
           /* Append a synthetic __VA_ARGS__ param so body-rewrite
            * matches the standard identifier directly. */
           if (pn == pcap) {
             u32 nc = pcap ? pcap * 2 : 4;
             Sym* nb = arena_array(pp->arena, Sym, nc);
             if (pcap) memcpy(nb, params, sizeof(Sym) * pcap);
             params = nb;
             pcap = nc;
           }
           params[pn++] = pp->sym_va_args;
           m->is_variadic = 1;
           ++i;
         } else if (line[i].kind == TOK_IDENT) {
           if (pn == pcap) {
             u32 nc = pcap ? pcap * 2 : 4;
             Sym* nb = arena_array(pp->arena, Sym, nc);
             if (pcap) memcpy(nb, params, sizeof(Sym) * pcap);
             params = nb;
             pcap = nc;
           }
           params[pn++] = tok_ident(&line[i]);
           ++i;
           /* GNU named variadic: `args...` — the named parameter itself collects
            * the trailing arguments (the body refers to it by name rather than
            * __VA_ARGS__). The variadic arg-collection below is positional on
            * the last param, so we just mark the macro variadic and eat the
            * ellipsis; the "'...' must be last" check still fires if a comma
            * follows. Linux UAPI headers use this (e.g. <linux/stddef.h>'s
            * __struct_group). */
           if (i < n && line[i].kind == TOK_PUNCT &&
               tok_punct(&line[i]) == P_ELLIPSIS) {
             m->is_variadic = 1;
             ++i;
           }
         } else {
           compiler_panic(pp->c, pp_materialize_loc(pp, line[i].loc),
                          "#define: bad parameter list");
         }
         if (i >= n) {
           compiler_panic(pp->c, pp_materialize_loc(pp, def_loc),
                          "#define: unterminated parameter list");
         }
         if (line[i].kind == TOK_PUNCT && tok_punct(&line[i]) == ')') {
           ++i;
           break;
         }
         if (m->is_variadic) {
           compiler_panic(pp->c, pp_materialize_loc(pp, line[i].loc),
                          "#define: '...' must be last parameter");
         }
         if (line[i].kind == TOK_PUNCT && tok_punct(&line[i]) == ',') {
           ++i;
           continue;
         }
         compiler_panic(pp->c, pp_materialize_loc(pp, line[i].loc),
                        "#define: expected ',' or ')'");
       }
     }
     m->params = params;
     m->n_params = pn;
   }
 
   /* Refuse define/undef of a few names the spec reserves: `defined`
    * and a small set of mandatory predefined macros. */
   if (name == pp->sym_defined || name == pp->sym_line__ ||
       name == pp->sym_file__ || name == pp->sym_date__ ||
       name == pp->sym_time__) {
     compiler_panic(pp->c, pp_materialize_loc(pp, def_loc),
                    "#define of a reserved / predefined name is not allowed");
   }
   /* Static predefineds are already in the macro table; redefining
    * with a different body is caught by the existing macros_equal
    * check below, but #define of __STDC__ et al. with the SAME body
    * should also be rejected. */
   if (name == pp->sym_stdc__ || name == pp->sym_stdc_hosted__ ||
       name == pp->sym_stdc_version__) {
     /* Allow re-registration of the predefined value at pp_new time
      * but reject user-level redefinition. We detect "user-level"
      * by checking whether it's already in the table — at pp_new the
      * first call goes through cleanly. */
     if (mt_get(pp, name)) {
       compiler_panic(pp->c, pp_materialize_loc(pp, def_loc),
                      "#define of a mandatory predefined macro is not allowed");
     }
   }
 
   /* Body: rewrite parameter occurrences to TOK_PP_PARAM. */
   {
     u32 body_n = n - i;
     u32 j;
     m->body = body_n ? arena_array(pp->arena, Tok, body_n) : NULL;
     m->body_len = body_n;
     for (j = 0; j < body_n; ++j) {
       Tok t = line[i + j];
       if (t.kind == TOK_PP_PASTE) m->has_paste = 1;
       if (m->is_func && t.kind == TOK_IDENT) {
         u32 p;
         for (p = 0; p < m->n_params; ++p) {
           if (m->params[p] == tok_ident(&t)) {
             /* Rewrite in place to TOK_PP_PARAM; leave `text` as the original
              * TEXT_SRC span (the param-name spelling, needed by the
              * redefinition compare). The parameter index lives in `aux`. */
             t.kind = TOK_PP_PARAM;
             t.aux = p;
             break;
           }
         }
       }
       /* §6.10.3 ¶5: __VA_ARGS__ outside a variadic macro is
        * undefined behavior; we diagnose. */
       if (!m->is_variadic && t.kind == TOK_IDENT &&
           tok_ident(&t) == pp->sym_va_args) {
         compiler_panic(pp->c, pp_materialize_loc(pp, t.loc),
                        "__VA_ARGS__ may only appear in a variadic macro body");
       }
       m->body[j] = t;
     }
     /* Drop the leading-space bit on the first body token: it reflects
      * the whitespace between the macro name (or close-paren) and the
      * body, which is irrelevant to expansion output. */
     if (m->body_len) m->body[0].flags &= (u16)~TF_HAS_SPACE;
   }
 
   existing = mt_get(pp, name);
   if (existing) {
     if (!macros_equal(pp, existing, m)) {
       compiler_panic(pp->c, pp_materialize_loc(pp, def_loc),
                      "macro redefined with different replacement");
     }
     return;
   }
   mt_put(pp, name, m);
 }
 
 void do_undef(Pp* pp, const Tok* line, u32 n) {
   Sym name;
   if (!n || line[0].kind != TOK_IDENT) {
     compiler_panic(pp->c,
                    n ? pp_materialize_loc(pp, line[0].loc) : (SrcLoc){0, 0, 0},
                    "#undef: expected identifier");
   }
   name = tok_ident(&line[0]);
   if (name == pp->sym_defined || name == pp->sym_line__ ||
       name == pp->sym_file__ || name == pp->sym_date__ ||
       name == pp->sym_time__ || name == pp->sym_stdc__ ||
       name == pp->sym_stdc_hosted__ || name == pp->sym_stdc_version__) {
     compiler_panic(pp->c, pp_materialize_loc(pp, line[0].loc),
                    "#undef of a mandatory predefined name is not allowed");
   }
   mt_del(pp, name);
 }
 
 /* ============================================================
  * Body comparison helpers
  * ============================================================ */
 
 static int body_tokens_equal(Pp* pp, const Tok* a, u32 na, const Tok* b,
                              u32 nb) {
   u32 i;
   if (na != nb) return 0;
   for (i = 0; i < na; ++i) {
     KitSlice as, bs;
     if (a[i].kind != b[i].kind) return 0;
     /* Two body tokens are equal iff their exact spellings match (§6.10.3 ¶1).
      * Compare the resolved bytes directly: a TOK_PP_PARAM keeps its original
      * param-name span, so identical parameters compare equal byte-for-byte,
      * and interned/source spellings compare the same way the old Sym-equality
      * did (identical interned spelling == identical bytes). */
     as = pp_text_slice(pp, &a[i]);
     bs = pp_text_slice(pp, &b[i]);
     if (as.len != bs.len || (as.len && memcmp(as.s, bs.s, as.len) != 0)) {
       return 0;
     }
     /* Whitespace separation must match (§6.10.3 ¶2). The first body
      * token's leading-space bit is meaningless (it's whatever was
      * between macro name and body); skip i==0 for that bit. */
     if (i > 0) {
       if ((a[i].flags & TF_HAS_SPACE) != (b[i].flags & TF_HAS_SPACE)) {
         return 0;
       }
     }
   }
   return 1;
 }
 
 static int macros_equal(Pp* pp, const Macro* a, const Macro* b) {
   if (a->is_func != b->is_func) return 0;
   if (a->is_variadic != b->is_variadic) return 0;
   if (a->n_params != b->n_params) return 0;
   {
     u32 i;
     for (i = 0; i < a->n_params; ++i) {
       if (a->params[i] != b->params[i]) return 0;
     }
   }
   return body_tokens_equal(pp, a->body, a->body_len, b->body, b->body_len);
 }
 
 /* ============================================================
  * Object-macro expansion
  * ============================================================ */
 
 static void subst_phase2(Pp* pp, const Tok* in, u32 nin, const Tok* invoke,
                          TokVec* out);
 
 /* Build a buffer of the macro's body (with hidesets) and push it. The
  * first expanded token inherits the invocation token's TF_AT_BOL /
  * TF_HAS_SPACE so output formatting matches the invocation site. */
 static void expand_object_macro(Pp* pp, Macro* m, const Tok* invoke) {
   TokVec body = {0};
   Tok* tmp;
   u32 i;
 
   if (m->body_len == 0) {
     return; /* placemarker: nothing to push */
   }
   /* No `##`: an object body has no params and no placemarkers, so the paste
    * phase is a pure identity copy. Skip both copies — replay the immutable
    * definition-time m->body by pointer, applying the per-invocation loc and
    * first-token flag transfer at read time (push_buf_replay). Byte-identical
    * to the slow path (same tokens, same loc, same first-token flags), with
    * zero per-invocation token copies or body allocs. The frame disables `m`
    * for the whole rescan of its replacement (cpplib model). */
   if (!m->has_paste) {
     u16 ff = (u16)((m->body[0].flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
                    (invoke->flags & (TF_AT_BOL | TF_HAS_SPACE)));
     push_buf_replay(pp, m->body, m->body_len, m, invoke->loc, ff);
     return;
   }
 
   /* Run the body through the paste phase: object-like macros may use
    * `##`. There are no parameters, so phase 1 reduces to a copy. */
   tmp = arena_array(pp->xarena, Tok, m->body_len);
   for (i = 0; i < m->body_len; ++i) tmp[i] = m->body[i];
   subst_phase2(pp, tmp, m->body_len, invoke, &body);
 
   if (body.n == 0) return;
 
   /* Transfer invocation flags onto the first emitted token. */
   body.data[0].flags =
       (u16)((body.data[0].flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
             (invoke->flags & (TF_AT_BOL | TF_HAS_SPACE)));
   for (i = 0; i < body.n; ++i) body.data[i].loc = invoke->loc;
 
   push_buf_uniform(pp, body.data, body.n, m);
 }
 
 /* ============================================================
  * Function-like macro expansion
  * ============================================================ */
 
 /* Peek for an open paren after the just-consumed identifier (which named
  * a function-like macro). Newlines are whitespace inside an invocation.
  * Returns 1 with `*ws_has_space_out` indicating whether any whitespace
  * (newlines or HAS_SPACE) sat between the ident and the `(`. Returns 0 if
  * no `(` follows; pushed-back tokens (NLs + the non-`(` token, if any)
  * are restored as a buffer source so subsequent reads still see them. */
 int peek_for_invoke_paren(Pp* pp, int* ws_has_space_out) {
   TokVec saved = {0};
   int saw_ws = 0;
   Tok t;
 
   for (;;) {
     t = src_next_raw(pp, NULL);
     if (t.kind == TOK_NEWLINE) {
       saw_ws = 1;
       tv_push(pp, &saved, t);
       continue;
     }
     if (t.kind == TOK_EOF) {
       /* No '(' — push back saved tokens, leave EOF for next read. */
       if (saved.n) push_buf(pp, saved.data, saved.n);
       *ws_has_space_out = saw_ws;
       return 0;
     }
     if (t.flags & TF_HAS_SPACE) saw_ws = 1;
     if (t.kind == TOK_PUNCT && tok_punct(&t) == '(') {
       /* Consumed. The newlines we walked past are whitespace and
        * dropped (per spec); they don't go back on the stack. */
       *ws_has_space_out = saw_ws;
       return 1;
     }
     /* Save this non-`(` token too and push back. */
     tv_push(pp, &saved, t);
     push_buf(pp, saved.data, saved.n);
     *ws_has_space_out = saw_ws;
     return 0;
   }
 }
 
 /* Run macro expansion on a fixed token sequence to completion, yielding the
  * fully-expanded token sequence. Used to pre-expand each function-macro
  * argument before substitution (§6.10.3.1 ¶1). */
 void expand_arg_to_eof(Pp* pp, Tok* in, u32 nin, TokVec* out) {
   TokSrc src;
   Tok t;
 
   memset(&src, 0, sizeof(src));
   src.kind = SRC_BUF;
   src.scope_top = 1;
   src.toks = in;
   src.n = nin;
   src_push(pp, src);
 
   for (;;) {
     pp_next_raw(pp, &t); /* drives macro expansion within this scope */
     if (t.kind == TOK_EOF) break;
     if (t.kind == TOK_NEWLINE) {
       /* Newlines inside an arg act as whitespace; convert to
        * "next-token has TF_HAS_SPACE". Drop the NL token itself. */
       continue;
     }
     tv_push(pp, out, t);
   }
   /* Pop our scope source directly (not src_pop). This is the one place the
    * disabled-frame decrement in src_pop is intentionally bypassed: the
    * arg-prescan scope frame is pushed above with src.disabled_owner == NULL (it
    * disables no macro — arguments are pre-expanded with the invoking macro NOT
    * yet disabled), so there is no disabled_depth to decrement here. */
   --pp->nsources;
 }
 
 /* Argument list for a function-like invocation. Stored as parallel
  * (start, end) ranges into a flat unexpanded token vector and a flat
  * expanded token vector. */
 typedef struct ArgList {
   /* Unexpanded arg tokens (raw as collected from invocation). */
   Tok* raw;
   u32 raw_n;
   u32* raw_start; /* size n_args + 1 (sentinel = raw_n) */
   /* Pre-expanded tokens. */
   Tok* exp;
   u32 exp_n;
   u32* exp_start; /* size n_args + 1 (sentinel = exp_n) */
   u32 n_args;
 } ArgList;
 
 /* Collect arguments. Caller has just consumed the opening `(`. Returns the
  * close-paren's token (used as the invocation's last source location). */
 static Tok read_invocation_args(Pp* pp, const Macro* m, LocRef invoke_loc,
                                 ArgList* out) {
   TokVec raw = {0};
   u32* starts;
   u32 starts_cap = 0;
   u32 n_args = 0;
   u32 cur_start = 0;
   int depth = 0;
   Tok t;
   int first_token_of_arg = 1;
   Tok close_tok;
 
   memset(out, 0, sizeof(*out));
   starts = arena_array(pp->xarena, u32, 8);
   starts_cap = 8;
   starts[0] = 0;
 
   for (;;) {
     t = src_next_raw(pp, NULL);
     if (t.kind == TOK_EOF) {
       compiler_panic(pp->c, pp_materialize_loc(pp, invoke_loc),
                      "unterminated function-like macro invocation");
     }
     if (t.kind == TOK_NEWLINE) {
       /* A newline inside an invocation is just whitespace; drop it.
        * The lexer already records the surrounding-whitespace bits on
        * the next token, so nothing needs to be carried across here. */
       continue;
     }
 
     if (t.kind == TOK_PUNCT) {
       u32 p = tok_punct(&t);
       if (p == '(') {
         ++depth;
       } else if (p == ')') {
         if (depth == 0) {
           /* End of invocation. Close the current argument. The
            * empty-args case (no commas seen, no tokens
            * collected) emits a slot only when the macro expects
            * at least one argument; arity-0 macros take none. */
           close_tok = t;
           {
             int empty_call =
                 (n_args == 0 && raw.n == cur_start && first_token_of_arg);
             int want_slot = !empty_call || (m->n_params > 0) || m->is_variadic;
             if (want_slot) {
               if (n_args + 1 >= starts_cap) {
                 u32 nc = starts_cap * 2;
                 u32* nb = arena_array(pp->xarena, u32, nc);
                 memcpy(nb, starts, sizeof(u32) * starts_cap);
                 starts = nb;
                 starts_cap = nc;
               }
               ++n_args;
               starts[n_args] = raw.n;
             }
           }
           goto done;
         }
         --depth;
       } else if (p == ',' && depth == 0) {
         /* Variadic: once we've filled all named params, the rest
          * (commas included) collect into __VA_ARGS__. */
         if (m->is_variadic && n_args + 1 >= m->n_params) {
           /* This comma is part of __VA_ARGS__. Push it. */
           tv_push(pp, &raw, t);
           first_token_of_arg = 0;
           continue;
         }
         /* Close current arg, start next. */
         if (n_args + 1 >= starts_cap) {
           u32 nc = starts_cap * 2;
           u32* nb = arena_array(pp->xarena, u32, nc);
           memcpy(nb, starts, sizeof(u32) * starts_cap);
           starts = nb;
           starts_cap = nc;
         }
         ++n_args;
         starts[n_args] = raw.n;
         cur_start = raw.n;
         first_token_of_arg = 1;
         continue;
       }
     }
     tv_push(pp, &raw, t);
     first_token_of_arg = 0;
   }
 done:
   /* Validate arity. */
   {
     u32 expected = m->n_params;
     if (m->is_variadic) {
       if (n_args < (expected ? expected - 1 : 0)) {
         /* Allow exactly expected-1 (empty __VA_ARGS__) by
          * synthesizing an empty trailing arg. */
         if (n_args + 1 == (expected ? expected - 1 : 0)) {
           /* off by one — fall through to error */
         }
         compiler_panic(pp->c, pp_materialize_loc(pp, invoke_loc),
                        "too few arguments to variadic macro invocation");
       }
       /* Synthesize an empty __VA_ARGS__ if caller passed exactly
        * the named-parameter count. */
       if (n_args + 1 == expected) {
         if (n_args + 1 >= starts_cap) {
           u32 nc = starts_cap * 2;
           u32* nb = arena_array(pp->xarena, u32, nc);
           memcpy(nb, starts, sizeof(u32) * starts_cap);
           starts = nb;
           starts_cap = nc;
         }
         ++n_args;
         starts[n_args] = raw.n;
       }
     } else {
       if (n_args != expected) {
         /* Spec: arity-0 macro `M()` invoked as `M()` is allowed and
          * has 0 args. Above logic produces 0 in that case. */
         compiler_panic(pp->c, pp_materialize_loc(pp, invoke_loc),
                        "wrong number of arguments to function-like macro");
       }
     }
   }
   out->raw = raw.data;
   out->raw_n = raw.n;
   out->raw_start = starts;
   out->n_args = n_args;
   return close_tok;
 }
 
 /* Build pre-expanded args. */
 static void preexpand_args(Pp* pp, ArgList* a) {
   TokVec exp = {0};
   u32* exp_start;
   u32 i;
   exp_start = arena_array(pp->xarena, u32, a->n_args + 1);
   exp_start[0] = 0;
   for (i = 0; i < a->n_args; ++i) {
     u32 lo = a->raw_start[i];
     u32 hi = a->raw_start[i + 1];
     if (hi > lo) {
       /* Copy the slice into a fresh buffer so expand_arg_to_eof can
        * own it without aliasing. */
       Tok* slice = arena_array(pp->xarena, Tok, hi - lo);
       memcpy(slice, &a->raw[lo], sizeof(Tok) * (hi - lo));
       expand_arg_to_eof(pp, slice, hi - lo, &exp);
     }
     exp_start[i + 1] = exp.n;
   }
   a->exp = exp.data;
   a->exp_n = exp.n;
   a->exp_start = exp_start;
 }
 
 /* Build a stringized TOK_STR from the unexpanded argument tokens
  * `arg[lo..hi)`. The first token's leading-space flag is ignored (leading
  * whitespace stripped). Inside string/char-literal spellings, '"' and '\'
  * are escaped. */
 static Tok make_stringize(Pp* pp, const Tok* arg, u32 lo, u32 hi, LocRef loc) {
   CharBuf b = {0};
   u32 i;
   Tok t;
 
   cb_putc(pp, &b, '"');
   for (i = lo; i < hi; ++i) {
     const Tok* at = &arg[i];
     KitSlice sl = pp_text_slice(pp, at);
     const char* s = sl.s;
     size_t slen = sl.len;
     if (i > lo && (at->flags & TF_HAS_SPACE)) cb_putc(pp, &b, ' ');
     if (s && slen) {
       int esc = (at->kind == TOK_STR || at->kind == TOK_CHR);
       size_t k;
       for (k = 0; k < slen; ++k) {
         char c = s[k];
         if (esc && (c == '\\' || c == '"')) cb_putc(pp, &b, '\\');
         cb_putc(pp, &b, c);
       }
     }
   }
   cb_putc(pp, &b, '"');
 
   memset(&t, 0, sizeof(t));
   t.kind = TOK_STR;
   t.aux = 0;
   t.loc = loc;
   t.text = text_intern_ref(pp, (KitSlice){.s = b.data, .len = b.len});
   return t;
 }
 
 /* Concatenate two token spellings and re-lex into a single token. Empty
  * (placemarker) sides collapse to the other side per §6.10.3.3 ¶2. */
 static Tok paste_tokens(Pp* pp, Tok lhs, Tok rhs, LocRef loc) {
   char buf[1024];
   size_t alen = 0, blen = 0;
   const char* a;
   const char* b;
   Lexer* lex;
   Tok t1, t2;
 
   if (lhs.kind == TOK_PP_PLACEMARKER) return rhs;
   if (rhs.kind == TOK_PP_PLACEMARKER) return lhs;
 
   {
     KitSlice s = pp_text_slice(pp, &lhs);
     if (s.len) {
       a = s.s;
       alen = s.len;
     } else {
       a = "";
     }
   }
   {
     KitSlice s = pp_text_slice(pp, &rhs);
     if (s.len) {
       b = s.s;
       blen = s.len;
     } else {
       b = "";
     }
   }
   if (alen + blen + 2 > sizeof(buf)) {
     compiler_panic(pp->c, pp_materialize_loc(pp, loc),
                    "token paste: spelling too long");
   }
   if (alen) memcpy(buf, a, alen);
   if (blen) memcpy(buf + alen, b, blen);
   buf[alen + blen] = '\n';
   buf[alen + blen + 1] = 0;
   /* INVARIANT for the re-lex's no-splice fast path: this buffer is two
    * already-folded token spellings (a, b) followed by exactly one '\n'. A
    * token spelling never ends in a lone '\\' immediately before that '\n' (a
    * backslash-newline would have been folded out at the original lex), so the
    * buffer is free of `\<newline>` line-splices and the paste lexer is opened
    * with SRC_NO_SPLICES to skip the splice scan. */
 
   /* Intern the "<paste>" name once and reuse a single lexer across pastes: the
    * lexer is re-pointed (not reallocated) per paste, but still draws a fresh
    * sequential file_id so the file_id order matches a per-paste open exactly.
    * t1.loc is overwritten below, so the paste file_id is dead for output. */
   if (!pp->paste_name_sym) {
     pp->paste_name_sym = kit_sym_intern(pp->c, kit_slice_cstr("<paste>"));
   }
   {
     SourceSpec spec;
     memset(&spec, 0, sizeof spec);
     spec.name_sym = pp->paste_name_sym;
     spec.bytes = buf;
     spec.len = (u32)(alen + blen + 1);
     spec.flags = SRC_NO_SPLICES;
     if (pp->paste_lex) {
       lex_reset(pp->paste_lex, &spec);
     } else {
       pp->paste_lex = lex_open(pp->c, &spec);
     }
   }
   lex = pp->paste_lex;
   lex_next(lex, &t1);
   lex_next(lex, &t2);
   if (t1.kind == TOK_EOF) {
     /* Both empty (shouldn't reach here since we handled placemarkers). */
     return lhs;
   }
   if (t2.kind != TOK_NEWLINE && t2.kind != TOK_EOF) {
     compiler_panic(pp->c, pp_materialize_loc(pp, loc),
                    "token pasting yields multiple tokens, invalid");
   }
 
   /* The paste buffer is transient (a stack array re-pointed per paste), so a
    * source-derived spelling must be interned before the buffer is reused. */
   if (t1.text.kind == TEXT_SRC) {
     t1.text = text_intern_ref(pp, lex_text_slice(lex, t1.text));
   }
 
   /* Inherit positional flags from LHS (it sat in the same slot). */
   t1.flags = (u16)((t1.flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
                    (lhs.flags & (TF_AT_BOL | TF_HAS_SPACE)));
   t1.loc = loc;
   return t1;
 }
 
 /* Phase 1 (param substitution). For each parameter occurrence in the
  * body: if adjacent to ## or # (handled separately), substitute the raw
  * argument tokens; otherwise substitute the pre-expanded form. Empty raw
  * args become a TOK_PP_PLACEMARKER which phase 2 collapses.
  *
  * `no_paste` is the no-`##` fast path (set only when m->has_paste == 0, so the
  * body provably has no TOK_PP_PASTE and adj_paste is always false): instead of
  * emitting a placemarker for an empty arg, it accumulates the empty arg's
  * BOL/SPACE flags into a `carry` and ORs them onto the NEXT emitted token —
  * exactly what phase 2's placemarker-strip would do. The output is then already
  * placemarker-free and identity under phase 2, so substitute_body can skip the
  * second pass entirely. A trailing empty arg's carry has no surviving token to
  * land on and is dropped, matching phase 2 (which leaves a trailing carry
  * unused). */
 static void subst_phase1_impl(Pp* pp, const Macro* m, ArgList* a,
                               const Tok* invoke, TokVec* out, int no_paste) {
   u32 j;
   u16 carry = 0; /* no_paste only: pending BOL/SPACE from stripped empty args */
   /* Reserve a cheap upper bound up front so the per-expansion output vector
    * doesn't re-grow from cap 0 (0->8->16->32 doubling memcpys); arg expansion
    * may exceed it, which just falls back to the doubling. */
   tv_grow(pp, out, m->body_len + 4u);
   for (j = 0; j < m->body_len; ++j) {
     const Tok* bt = &m->body[j];
     if (bt->kind == TOK_PP_HASH) {
       /* §6.10.3.2: # must be followed by a parameter. EXCEPT under
        * assembler-with-cpp (__ASSEMBLER__ defined), where a `#` not before a
        * parameter is the assembler's immediate prefix (e.g. `cmp r0, #0` inside
        * a function-like macro) — pass it through literally, matching gcc/clang.
        * It is emitted mid-line, so it never starts a directive downstream. */
       if (j + 1 >= m->body_len || m->body[j + 1].kind != TOK_PP_PARAM) {
         if (mt_get(pp, pp->sym_assembler)) {
           Tok lit = *bt;
           if (carry) {
             lit.flags |= carry;
             carry = 0;
           }
           tv_push(pp, out, lit);
           continue;
         }
         compiler_panic(pp->c, pp_materialize_loc(pp, bt->loc),
                        "'#' is not followed by a macro parameter");
       }
       {
         u32 p = m->body[j + 1].aux;
         u32 lo = a->raw_start[p];
         u32 hi = a->raw_start[p + 1];
         Tok s = make_stringize(pp, a->raw, lo, hi, invoke->loc);
         s.flags = (u16)((s.flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
                         (bt->flags & (TF_AT_BOL | TF_HAS_SPACE)));
         if (carry) {
           s.flags |= carry;
           carry = 0;
         }
         tv_push(pp, out, s);
         ++j;
         continue;
       }
     }
     if (bt->kind == TOK_PP_PARAM) {
       u32 p = bt->aux;
       int adj_paste =
           (j > 0 && m->body[j - 1].kind == TOK_PP_PASTE) ||
           (j + 1 < m->body_len && m->body[j + 1].kind == TOK_PP_PASTE);
 
       u32 lo, hi;
       if (adj_paste) {
         lo = a->raw_start[p];
         hi = a->raw_start[p + 1];
       } else {
         lo = a->exp_start[p];
         hi = a->exp_start[p + 1];
       }
 
       if (lo == hi) {
         if (no_paste) {
           /* No-`##` fast path: a placemarker only ever matters as a phase-2
            * strip carrier, so skip it and accumulate its BOL/SPACE for the
            * next emitted token (identical to phase 2's strip carry). */
           carry |= bt->flags & (TF_AT_BOL | TF_HAS_SPACE);
         } else {
           /* Empty argument → placemarker. */
           Tok pm;
           memset(&pm, 0, sizeof(pm));
           pm.kind = TOK_PP_PLACEMARKER;
           pm.flags = bt->flags & (TF_AT_BOL | TF_HAS_SPACE);
           pm.loc = invoke->loc;
           tv_push(pp, out, pm);
         }
       } else {
         u32 k;
         int first = 1;
         Tok* src = adj_paste ? a->raw : a->exp;
         for (k = lo; k < hi; ++k) {
           Tok t = src[k];
           if (first) {
             t.flags = (u16)((t.flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
                             (bt->flags & (TF_AT_BOL | TF_HAS_SPACE)));
             first = 0;
           }
           if (carry) {
             t.flags |= carry;
             carry = 0;
           }
           tv_push(pp, out, t);
         }
       }
       continue;
     }
     if (carry) {
       Tok t = *bt;
       t.flags |= carry;
       carry = 0;
       tv_push(pp, out, t);
     } else {
       tv_push(pp, out, *bt);
     }
   }
   /* A trailing carry (empty arg(s) at the very end with no following token)
    * is dropped, matching phase 2: its strip pass leaves a trailing carry with
    * no surviving token to OR onto. */
   (void)carry;
 }
 
 /* Phase 1 in the general (placemarker-emitting) form; phase 2 follows. */
 static void subst_phase1(Pp* pp, const Macro* m, ArgList* a, const Tok* invoke,
                          TokVec* out) {
   subst_phase1_impl(pp, m, a, invoke, out, /*no_paste=*/0);
 }
 
 /* Phase 2 (paste). Walk the post-substitute buffer; for each TOK_PP_PASTE,
  * splice the previous output token with the next input token. Then strip
  * remaining placemarkers. */
 static void subst_phase2(Pp* pp, const Tok* in, u32 nin, const Tok* invoke,
                          TokVec* out) {
   u32 i;
   /* Track whether any placemarker actually lands in `out`. The strip pass
    * below is a full O(out->n) re-walk + element-wise compaction; the common
    * macro body has no empty-arg/paste placemarkers, so when none was emitted
    * we skip the second walk entirely. */
   int had_pm = 0;
   /* Phase-2 output is at most the input length (paste/placemarker only
    * shrink); reserve it to avoid re-growing from cap 0. */
   tv_grow(pp, out, nin);
   for (i = 0; i < nin; ++i) {
     Tok t = in[i];
     if (t.kind == TOK_PP_PASTE) {
       Tok lhs, rhs, pasted;
       if (out->n == 0 || i + 1 >= nin) {
         compiler_panic(pp->c, pp_materialize_loc(pp, invoke->loc),
                        "'##' at start or end of replacement list");
       }
       lhs = out->data[--out->n];
       rhs = in[++i];
       /* paste_tokens collapses a placemarker operand to the other side, so
        * the result is a placemarker only when both operands were. */
       pasted = paste_tokens(pp, lhs, rhs, invoke->loc);
       if (pasted.kind == TOK_PP_PLACEMARKER) had_pm = 1;
       tv_push(pp, out, pasted);
       continue;
     }
     if (t.kind == TOK_PP_PLACEMARKER) had_pm = 1;
     tv_push(pp, out, t);
   }
   if (!had_pm) return; /* no placemarker to strip — skip the compaction walk */
   /* Strip placemarkers, preserving leading-space flag on the next token. */
   {
     u32 r = 0, w = 0;
     u16 carry = 0;
     for (r = 0; r < out->n; ++r) {
       if (out->data[r].kind == TOK_PP_PLACEMARKER) {
         carry |= out->data[r].flags & (TF_AT_BOL | TF_HAS_SPACE);
         continue;
       }
       if (carry) {
         out->data[r].flags |= carry;
         carry = 0;
       }
       if (w != r) out->data[w] = out->data[r];
       ++w;
     }
     out->n = w;
   }
 }
 
 /* Wrapper: phases 1 and 2 in sequence, plus invocation-loc / flag transfer. */
 static void substitute_body(Pp* pp, const Macro* m, ArgList* a,
                             const Tok* invoke, TokVec* out) {
   TokVec phase1 = {0};
   u32 i;
   if (!m->has_paste) {
     /* No `##` in the body: phase 2 (paste + placemarker strip) is the identity
      * on phase-1 output once empty-arg placemarkers are elided. Run phase 1 in
      * its no_paste mode straight into `out`, skipping the second full-body copy
      * + re-walk. Byte-identical: no_paste carries empty-arg BOL/SPACE forward
      * exactly as the strip pass would, and with no TOK_PP_PASTE there is no
      * paste work to do. */
     subst_phase1_impl(pp, m, a, invoke, out, /*no_paste=*/1);
   } else {
     subst_phase1(pp, m, a, invoke, &phase1);
     subst_phase2(pp, phase1.data, phase1.n, invoke, out);
   }
   /* Invocation flags onto first emitted token. */
   if (out->n) {
     out->data[0].flags =
         (u16)((out->data[0].flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
               (invoke->flags & (TF_AT_BOL | TF_HAS_SPACE)));
   }
   /* Locations to invocation site. */
   for (i = 0; i < out->n; ++i) out->data[i].loc = invoke->loc;
 }
 
 /* Expand a function-like macro invocation: peek for `(`, collect args,
  * pre-expand them, substitute the body, push the result. Returns 1 if
  * the invocation was performed, 0 if there was no `(` (the caller should
  * emit the identifier as-is). */
 static int try_expand_func_macro(Pp* pp, Macro* m, const Tok* invoke) {
   int saw_ws;
   ArgList args;
   TokVec body = {0};
   Tok close_tok;
 
   if (!peek_for_invoke_paren(pp, &saw_ws)) {
     return 0;
   }
   (void)saw_ws;
   read_invocation_args(pp, m, invoke->loc, &args);
   /* Note: assigned to silence unused-result; we don't use the close tok yet. */
   close_tok.kind = 0;
   (void)close_tok;
   preexpand_args(pp, &args);
 
   substitute_body(pp, m, &args, invoke, &body);
 
   /* Push the substituted body as a disabled frame owned by `m`: the macro is
    * unavailable for the whole rescan of this replacement (cpplib model), so a
    * recursive occurrence of `m` is returned permanently un-expanded. The
    * arguments were already pre-expanded above with `m` NOT disabled. */
   push_buf_uniform(pp, body.data, body.n, m);
   return 1;
 }
 
 /* ============================================================
  * pp_next_raw — mutual recursion entry (called from expand_arg_to_eof)
  * Defined here; also declared in pp_priv.h so pp.c can call it.
  * ============================================================ */
 
 /* pp_pull_into: shared core of pp_next_raw / pp_next_into. Reads from the
  * top source into *out, applies macro expansion when an identifier names a
  * macro that isn't blue-painted, and consumes directives in-place.
  *
  * `skip_nl` short-circuits the parser-facing newline drop ONE FRAME LOWER: the
  * public pp_next (the C-parser feed) discards every non-directive TOK_NEWLINE
  * at the top of its loop, and 51% of all produced tokens are exactly those
  * newlines. When skip_nl is set, such a newline is consumed here (the loop just
  * continues) instead of being returned and round-tripped through pp_next's sret
  * — the token never escapes this frame. This is sound because a TOK_NEWLINE can
  * never be a directive introducer (TOK_PP_HASH), a macro-name TOK_IDENT, or
  * TOK_EOF, so in the un-skipped path it falls straight through to the trailing
  * `return`; continuing the loop instead is byte-identical to the caller having
  * read and dropped it, modulo the (idempotent) arena-reset re-check at the loop
  * head. skip_nl is set ONLY by pp_next, which never runs during #if expansion,
  * so the in_if_expansion `defined`-operator state machine below is unaffected.
  * pp_emit_text and the pragma-swallow / hash-peek paths pass skip_nl=0 so they
  * still observe newlines for -E line reconstruction and directive termination.
  *
  * Tok is 24B, so the by-value form (pp_next_raw) returns indirectly via sret
  * and pays a 24B inter-frame copy at every `return t`. Writing through `out`
  * lets the token — and lex_next's own sret — land in the caller's slot
  * directly. The hot SRC_BUF case (macro replay) is inlined here so the common
  * token never pays a call into src_next_raw_into. */
 static void pp_pull_into(Pp* pp, Tok* out, int skip_nl) {
   u8 src_kind;
   for (;;) {
     TokSrc* s;
     /* Reclaim the transient expansion scratch whenever the source stack has
      * drained back to a lexer. A SRC_BUF always sits above every SRC_LEX (a
      * macro can't #include), so a lexer on top means no expansion buffer is
      * live -- nothing in xarena is referenced, and the previously returned
      * token was a value copy with a pool-interned spelling. This is the only
      * reset site; it never fires mid-expansion (a SRC_BUF, incl. the
      * scope_top arg-prescan buffer, is on top then) nor during raw argument
      * collection (which uses src_next_raw, not this function). Skipped inside
      * #if expansion, whose bounded condition keeps its scratch live. The reset
      * is further skipped when the xarena is already at its baseline (a plain
      * lexer run that allocated no scratch): kit_arena_is_empty is the pristine
      * guard, so the dead reset on plain-token runs is elided -- byte-identical
      * since resetting an already-pristine arena does nothing. */
     if (!pp->in_if_expansion &&
         (pp->nsources == 0 || pp->sources[pp->nsources - 1].kind == SRC_LEX)) {
       if (!kit_arena_is_empty(pp->xarena)) kit_arena_reset(pp->xarena);
     }
     /* Fast path: top source is a non-exhausted SRC_BUF (the dominant
      * macro-replay case). Pull the token and kind inline so the common token
      * avoids the call + sret setup of src_next_raw_into. The scope_top-EOF and
      * #line-delta cases never apply to a plain in-bounds SRC_BUF read, so they
      * stay on the general (cold) path. */
     if (pp->nsources != 0 &&
         (s = &pp->sources[pp->nsources - 1])->kind == SRC_BUF && s->i < s->n) {
       *out = s->toks[s->i];
       src_kind = SRC_BUF;
       /* Pointer-replayed body: apply the per-invocation loc and (first token
        * only) BOL/SPACE flags the old fresh-copy path baked in. Identical to
        * the slow read site in src_next_raw_into so both paths agree. */
       if (s->has_loc_override) {
         if (s->i == 0) {
           out->flags = (u16)((out->flags & ~(TF_AT_BOL | TF_HAS_SPACE)) |
                              s->first_flags_or);
         }
         out->loc = s->loc_override;
       }
       ++s->i;
     } else {
       src_next_raw_into(pp, out, &src_kind);
     }
     if (out->kind == TOK_EOF) return;
     /* §C.3a/§C.3b: absorb the parser-facing non-directive newline drop here so
      * it never round-trips up through pp_next's sret. A newline is whitespace,
      * not a directive introducer, so dropping it before the directive / macro
      * checks below is the same as pp_next dropping it after the return.
      *
      * §C.3b drains a run of newlines in place via src_next_raw_into rather than
      * `continue`-ing the outer loop per newline. The outer loop's only
      * per-iteration work before this point is the xarena reclaim
      * (kit_arena_reset, a real call) and the SRC_BUF fast-path probe; for a
      * newline both are redundant — the source stack is unchanged between a
      * newline and the token that follows it (a newline never pushes a source,
      * and a SRC_BUF can only sit *above* the SRC_LEX a file newline came from,
      * so it is not on top here). Skipping them is byte-identical to the §C.3a
      * `continue` while removing one kit_arena_reset call + probe per dropped
      * newline (51% of produced tokens). The drained token re-enters the normal
      * directive / macro flow below, so a `#` or content token after the
      * newline is handled exactly as before. */
     if (skip_nl && out->kind == TOK_NEWLINE) {
       do {
         src_next_raw_into(pp, out, &src_kind);
       } while (out->kind == TOK_NEWLINE);
       if (out->kind == TOK_EOF) return;
     }
     if (out->kind == TOK_PP_HASH && (out->flags & TF_AT_BOL) &&
         src_kind == SRC_LEX) {
       process_directive(pp, out->loc);
       /* No synthesized newline: the comparator collapses
        * whitespace, so blank-line replacement of consumed
        * directives isn't observable here. Directives that produce
        * content (e.g. #include, #embed, #pragma) push their own
        * tokens onto the source stack, which the next loop
        * iteration picks up. */
       continue;
     }
     /* While expanding an #if condition, suppress macro expansion of
      * `defined`-operator operands so a `defined(X)` produced by a
      * macro body whose argument was pasted via ## doesn't accidentally
      * expand an already-defined X to its body (typically empty). See
      * the `defined_skip` field comment in pp_priv.h. */
     if (pp->in_if_expansion) {
       if (pp->defined_skip == 2) {
         if (out->kind == TOK_PUNCT && tok_punct(out) == '(') {
           pp->defined_skip = 3;
         } else if (out->kind == TOK_IDENT) {
           /* `defined IDENT` (no parens): mark the operand and reset. */
           out->flags |= TF_NO_EXPAND;
           pp->defined_skip = 0;
         } else {
           pp->defined_skip = 0;
         }
       } else if (pp->defined_skip == 3) {
         if (out->kind == TOK_IDENT) {
           out->flags |= TF_NO_EXPAND;
           pp->defined_skip = 4;
         } else if (out->kind == TOK_PUNCT && tok_punct(out) == ')') {
           pp->defined_skip = 0;
         }
       } else if (pp->defined_skip == 4) {
         if (out->kind == TOK_PUNCT && tok_punct(out) == ')') {
           pp->defined_skip = 0;
         }
       } else if (out->kind == TOK_IDENT && tok_ident(out) == pp->sym_defined) {
         pp->defined_skip = 2;
       }
     }
     if (out->kind == TOK_IDENT && (out->flags & TF_NO_EXPAND) == 0) {
       Sym id = tok_ident(out);
 
       /* Dynamic predefined macros: __LINE__ / __FILE__ /
        * __DATE__ / __TIME__. Always expand, ignoring the macro
        * table. */
       if (id == pp->sym_line__) {
         char tmp[16], buf[16];
         int k = 0, j = 0;
         u32 ln = pp_materialize_loc(pp, out->loc).line;
         if (ln == 0)
           buf[k++] = '0';
         else {
           while (ln) {
             tmp[j++] = (char)('0' + ln % 10);
             ln /= 10;
           }
           while (j > 0) buf[k++] = tmp[--j];
         }
         out->kind = TOK_NUM;
         out->aux = 0;
         out->text = text_intern_ref(pp, (KitSlice){.s = buf, .len = (size_t)k});
         return;
       }
       if (id == pp->sym_file__) {
         /* pp_materialize_file resolves the __FILE__ name at this location,
          * applying any active #line overlay; no current-lexer probe needed. */
         Sym name = pp_materialize_file(pp, out->loc);
         size_t nlen = 0;
         const char* nstr = NULL;
         char* buf;
         if (name) {
           KitSlice s = kit_sym_str(pp->pool->c, name);
           nstr = s.s;
           nlen = s.len;
         }
         /* The source name is the raw filesystem path (or a #line override,
          * destringized to logical bytes by do_line). Re-stringize it as a
          * valid C string literal: escape '\\' and '"'. On POSIX paths use
          * '/' so this was a no-op; on Windows the path holds backslashes
          * (e.g. C:\\Users\\...), and emitting them raw turns '\\U'/'\\u'/'\\x'
          * into bogus escape sequences (the "malformed UCN" on '\\Users'). */
         {
           size_t bn = 0;
           size_t i;
           buf = (char*)arena_alloc(pp->xarena, nlen * 2 + 2, 1);
           buf[bn++] = '"';
           for (i = 0; i < nlen; ++i) {
             char ch = nstr[i];
             if (ch == '\\' || ch == '"') buf[bn++] = '\\';
             buf[bn++] = ch;
           }
           buf[bn++] = '"';
           out->kind = TOK_STR;
           out->aux = 0;
           out->text = text_intern_ref(pp, (KitSlice){.s = buf, .len = bn});
         }
         return;
       }
       if (id == pp->sym_date__) {
         out->kind = TOK_STR;
         out->aux = 0;
         out->text = text_sym_ref(pp->val_date_str);
         return;
       }
       if (id == pp->sym_time__) {
         out->kind = TOK_STR;
         out->aux = 0;
         out->text = text_sym_ref(pp->val_time_str);
         return;
       }
       if (id == pp->sym__pragma) {
         if (try_expand_pragma_op(pp, out)) continue;
         /* No '(' — fall through and emit as plain ident. */
       }
 
       {
         Macro* m = mt_get(pp, id);
         if (m && m->disabled_depth != 0) {
           /* cpplib: the macro is disabled because one of its own
            * replacement-list frames is still being rescanned. Mark this
            * occurrence permanently un-expandable, so the bit survives if this
            * token is later captured as an argument, substituted into another
            * macro, or replayed (this is what keeps `foo(foo)(1)` and
            * `id(foo(foo)(1))` as `foo(1)`). */
           out->flags |= TF_NO_EXPAND;
           return;
         }
         if (m) {
           if (!m->is_func) {
             expand_object_macro(pp, m, out);
             continue;
           }
           if (try_expand_func_macro(pp, m, out)) {
             continue;
           }
           /* Function-like macro name NOT followed by '(': emit as a plain
            * identifier WITHOUT setting TF_NO_EXPAND, so a later rescan that
            * sees a '(' supplied by another macro can still expand it (keeps
            * `id(f L 1))` with `#define L (` able to become `1`). */
         }
       }
     }
     return;
   }
 }
 
 /* pp_next_raw: public out-pointer form (pp.h), every token surfaced
  * (TOK_NEWLINE preserved for pp_emit_text and for directive/pragma line
  * termination). The mutual-recursion entry: expand_arg_to_eof and the -E loop
  * call it; it drives directives and expansion. */
 void pp_next_raw(Pp* pp, Tok* out) { pp_pull_into(pp, out, /*skip_nl=*/0); }
 
 /* pp_next_into: out-pointer form for the C-parser feed (pp_next_parse).
  * Identical to pp_next_raw but drops non-directive newlines internally, so the
  * 51% of produced tokens that pp_next_parse would otherwise discard never
  * escape this frame. Declared in pp_priv.h. */
 void pp_next_into(Pp* pp, Tok* out) { pp_pull_into(pp, out, /*skip_nl=*/1); }