kit

pp_priv.h (22228B)

---

 /* pp_priv.h — shared types, helpers, and cross-module forward declarations
  * for the preprocessor split (pp.c / pp_expand.c / pp_directive.c).
  * NOT part of the public API; included only within lang/cpp/pp/. */
 
 #ifndef KIT_PP_PRIV_H
 #define KIT_PP_PRIV_H
 
 #include <kit/support/symtab.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "cpp_support.h"
 #include "pp/pp.h"
 
 /* ============================================================
  * Internal token kinds
  * ============================================================ */
 
 /* Outside the range used by the lexer (TOK_KW_LAST = 0x1000). */
 #define TOK_PP_PARAM ((u16)0x1100)
 #define TOK_PP_PLACEMARKER ((u16)0x1101) /* empty-arg substitution marker */
 
 /* ============================================================
  * Types
  * ============================================================ */
 
 typedef struct Macro {
   Sym name;
   LocRef def_loc;
   u8 is_func;
   u8 is_variadic;
   /* Pure cache of the body (NOT part of macro identity): 1 iff the body
    * contains at least one TOK_PP_PASTE (`##`). Lets a no-`##` body be
    * replayed by pointer without the per-invocation copy + paste pass. */
   u8 has_paste;
   u8 pad[1];
   /* cpplib-style macro-disabled count (replaces the Prosser hideset): the
    * number of this macro's replacement-list frames currently live on the
    * source stack. Incremented when a replacement frame for this macro is
    * pushed, decremented when it pops (see push_buf_uniform/replay + src_pop).
    * While > 0 the macro is unavailable for expansion, so a recursive
    * occurrence in its own (or a nested) replacement is returned with the
    * permanent TF_NO_EXPAND bit set instead of re-expanding. Auto-zeros: do_define
    * mints Macro via arena_znew. */
   u16 disabled_depth;
   u32 n_params;
   Sym* params; /* parameter names                              */
   Tok* body;   /* body tokens; TOK_PP_PARAM kind + aux=param idx */
   u32 body_len;
 } Macro;
 
 /* MacroTab = Sym -> Macro* (NULL = not a macro). Sym-indexed dense table:
  * the macro binding is read with one in-range load on every identifier in
  * the expander's hot loop, replacing a per-identifier hashmap probe. */
 KIT_SYMTAB_DEFINE(MacroTab, Macro*);
 
 typedef enum { SRC_LEX = 1, SRC_BUF = 2 } SrcKind;
 
 /* Multiple-include-guard detection state for a SRC_LEX file (the classic
  * "controlling macro" optimization). A file whose entire body is wrapped in
  * `#ifndef GUARD ... #endif` (with nothing but whitespace/comments outside)
  * can be skipped on a later #include while GUARD is defined — it would emit no
  * tokens anyway. Detected live as the file is first lexed (no second scan):
  * START -> IN on the opening `#ifndef GUARD`; IN -> AFTER on the matching
  * `#endif`; any code before the opening directive or after the controlling
  * `#endif` -> FAILED. Committed to the IncCache at src_pop (see pp.c). */
 typedef enum GuardState {
   GUARD_START = 0, /* only whitespace/newlines seen so far              */
   GUARD_IN = 1,    /* inside the controlling #ifndef ... #endif         */
   GUARD_AFTER = 2, /* controlling #endif closed; only ws may follow     */
   GUARD_FAILED = 3 /* not a whole-file guard                            */
 } GuardState;
 
 typedef struct TokSrc {
   u8 kind;
   /* When set on a SRC_BUF: src_next_raw returns TOK_EOF when this is
    * the top source and it's exhausted, instead of popping. The caller
    * (e.g. argument pre-expansion) explicitly pops the scope when done.
    * This bounds expansion to a single argument's token stream. */
   u8 scope_top;
   /* Read-time loc/flags override for a pointer-replayed macro body (SRC_BUF).
    * When has_loc_override, every token read from this buffer gets
    * out->loc = loc_override, and the FIRST token (s->i == 0 at read) also has
    * its BOL/SPACE bits replaced by first_flags_or. This reproduces the
    * mutations expand_object_macro used to bake into a fresh body copy, so the
    * immutable definition-time m->body can be replayed in place. 0 = no
    * override (every push_buf/push_buf_uniform memsets the whole TokSrc). */
   u8 has_loc_override;
   u8 pad[1];
   u16 first_flags_or;
   /* SRC_LEX */
   Lexer* lex;
   /* SRC_BUF */
   Tok* toks;
   /* cpplib macro-disabled frame owner. Non-NULL only on a macro
    * replacement-list frame (object-like replay, function-like substituted body,
    * paste result); pushing such a frame increments owner->disabled_depth and
    * src_pop decrements it, so the macro stays unavailable for the whole rescan
    * of its replacement (and any nested replacement above it). NULL on every
    * non-macro buffer (pushed-back peek tokens, #pragma/#embed payloads,
    * arg-prescan scopes); every push_buf* memsets the whole TokSrc, so NULL is
    * the default. */
   Macro* disabled_owner;
   /* Per-token loc applied when has_loc_override (see scope_top block). */
   LocRef loc_override;
   u32 i;
   u32 n;
   /* #line state is recorded positionally in the source's SrcInfo overlay
    * segments (see SrcInfo / pp_materialize_loc), not on the frame: a lean
    * token carries only a byte offset, and line/file are resolved on demand. */
   /* SRC_LEX only: the index into pp->inc_dirs from which a `#include_next`
    * (or `__has_include_next`) appearing in this file begins its search —
    * i.e. one past the search dir this file was itself found in. 0 for the
    * top-level translation unit and for files found via the includer-relative
    * ("...") step or an absolute path, so an `#include_next` there scans the
    * whole search path (matching GCC). */
   u32 inc_next_start;
   /* Multiple-include-guard detection (SRC_LEX only). guard_if_base is the
    * if-stack depth at file entry, used to recognize the controlling #endif;
    * path_key is the IncCache key (interned resolved path, 0 for the top-level
    * TU and other non-cached sources) used to commit the memo at src_pop. */
   u8 guard_state; /* GuardState */
   u8 once;        /* #pragma once seen in this file */
   Sym guard_macro;
   u32 guard_if_base;
   Sym path_key;
 } TokSrc;
 
 typedef enum IfState {
   IF_INCLUDE = 1,   /* group active, emit code                       */
   IF_SEEK_TRUE = 2, /* skip, looking for the first true elif/else    */
   IF_DONE = 3,      /* skip, already had a true branch               */
 } IfState;
 
 typedef struct IfFrame {
   u8 state;
   u8 has_else;
   u8 pad[2];
   LocRef loc;
 } IfFrame;
 
 /* ============================================================
  * Source-info registry (lazy loc + retained text)
  * ============================================================ */
 
 /* A #line overlay segment: from byte offset `off` onward (until the next
  * segment), the reported line number is the physical line plus `delta`, and
  * __FILE__ reports `file` (0 = the source's registry name). Recorded by do_line
  * in offset order; consulted by pp_materialize_loc / pp_materialize_file. */
 typedef struct LineSeg {
   u32 off;
   i32 delta;
   Sym file;
 } LineSeg;
 
 /* Per-source retained metadata, keyed by file_id. The folded logical buffer and
  * splice table are retained until pp_free so that (a) TEXT_SRC spellings stay
  * resolvable after the lexer is popped (macro bodies, parser lookahead/replay),
  * and (b) line/col can be reconstructed lazily from a byte offset. The line
  * index is built on first materialization by scanning the buffer for '\n' and
  * merging the splice fold points. */
 typedef struct SrcInfo {
   const char* buf; /* folded logical buffer (borrowed/arena/owned) */
   u32 len;
   u8 owns_buf;   /* PP frees buf at pp_free (a folded heap copy) */
   u8 line_built; /* line_off has been computed */
   u32 shebang_off;
   u32* splices; /* fold offsets; PP frees at pp_free */
   u32 nsplices;
   u32* line_off; /* line_off[k] = start offset of physical line k+1 */
   u32 nlines;
   u32 line_cursor; /* last line index returned — loc materialization is
                     * near-monotonic, so this makes the common lookup O(1) */
   LineSeg* segs;   /* #line overlay, offset-ordered */
   u32 nsegs;
   u32 segs_cap;
 } SrcInfo;
 
 /* Sym-keyed hashmaps below (include caches). See core/hashmap.h. */
 #include <kit/support/hashmap.h>
 
 /* IncCache = Sym(resolved-path) -> cached header bytes. The same guarded
  * header included many times across a TU is read from disk once; later
  * inclusions reuse the arena-resident bytes (lex_open only borrows
  * them and the pp arena keeps them alive until pp_free), saving the
  * open+fstat+read+close + malloc/memcpy/free per repeat. Keyed strictly
  * on the resolved-path STRING so distinct spellings/symlinks never
  * over-merge; the cached bytes are byte-identical to a fresh read. */
 typedef struct IncEntry {
   const u8* data;
   size_t size;
   /* Multiple-include optimization (committed at src_pop, consulted at the next
    * #include of the same resolved path). guard = the file's controlling
    * #ifndef macro (0 = none); once = the file carried #pragma once. */
   Sym guard;
   u8 once;
 } IncEntry;
 static inline u32 inc_hash_(Sym s) { return kit_hash_u32((u32)s); }
 KIT_HASHMAP_DEFINE(IncCache, Sym, IncEntry, inc_hash_);
 
 /* IncResolveMap = Sym(spelling key) -> resolved (path Sym, system flag).
  * Memoizes the winner of the -I/-isystem dir search so a repeated request
  * for the same header spelling skips the ENOENT storm of failed opens on
  * the earlier dirs. The key folds the system flag (and, for quoted form,
  * the includer directory) into the spelling so the resolved result is the
  * exact one a fresh search would produce — byte-identical resolved path
  * and resolved_system flag feeding source_add_include / DWARF / -M. */
 typedef struct IncResolved {
   Sym path;       /* interned resolved path string                  */
   u8 system;      /* the resolved_system flag (winning -isystem)   */
   u32 next_start; /* inc_dirs index a #include_next from the   */
                   /* resolved file starts at (see TokSrc)      */
 } IncResolved;
 static inline u32 incres_hash_(Sym s) { return kit_hash_u32((u32)s); }
 KIT_HASHMAP_DEFINE(IncResolveMap, Sym, IncResolved, incres_hash_);
 
 /* ============================================================
  * Pp struct (definition shared across all three TUs)
  * ============================================================ */
 
 struct Pp {
   Compiler* c;
   Pool* pool;
 
   /* Source stack — top of stack is sources[nsources-1]. */
   TokSrc* sources;
   u32 nsources;
   u32 sources_cap;
 
   /* Per-file retained metadata, indexed by compiler file_id (dense from 1).
    * Holds the retained source buffer + splice table + lazy line index + #line
    * overlay so loc/text materialize after the lexer is popped. */
   SrcInfo* srcinfo;
   u32 srcinfo_cap;
 
   /* Macro table (Sym-indexed dense; value = Macro*, NULL = not a macro). */
   MacroTab macros;
 
   /* Header-content cache (key = interned resolved-path Sym, value =
    * arena-resident bytes). Collapses N re-includes of the same header to
    * one read + N-1 cache hits. Lazily initialized on first #include; its
    * slot table is backed by inc_cache_heap (an arena adapter over
    * pp->arena), so nothing needs freeing at pp_free — the arena is. */
   IncCache inc_cache;
   KitHeap inc_cache_heap;
   u8 inc_cache_ready;
 
   /* Include-resolution memo (spelling -> winning resolved path + system
    * flag). Shares inc_cache_heap (gated by inc_cache_ready). */
   IncResolveMap inc_resolve;
 
   /* Conditional inclusion stack (#if / #ifdef / #ifndef → #endif). */
   IfFrame* ifstk;
   u32 ifstk_n;
   u32 ifstk_cap;
 
   /* Include directories (stage 9). */
   struct {
     const char* path;
     u8 system;
   }* inc_dirs;
   u32 ninc_dirs;
   u32 inc_dirs_cap;
 
   /* Current #pragma pack maximum field alignment. 0 means natural. */
   u32 pack_align;
   u32 pack_stack[16];
   u32 pack_stack_n;
 
   /* Permanent arena: macro bodies, params, #include file data.
    * Lives until pp_free. */
   KitArena* arena;
 
   /* Transient expansion scratch: the macro-substitution token buffers
    * (tv_grow), arg slices, paste/stringize buffers -- everything that
    * backs a SRC_BUF or is consumed within one expansion. Reset to its
    * high-water mark by pp_next_raw whenever the source stack drains back to a
    * lexer (no SRC_BUF live), so memory is O(expansion depth), not
    * O(expansions). Nothing the caller keeps points in here: tokens are
    * returned by value and spellings are interned in the pool. */
   KitArena* xarena;
 
   /* Cached interned identifiers used for directive recognition. */
   Sym sym_define;
   Sym sym_undef;
   Sym sym_include;
   Sym sym_include_next;     /* GCC/clang #include_next extension       */
   Sym sym_assembler;        /* __ASSEMBLER__ — asm-with-cpp `#` leniency */
   Sym sym_has_include;      /* __has_include() #if operator            */
   Sym sym_has_include_next; /* __has_include_next() #if operator       */
   Sym sym_if;
   Sym sym_ifdef;
   Sym sym_ifndef;
   Sym sym_elif;
   Sym sym_else;
   Sym sym_endif;
   Sym sym_line;
   Sym sym_pragma;
   Sym sym_once; /* "once" — #pragma once (multiple-include opt) */
   Sym sym_error;
   Sym sym_warning;
   Sym sym_embed;
   Sym sym_defined;
   Sym sym_va_args;
   Sym sym_line__; /* __LINE__   */
   Sym sym_file__; /* __FILE__   */
   Sym sym_date__; /* __DATE__   */
   Sym sym_time__; /* __TIME__   */
   Sym sym_stdc__; /* __STDC__   */
   Sym sym_stdc_hosted__;
   Sym sym_stdc_version__;
   Sym sym__pragma;   /* _Pragma operator */
   Sym sym_pragma_kw; /* "pragma" — for synthesized #pragma */
 
   /* Pre-formatted "Mmm dd yyyy" / "hh:mm:ss" string spellings for
    * __DATE__ and __TIME__, derived from SOURCE_DATE_EPOCH (or
    * time(NULL) if unset). */
   Sym val_date_str;
   Sym val_time_str;
 
   /* Token-paste (`##`) scratch. paste_name_sym is the interned "<paste>" name,
    * cached so each paste open doesn't re-intern the literal. paste_lex is a
    * single reused memory lexer (re-pointed via lex_reset_mem per paste) so the
    * thousands of tiny paste buffers don't each alloc/free a 288B Lexer. Both
    * are lazily initialized on the first paste and torn down in pp_free. */
   Sym paste_name_sym;
   Lexer* paste_lex;
 
   /* Defined-operator handling during #if expansion.
    *
    * The first prepass in eval_if_expr replaces `defined X` / `defined
    * (X)` literally found in the directive line, but `defined()` can
    * also come from macro bodies (mingw's intrin-impl.h uses
    * `defined(__INTRINSIC_DEFINED_ ## name)` inside a #define).  When
    * the expander processes such a body, the identifier inside
    * `defined(...)` must NOT be macro-expanded — otherwise an empty
    * macro X would turn `defined(X)` into `defined()` and the
    * post-expansion prepass would reject it.
    *
    * This pair of fields tracks the state across `pp_next_raw` calls
    * within `expand_for_if`:
    *   in_if_expansion: 1 inside an #if's expand_arg_to_eof call
    *   defined_skip:    0 normally; 2 after emitting `defined` (a bare
    *                    `defined IDENT` marks the IDENT and resets, while
    *                    a `(` advances to 3); 3 after `defined (` (waiting
    *                    for the operand IDENT, advancing to 4); 4 after
    *                    `defined ( IDENT` (waiting for the closing `)`).
    * The expander uses these to mark the operand IDENT TF_NO_EXPAND
    * before the macro-expansion check at the head of pp_next_raw. */
   u8 in_if_expansion;
   u8 defined_skip;
   /* Set while read_directive_line is pulling a directive's own tokens from the
    * lexer, so the multiple-include-guard content check in src_next_raw_into
    * doesn't count directive-internal tokens (e.g. the `ifndef` / macro name of
    * the guard itself) as file content. */
   u8 reading_directive;
   /* cc parser-feed mode: set when the primary source is pushed with
    * SRC_PARSER_FEED, so every SRC_LEX lexer (primary + each #include) is opened
    * with newline suppression (the parser drops non-directive newlines). The
    * -E / cpp path leaves this clear so newlines surface for text
    * reconstruction. */
   u8 parser_feed;
 };
 
 /* ============================================================
  * Allocation helpers (defined in pp.c, used everywhere)
  * ============================================================ */
 
 static inline Heap* pp_heap(Pp* pp) {
   return kit_compiler_context(pp->c)->heap;
 }
 
 static inline void* pp_xrealloc(Pp* pp, void* p, size_t old_n, size_t new_n,
                                 size_t align) {
   Heap* h = pp_heap(pp);
   void* q = h->realloc(h, p, old_n, new_n, align);
   if (!q) compiler_panic(pp->c, (SrcLoc){0, 0, 0}, "pp: out of memory");
   return q;
 }
 
 static inline void pp_xfree(Pp* pp, void* p, size_t n) {
   if (p) pp_heap(pp)->free(pp_heap(pp), p, n);
 }
 
 /* ============================================================
  * Lean-token constructors (synthetic tokens)
  * ============================================================ */
 
 /* text_none_ref / text_sym_ref / locref_none live in lex.h (shared with the
  * parser). text_intern_ref additionally interns, so it needs the Pp pool. */
 static inline TextRef text_intern_ref(Pp* pp, KitSlice sl) {
   return text_sym_ref(kit_sym_intern(pp->pool->c, sl));
 }
 
 /* ============================================================
  * Token-vector helpers
  * ============================================================ */
 
 typedef struct TokVec {
   Tok* data;
   u32 n;
   u32 cap;
 } TokVec;
 
 static inline void tv_grow(Pp* pp, TokVec* v, u32 want) {
   u32 nc;
   if (v->cap >= want) return;
   nc = v->cap ? v->cap * 2 : 8;
   while (nc < want) nc *= 2;
   {
     Tok* nb = arena_array(pp->xarena, Tok, nc);
     if (v->n) memcpy(nb, v->data, sizeof(Tok) * v->n);
     v->data = nb;
     v->cap = nc;
   }
 }
 
 static inline void tv_push(Pp* pp, TokVec* v, Tok t) {
   tv_grow(pp, v, v->n + 1);
   v->data[v->n++] = t;
 }
 
 /* Growable char buffer (arena-backed). */
 typedef struct CharBuf {
   char* data;
   u32 len;
   u32 cap;
 } CharBuf;
 
 static inline void cb_append(Pp* pp, CharBuf* b, const char* s, u32 n) {
   if (b->len + n > b->cap) {
     u32 nc = b->cap ? b->cap * 2 : 64;
     while (nc < b->len + n) nc *= 2;
     {
       char* nb = (char*)arena_alloc(pp->xarena, nc, 1);
       if (b->len) memcpy(nb, b->data, b->len);
       b->data = nb;
       b->cap = nc;
     }
   }
   if (n) memcpy(b->data + b->len, s, n);
   b->len += n;
 }
 
 static inline void cb_putc(Pp* pp, CharBuf* b, char c) {
   cb_append(pp, b, &c, 1);
 }
 
 /* ============================================================
  * Cross-module forward declarations
  * ============================================================ */
 
 /* --- pp.c (source stack) → pp_expand.c, pp_directive.c --- */
 /* Out-pointer form (hot path); src_next_raw is the by-value shim over it for
  * the cold/general callers. */
 void src_next_raw_into(Pp* pp, Tok* out, u8* src_kind_out);
 Tok src_next_raw(Pp* pp, u8* src_kind_out);
 void src_push(Pp* pp, TokSrc s);
 void src_pop(Pp* pp);
 void push_buf(Pp* pp, Tok* toks, u32 n);
 /* Push a macro replacement-list buffer. `owner` is the macro whose replacement
  * this is (NULL for a non-macro payload): a non-NULL owner increments
  * owner->disabled_depth at push and src_pop decrements it, so the macro stays
  * unavailable for the whole rescan of its replacement (cpplib disabled-frame
  * model). */
 void push_buf_uniform(Pp* pp, Tok* toks, u32 n, Macro* owner);
 /* Push an immutable, read-only token buffer (e.g. a no-`##` macro body) for
  * pointer-replay. The buffer is never written through s->toks; per-read
  * loc/flags overrides reproduce the mutations the old body-copy path baked in.
  * `owner` follows the same disabled-frame rule as push_buf_uniform. */
 void push_buf_replay(Pp* pp, const Tok* toks, u32 n, Macro* owner,
                      LocRef loc_override, u16 first_flags_or);
 
 /* pp_next_raw (public, in pp.h) is the mutual-recursion entry:
  * expand_arg_to_eof and the -E loop call it; it drives directives and
  * expansion. pp_next_into is the internal newline-dropping pull (skip_nl=1)
  * used by pp_next_parse. */
 void pp_next_into(Pp* pp, Tok* out);
 
 /* --- SrcInfo registry (pp.c) → pp_directive.c --- */
 /* Get (growing as needed) the SrcInfo for a file_id. */
 SrcInfo* pp_srcinfo(Pp* pp, u32 file_id);
 /* Register a freshly-opened, about-to-be-pushed lexer: adopt its folded buffer
  * + splice table + shebang offset into the SrcInfo registry (retained to
  * pp_free) so loc/text stay materializable after the lexer is popped. */
 void pp_register_srcinfo(Pp* pp, Lexer* lex);
 /* Record a #line overlay segment (effect offset, line delta, file name sym). */
 void pp_add_line_seg(Pp* pp, u32 file_id, u32 off, i32 delta, Sym file);
 /* Physical line at a location, ignoring #line overlay (used by do_line). */
 u32 pp_phys_line(Pp* pp, LocRef loc);
 /* __FILE__ name sym at a location (overlay file if set, else registry name). */
 Sym pp_materialize_file(Pp* pp, LocRef loc);
 
 /* --- pp_expand.c → pp.c, pp_directive.c --- */
 void expand_arg_to_eof(Pp* pp, Tok* in, u32 nin, TokVec* out);
 
 /* Macro binding lookup/define/undef. Inlined Sym-indexed loads (no hashmap
  * probe): mt_get runs on every identifier the expander sees. NULL/absent =
  * not a macro; #undef stores NULL. */
 static inline Macro* mt_get(Pp* pp, Sym name) {
   return MacroTab_get(&pp->macros, name);
 }
 static inline void mt_put(Pp* pp, Sym name, Macro* m) {
   MacroTab_set(&pp->macros, name, m);
 }
 static inline void mt_del(Pp* pp, Sym name) {
   MacroTab_set(&pp->macros, name, NULL);
 }
 
 /* --- pp_directive.c → pp_expand.c --- */
 i64 eval_if_expr(Pp* pp, const Tok* line, u32 n, LocRef loc);
 void process_directive(Pp* pp, LocRef hash_loc);
 
 /* --- pp_directive.c internal helpers called from pp_expand.c --- */
 void emit_pragma_line(Pp* pp, const Tok* line, u32 n, LocRef loc);
 int peek_for_invoke_paren(Pp* pp, int* ws_has_space_out);
 int try_expand_pragma_op(Pp* pp, const Tok* invoke);
 
 /* --- pp_directive.c: read_directive_line (used by pp.c/pp_define) --- */
 void read_directive_line(Pp* pp, Tok** out_toks, u32* out_n);
 
 /* --- pp_expand.c: do_define / do_undef (used by pp.c/pp_define) --- */
 void do_define(Pp* pp, const Tok* line, u32 n);
 void do_undef(Pp* pp, const Tok* line, u32 n);
 
 /* --- pp_directive.c helpers needed by pp_expand.c (_Pragma) --- */
 TokSrc* current_lex_src(Pp* pp);
 
 #endif /* KIT_PP_PRIV_H */