lex.h (9199B)
1 #ifndef KIT_LEX_H 2 #define KIT_LEX_H 3 4 #include "cpp_support.h" 5 6 /* C11 lexer boundary (§6.4) — the "lean token" contract. 7 * 8 * The scanner streams tokens out of a borrowed, line-splice-folded source 9 * buffer into a caller-provided Tok slot. Unlike the historical token, the lean 10 * token defers the two expensive per-token operations: 11 * 12 * - SOURCE LOCATION is a compact (file_id, byte_off) reference, not an eager 13 * (file_id, line, col) triple. line/col are recovered on demand from the 14 * owning source's retained buffer + splice table (see TextRef / the PP 15 * SrcInfo registry). The scanner no longer tracks a running line number. 16 * 17 * - EXACT SPELLING is a text reference, not an eagerly interned Sym. Source 18 * tokens name a byte span in the post-splice buffer; synthetic tokens carry 19 * an interned Sym; canonical punctuators carry neither and reconstruct 20 * their spelling from the punctuator code. Identifiers are still interned 21 * eagerly (their Sym is the macro/keyword lookup key) and that Sym lives in 22 * `aux`. 23 */ 24 25 typedef enum TokKind { 26 TOK_EOF = 0, 27 TOK_IDENT, /* aux = interned Sym; text = source span */ 28 TOK_NUM, /* text = spelling; flags carry TF_INT_* */ 29 TOK_FLT, /* text = spelling; flags carry TF_FLT_* */ 30 TOK_STR, /* text = spelling; flags carry TF_STR_* / TF_LITERAL_BAD */ 31 TOK_CHR, /* text = spelling; flags carry TF_STR_* / TF_LITERAL_BAD */ 32 TOK_PUNCT, /* aux = Punct code; text = NONE (canonical) or span (digraph) */ 33 TOK_PP_HASH, /* # — aux = '#' */ 34 TOK_PP_PASTE, /* ## — aux = P_HASH_HASH */ 35 TOK_HEADER, /* header-name in #include / #embed; text = source span */ 36 TOK_NEWLINE, /* visible to PP only */ 37 TOK_KW_FIRST, 38 /* Historical keyword range, unused by the parser (keyword classification is 39 * Sym-based). Kept for value stability. */ 40 TOK_KW_LAST = 0x1000, 41 } TokKind; 42 43 typedef enum TokFlag { 44 TF_AT_BOL = 1u << 0, 45 TF_HAS_SPACE = 1u << 1, 46 TF_NO_EXPAND = 1u << 2, 47 TF_INT_U = 1u << 3, 48 TF_INT_L = 1u << 4, 49 TF_INT_LL = 1u << 5, 50 TF_FLT_F = 1u << 6, 51 TF_FLT_L = 1u << 7, 52 TF_STR_WIDE = 1u << 8, 53 TF_STR_U8 = 1u << 9, 54 TF_STR_U16 = 1u << 10, 55 TF_STR_U32 = 1u << 11, 56 TF_LITERAL_BAD = 1u << 12, 57 } TokFlag; 58 59 typedef enum Punct { 60 P_NONE = 0, 61 /* Single-char punctuators reuse their ASCII codepoint here. */ 62 P_ARROW = 256, 63 P_INC, 64 P_DEC, 65 P_SHL, 66 P_SHR, 67 P_LE, 68 P_GE, 69 P_EQ, 70 P_NE, 71 P_AND, 72 P_OR, 73 P_ADD_ASSIGN, 74 P_SUB_ASSIGN, 75 P_MUL_ASSIGN, 76 P_DIV_ASSIGN, 77 P_MOD_ASSIGN, 78 P_AND_ASSIGN, 79 P_OR_ASSIGN, 80 P_XOR_ASSIGN, 81 P_SHL_ASSIGN, 82 P_SHR_ASSIGN, 83 P_ELLIPSIS, 84 P_HASH_HASH, 85 } Punct; 86 87 /* ============================================================ 88 * Location reference 89 * ============================================================ */ 90 91 /* A compact source location: a byte offset into the owning source's logical 92 * (post-line-splice-fold) buffer. line/col are materialized on demand via 93 * pp_materialize_loc / lex_materialize_loc, which consult the source's retained 94 * buffer + splice table and the active #line overlay. file_id == 0 is the 95 * "no location" sentinel (materializes to {0,0,0}). */ 96 typedef struct LocRef { 97 u32 file_id; 98 u32 off; 99 } LocRef; 100 101 /* ============================================================ 102 * Text reference 103 * ============================================================ */ 104 105 typedef enum TextKind { 106 TEXT_NONE = 0, /* no spelling (EOF/NEWLINE/placemarker); a canonical 107 * punctuator reconstructs its spelling from `aux` instead */ 108 TEXT_SRC = 1, /* a byte span in source `file_id`'s logical buffer */ 109 TEXT_SYM = 2, /* an interned Sym (synthetic / predefined tokens) */ 110 } TextKind; 111 112 /* Exact spelling reference. For TEXT_SRC, (file_id, off, len) names a span in 113 * the retained post-splice buffer of that source — valid until pp_free, because 114 * the PP SrcInfo registry retains every source buffer. For TEXT_SYM, `sym` 115 * holds the interned spelling. The two share storage in a small flat struct so 116 * the lean token stays trivially copyable. */ 117 typedef struct TextRef { 118 u32 kind; /* TextKind */ 119 u32 file_id; 120 u32 off; 121 u32 len_or_sym; /* TEXT_SRC: byte length; TEXT_SYM: Sym */ 122 } TextRef; 123 124 /* ============================================================ 125 * Token 126 * ============================================================ */ 127 128 typedef struct Tok { 129 u16 kind; /* TokKind (+ PP-internal kinds from pp_priv.h) */ 130 u16 flags; /* TokFlag bits */ 131 u32 aux; /* IDENT: Sym; PUNCT/PP_HASH/PP_PASTE: Punct code; 132 * PP_PARAM: parameter index; otherwise unused (0) */ 133 LocRef loc; 134 TextRef text; 135 } Tok; 136 137 _Static_assert(sizeof(struct Tok) == 32, 138 "lean token-stream cell stays the compact 32-byte storage unit"); 139 140 /* Canonical name for the shared front-half cell. 141 * `Tok` stays the primary spelling across lexer/PP/parser; `CFeCell` is the 142 * alias newer ring/shared code uses. They are the same type. */ 143 typedef struct Tok CFeCell; 144 145 /* Lean accessors. The parser/PP read identifier identity and punctuator code 146 * straight off `aux` — never a symbol-table probe for punctuators. */ 147 static inline Sym tok_ident(const Tok* t) { return (Sym)t->aux; } 148 static inline u32 tok_punct(const Tok* t) { return t->aux; } 149 150 /* Lean-token reference constructors (shared by lexer, PP, and parser). */ 151 static inline TextRef text_none_ref(void) { 152 TextRef t; 153 t.kind = TEXT_NONE; 154 t.file_id = 0; 155 t.off = 0; 156 t.len_or_sym = 0; 157 return t; 158 } 159 static inline TextRef text_sym_ref(Sym s) { 160 TextRef t; 161 t.kind = TEXT_SYM; 162 t.file_id = 0; 163 t.off = 0; 164 t.len_or_sym = (u32)s; 165 return t; 166 } 167 static inline LocRef locref_none(void) { 168 LocRef l; 169 l.file_id = 0; 170 l.off = 0; 171 return l; 172 } 173 174 /* Canonical spelling for a non-digraph punctuator code (incl. '#'/P_HASH_HASH). 175 * Returns a static NUL-terminated string and writes its length to *len. Used to 176 * reconstruct punctuator text for -E / stringize / paste / diagnostics without 177 * a per-lexer spelling cache. Returns "" (len 0) for an unknown code. */ 178 const char* punct_canon(u32 punct, u32* len); 179 180 /* ============================================================ 181 * Source spec 182 * ============================================================ */ 183 184 typedef enum SourceFlag { 185 SRC_PRIMARY = 1u << 0, /* primary translation unit (allows shebang skip) */ 186 SRC_SYSTEM = 1u << 1, /* system header (diagnostics/deps property) */ 187 SRC_PARSER_FEED = 1u << 2, /* suppress non-directive newline tokens */ 188 SRC_NO_SPLICES = 1u << 3, /* caller proves no backslash-newline splice */ 189 } SourceFlag; 190 191 typedef struct SourceSpec { 192 KitSlice name; /* source name; interned unless name_sym != 0 */ 193 Sym name_sym; /* pre-interned name (0 = intern `name`) */ 194 const char* bytes; 195 u32 len; 196 u32 flags; /* SourceFlag bits */ 197 } SourceSpec; 198 199 /* ============================================================ 200 * Lexer 201 * ============================================================ */ 202 203 typedef struct Lexer Lexer; 204 205 /* Open a memory lexer over the borrowed (spec->bytes, spec->len). Registers a 206 * fresh sequential compiler file_id for the source. Phase-2 line splices are 207 * folded up front: with splices the lexer builds a folded copy (lex_owns_buf 208 * true) and records each fold offset; without, it borrows the bytes verbatim. 209 * SRC_NO_SPLICES skips the fold scan (caller-guaranteed splice-free buffer). 210 * 211 * Ownership: a Lexer pushed onto PP is owned by PP (closed at EOF-pop or 212 * pp_free). The borrowed bytes — and, once PP adopts it, the folded buffer — 213 * must outlive pp_free. */ 214 Lexer* lex_open(Compiler*, const SourceSpec* spec); 215 /* Re-point an existing lexer at a fresh buffer, reusing the Lexer allocation. 216 * Registers a fresh file_id. Used by the token-paste re-lex (splice-free). */ 217 void lex_reset(Lexer*, const SourceSpec* spec); 218 void lex_close(Lexer*); 219 220 /* Skip a leading `#!` shebang line on a freshly-opened primary lexer (no-op 221 * otherwise). Must be called before the first token is pulled. */ 222 void lex_skip_shebang(Lexer*); 223 224 /* Streaming. Fills the caller's slot; returns TOK_EOF repeatedly at end. */ 225 void lex_next(Lexer*, Tok* out); 226 227 /* The location of the current cursor (used by PP for directive/diagnostic 228 * positions). */ 229 LocRef lex_here(const Lexer*); 230 u32 lex_file_id(const Lexer*); 231 232 /* Buffer adoption — the PP SrcInfo registry takes over the lexer's logical 233 * buffer + splice metadata so line/col and TEXT_SRC spellings remain 234 * materializable after the lexer is popped. After lex_disown_buf the lexer no 235 * longer frees the buffer at close. */ 236 const char* lex_buf(const Lexer*); 237 u32 lex_buf_len(const Lexer*); 238 int lex_owns_buf(const Lexer*); 239 const u32* lex_splices(const Lexer*, u32* nsplices_out); 240 u32 lex_shebang_off( 241 const Lexer*); /* logical offset of the shebang newline, or 0 */ 242 void lex_disown_buf(Lexer*); 243 244 /* Resolve a TEXT_SRC reference into this lexer's own live buffer (used by the 245 * paste/_Pragma re-lex, before the result is interned). */ 246 KitSlice lex_text_slice(const Lexer*, TextRef text); 247 248 #endif