lex.c (31115B)
1 /* C11 lexer (§6.4). Streams lean tokens out of a borrowed source buffer. 2 * 3 * Tokens are recognized per the standard's lexical grammar: 4 * - identifiers (§6.4.2) — keyword bucketing happens later in parse_c 5 * - pp-numbers (§6.4.8), classified into TOK_NUM / TOK_FLT 6 * - string literals (§6.4.5) and character constants (§6.4.4.4) 7 * including the L/u/u8/U encoding prefixes 8 * - punctuators (§6.4.6), longest-match, including digraphs 9 * - `#` and `##` (and their digraph forms `%:` and `%:%:`) become 10 * TOK_PP_HASH / TOK_PP_PASTE so PP can recognize directives and paste 11 * 12 * Comments (§6.4.9) are consumed as whitespace; physical newlines surface as 13 * TOK_NEWLINE so PP can implement directive-line semantics. 14 * 15 * Lean-token shape: each token carries a compact LocRef (file_id + byte offset 16 * into the folded buffer) and a TextRef (a source span for source tokens, an 17 * interned Sym for synthetic tokens, or nothing for canonical punctuators). The 18 * scanner does NOT track a running line number — line/col are reconstructed on 19 * demand from the source's retained buffer + splice table (see the PP SrcInfo 20 * registry). The hot path holds `cur`/`end` as pointers into the 21 * (line-splice-folded) buffer and classifies each byte with one load from 22 * `cclass[256]`; because phase-2 splices are folded out up front, the scan 23 * walks a clean buffer with a single code path (no per-byte splice test). */ 24 25 #include "lex/lex.h" 26 27 #include <string.h> 28 29 /* Per-byte character classes (§6.4). One table load replaces the branchy 30 * is_space/is_alpha/is_alnum/is_digit cascades and drives the first-char 31 * dispatch. Bytes ≥ 0x80 are accepted as identifier characters (the 32 * implementation-defined "other" source characters — UTF-8 lead/continuation 33 * bytes in practice). '\n' is deliberately NOT in CC_SPACE: it surfaces as 34 * TOK_NEWLINE, never skipped here. */ 35 #define CC_SPACE 0x01u /* ' ' '\t' '\r' '\v' '\f' (NOT '\n') */ 36 #define CC_DIGIT 0x02u /* '0'-'9' */ 37 #define CC_IDST 0x04u /* identifier-start: A-Z a-z _ and >= 0x80 */ 38 #define CC_IDCONT 0x08u /* identifier-continue: CC_IDST plus 0-9 */ 39 40 /* clang-format off */ 41 static const u8 cclass[256] = { 42 /*0x00*/ 0,0,0,0,0,0,0,0,0,CC_SPACE,0,CC_SPACE,CC_SPACE,CC_SPACE,0,0, 43 /*0x10*/ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 44 /*0x20*/ CC_SPACE,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 45 /*0x30*/ CC_DIGIT|CC_IDCONT,CC_DIGIT|CC_IDCONT,CC_DIGIT|CC_IDCONT,CC_DIGIT|CC_IDCONT, 46 CC_DIGIT|CC_IDCONT,CC_DIGIT|CC_IDCONT,CC_DIGIT|CC_IDCONT,CC_DIGIT|CC_IDCONT, 47 CC_DIGIT|CC_IDCONT,CC_DIGIT|CC_IDCONT,0,0,0,0,0,0, 48 /*0x40*/ 0,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 49 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 50 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 51 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 52 /*0x50*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 53 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 54 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,0,0,0,0, 55 CC_IDST|CC_IDCONT, 56 /*0x60*/ 0,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 57 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 58 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 59 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 60 /*0x70*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 61 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 62 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,0,0,0,0,0, 63 /*0x80*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 64 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 65 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 66 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 67 /*0x90*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 68 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 69 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 70 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 71 /*0xA0*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 72 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 73 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 74 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 75 /*0xB0*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 76 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 77 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 78 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 79 /*0xC0*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 80 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 81 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 82 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 83 /*0xD0*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 84 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 85 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 86 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 87 /*0xE0*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 88 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 89 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 90 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 91 /*0xF0*/ CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 92 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 93 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 94 CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT,CC_IDST|CC_IDCONT, 95 }; 96 /* clang-format on */ 97 98 /* Canonical spelling bytes for the single-char punctuators: punct1[c] == c for 99 * every punctuator codepoint, so a single-char Punct code reconstructs its 100 * spelling as the one byte at &punct1[code]. */ 101 /* clang-format off */ 102 static const char punct1[256] = { 103 ['!']='!', ['#']='#', ['%']='%', ['&']='&', ['(']='(', [')']=')', 104 ['*']='*', ['+']='+', [',']=',', ['-']='-', ['.']='.', ['/']='/', 105 [':']=':', [';']=';', ['<']='<', ['=']='=', ['>']='>', ['?']='?', 106 ['[']='[', [']']=']', ['^']='^', ['{']='{', ['|']='|', ['}']='}', 107 ['~']='~', 108 }; 109 /* clang-format on */ 110 111 const char* punct_canon(u32 punct, u32* len) { 112 if (punct < 256u) { 113 *len = 1u; 114 return &punct1[punct]; 115 } 116 switch (punct) { 117 case P_ARROW: 118 *len = 2; 119 return "->"; 120 case P_INC: 121 *len = 2; 122 return "++"; 123 case P_DEC: 124 *len = 2; 125 return "--"; 126 case P_SHL: 127 *len = 2; 128 return "<<"; 129 case P_SHR: 130 *len = 2; 131 return ">>"; 132 case P_LE: 133 *len = 2; 134 return "<="; 135 case P_GE: 136 *len = 2; 137 return ">="; 138 case P_EQ: 139 *len = 2; 140 return "=="; 141 case P_NE: 142 *len = 2; 143 return "!="; 144 case P_AND: 145 *len = 2; 146 return "&&"; 147 case P_OR: 148 *len = 2; 149 return "||"; 150 case P_ADD_ASSIGN: 151 *len = 2; 152 return "+="; 153 case P_SUB_ASSIGN: 154 *len = 2; 155 return "-="; 156 case P_MUL_ASSIGN: 157 *len = 2; 158 return "*="; 159 case P_DIV_ASSIGN: 160 *len = 2; 161 return "/="; 162 case P_MOD_ASSIGN: 163 *len = 2; 164 return "%="; 165 case P_AND_ASSIGN: 166 *len = 2; 167 return "&="; 168 case P_OR_ASSIGN: 169 *len = 2; 170 return "|="; 171 case P_XOR_ASSIGN: 172 *len = 2; 173 return "^="; 174 case P_SHL_ASSIGN: 175 *len = 3; 176 return "<<="; 177 case P_SHR_ASSIGN: 178 *len = 3; 179 return ">>="; 180 case P_ELLIPSIS: 181 *len = 3; 182 return "..."; 183 case P_HASH_HASH: 184 *len = 2; 185 return "##"; 186 default: 187 *len = 0; 188 return ""; 189 } 190 } 191 192 struct Lexer { 193 Compiler* c; 194 Heap* heap; 195 /* `src`/`len` are the post-phase-2 (line-splice-folded) logical bytes the 196 * scanner walks. When the input contains no `\<newline>` (the common case) 197 * this borrows the input verbatim (owns_src = 0, zero-copy); otherwise it is 198 * a folded heap copy. `cur` is the live cursor and `end` == src + len. */ 199 const char* src; 200 const char* end; 201 const char* cur; 202 u32 len; 203 u32 file_id; 204 u8 at_bol; 205 u8 had_space; 206 /* §5.1.1.2 phase 4 directive context for header-name lexing. 207 * 0 = none, 1 = saw pp-hash, 2 = saw `#include`/etc and the next 208 * token may be a header-name. */ 209 u8 dstate; 210 u8 owns_src; /* src is a heap-allocated folded copy this lexer must free */ 211 /* Newline-token policy. When set (default), every physical newline surfaces 212 * as a TOK_NEWLINE (the -E / cpp path needs them). When clear (cc parser 213 * feed), a NON-directive newline is consumed silently; the directive-line 214 * terminating newline is always emitted (Option B2). */ 215 u8 emit_newlines; 216 /* Set when a directive-introducing `#` (TOK_PP_HASH at BOL) was emitted on 217 * the current physical line; persists across the directive's tokens, cleared 218 * by the terminating newline. Forces that one newline to surface even when 219 * emit_newlines is off. */ 220 u8 directive_line; 221 /* Sorted logical offsets at which a `\<newline>` splice was folded out. The 222 * scanner never consults these (the folded buffer is splice-free) — they are 223 * retained metadata so the PP SrcInfo registry can reconstruct physical line 224 * numbers from a byte offset. NULL/empty for splice-free input. */ 225 u32* splices; 226 u32 nsplices; 227 /* Logical offset of the shebang line's terminating newline (the column origin 228 * of line 1 when a shebang was skipped); 0 when no shebang. */ 229 u32 shebang_off; 230 }; 231 232 /* §5.1.1.2 translation phase 2: fold every `\<newline>` line splice out of the 233 * input once, up front, so the scanner walks a clean buffer. The common 234 * (splice-free) input is borrowed verbatim; otherwise a heap copy holds the 235 * folded text and `splices[]` records each fold point (a logical offset) so the 236 * SrcInfo line-map can still account for the removed physical newline. */ 237 static void lex_fold_splices(Lexer* l, const char* src, u32 len) { 238 u32 i; 239 u32 nspl = 0; 240 char* buf; 241 u32* spl; 242 u32 w, s, wcap; 243 /* Count `\<newline>` splices via a memchr-driven sweep over backslashes. The 244 * `-1` keeps bs[1] in range so a trailing lone `\` is never counted. */ 245 if (len) { 246 const char* p = src; 247 const char* e = src + len; 248 while (p + 1 < e) { 249 const char* bs = (const char*)memchr(p, '\\', (size_t)(e - p) - 1); 250 if (!bs) break; 251 if (bs[1] == '\n') ++nspl; 252 p = bs + 1; 253 } 254 } 255 if (nspl == 0) { 256 l->src = src; 257 l->len = len; 258 l->owns_src = 0; 259 l->splices = NULL; 260 l->nsplices = 0; 261 return; 262 } 263 wcap = len - 2u * nspl; 264 spl = (u32*)l->heap->alloc(l->heap, nspl * sizeof(u32), _Alignof(u32)); 265 buf = wcap ? (char*)l->heap->alloc(l->heap, wcap, 1) : NULL; 266 if (!spl || (wcap && !buf)) { 267 if (buf) l->heap->free(l->heap, buf, wcap); 268 if (spl) l->heap->free(l->heap, spl, nspl * sizeof(u32)); 269 l->src = src; 270 l->len = len; 271 l->owns_src = 0; 272 l->splices = NULL; 273 l->nsplices = 0; 274 return; 275 } 276 w = 0; 277 s = 0; 278 for (i = 0; i < len;) { 279 if (i + 1 < len && src[i] == '\\' && src[i + 1] == '\n') { 280 spl[s++] = w; /* fold point = logical offset of the next byte */ 281 i += 2; 282 continue; 283 } 284 buf[w++] = src[i++]; 285 } 286 l->src = wcap ? buf : ""; 287 l->len = w; /* == wcap */ 288 l->owns_src = wcap ? 1u : 0u; 289 l->splices = spl; 290 l->nsplices = s; 291 } 292 293 /* The off-th logical byte at the cursor, or -1 at end of input. Cold lookahead 294 * only (encoding-prefix detection, punctuator longest-match, UCNs); the hot 295 * scan walks `cur`/`end` directly. */ 296 static int peek(const Lexer* l, size_t off) { 297 const char* p = l->cur + off; 298 return p < l->end ? (unsigned char)*p : -1; 299 } 300 301 static int is_digit(int c) { return c >= '0' && c <= '9'; } 302 static int is_hex_digit(int c) { 303 return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || 304 (c >= 'A' && c <= 'F'); 305 } 306 307 /* Consume a maximal run of identifier-continuation bytes (CC_IDCONT). */ 308 static void scan_ident_run(Lexer* l) { 309 const char* p = l->cur; 310 const char* end = l->end; 311 while (p < end && (cclass[(unsigned char)*p] & CC_IDCONT)) p++; 312 l->cur = p; 313 } 314 315 /* Match a UCN at offset `off`. Returns total length (6 for \uXXXX, 10 for 316 * \UXXXXXXXX), or 0 if no UCN matches. */ 317 static int ucn_len(const Lexer* l, size_t off) { 318 int n, i; 319 if (peek(l, off) != '\\') return 0; 320 if (peek(l, off + 1) == 'u') 321 n = 4; 322 else if (peek(l, off + 1) == 'U') 323 n = 8; 324 else 325 return 0; 326 for (i = 0; i < n; ++i) { 327 if (!is_hex_digit(peek(l, off + 2 + i))) return 0; 328 } 329 return 2 + n; 330 } 331 332 static LocRef loc_at(const Lexer* l, const char* p) { 333 LocRef loc; 334 loc.file_id = l->file_id; 335 loc.off = (u32)(p - l->src); 336 return loc; 337 } 338 339 /* A TEXT_SRC reference to [a,b) in this lexer's logical buffer. */ 340 static TextRef text_src(const Lexer* l, const char* a, const char* b) { 341 TextRef t; 342 t.kind = TEXT_SRC; 343 t.file_id = l->file_id; 344 t.off = (u32)(a - l->src); 345 t.len_or_sym = (u32)(b - a); 346 return t; 347 } 348 349 /* Reset the scanner cursor over (src, len). Shared by lex_open and lex_reset. 350 * Does NOT touch file_id (caller sets it). */ 351 static void lex_point_at(Lexer* l, const char* src, u32 len, int no_splices) { 352 if (no_splices) { 353 l->src = src ? src : ""; 354 l->len = src ? len : 0; 355 l->owns_src = 0; 356 l->splices = NULL; 357 l->nsplices = 0; 358 } else { 359 lex_fold_splices(l, src ? src : "", src ? len : 0); 360 } 361 l->cur = l->src; 362 l->end = l->src + l->len; 363 l->dstate = 0; 364 l->directive_line = 0; 365 l->at_bol = 1; 366 l->had_space = 0; 367 l->shebang_off = 0; 368 } 369 370 static u32 spec_file_id(Compiler* c, const SourceSpec* spec) { 371 u32 id = 0; 372 Sym name = spec->name_sym ? spec->name_sym : kit_sym_intern(c, spec->name); 373 (void)kit_source_add_memory_sym(c, name, &id); 374 return id; 375 } 376 377 Lexer* lex_open(Compiler* c, const SourceSpec* spec) { 378 Heap* h = (Heap*)kit_compiler_context(c)->heap; 379 Lexer* l = (Lexer*)h->alloc(h, sizeof(*l), _Alignof(Lexer)); 380 if (!l) return NULL; 381 l->c = c; 382 l->heap = h; 383 l->file_id = spec_file_id(c, spec); 384 l->emit_newlines = (spec->flags & SRC_PARSER_FEED) ? 0u : 1u; 385 lex_point_at(l, spec->bytes, spec->len, (spec->flags & SRC_NO_SPLICES) != 0); 386 return l; 387 } 388 389 void lex_reset(Lexer* l, const SourceSpec* spec) { 390 if (l->owns_src) l->heap->free(l->heap, (char*)l->src, l->len); 391 if (l->splices) l->heap->free(l->heap, l->splices, l->nsplices * sizeof(u32)); 392 l->owns_src = 0; 393 l->src = NULL; 394 l->splices = NULL; 395 l->nsplices = 0; 396 l->file_id = spec_file_id(l->c, spec); 397 l->emit_newlines = (spec->flags & SRC_PARSER_FEED) ? 0u : 1u; 398 lex_point_at(l, spec->bytes, spec->len, (spec->flags & SRC_NO_SPLICES) != 0); 399 } 400 401 void lex_close(Lexer* l) { 402 if (!l) return; 403 if (l->owns_src) l->heap->free(l->heap, (char*)l->src, l->len); 404 if (l->splices) l->heap->free(l->heap, l->splices, l->nsplices * sizeof(u32)); 405 l->heap->free(l->heap, l, sizeof(*l)); 406 } 407 408 /* Skip a leading `#!` shebang line. The shebang stays logical line 1; record 409 * the terminating newline's offset as the column origin so the materializer 410 * reports column 1 for it (matching the historical column-untracked skip). */ 411 void lex_skip_shebang(Lexer* l) { 412 if (!l || l->cur != l->src) return; 413 if (l->len < 2 || l->src[0] != '#' || l->src[1] != '!') return; 414 while (l->cur < l->end && *l->cur != '\n') l->cur++; 415 l->shebang_off = (u32)(l->cur - l->src); 416 } 417 418 LocRef lex_here(const Lexer* l) { return loc_at(l, l->cur); } 419 u32 lex_file_id(const Lexer* l) { return l->file_id; } 420 421 const char* lex_buf(const Lexer* l) { return l->src; } 422 u32 lex_buf_len(const Lexer* l) { return l->len; } 423 int lex_owns_buf(const Lexer* l) { return l->owns_src; } 424 u32 lex_shebang_off(const Lexer* l) { return l->shebang_off; } 425 const u32* lex_splices(const Lexer* l, u32* n_out) { 426 *n_out = l->nsplices; 427 return l->splices; 428 } 429 void lex_disown_buf(Lexer* l) { 430 l->owns_src = 0; 431 l->splices = NULL; /* PP now owns buffer + splice table */ 432 l->nsplices = 0; 433 } 434 435 KitSlice lex_text_slice(const Lexer* l, TextRef text) { 436 KitSlice s; 437 if (text.kind == TEXT_SRC) { 438 s.s = l->src + text.off; 439 s.len = text.len_or_sym; 440 return s; 441 } 442 if (text.kind == TEXT_SYM) return kit_sym_str(l->c, (Sym)text.len_or_sym); 443 s.s = ""; 444 s.len = 0; 445 return s; 446 } 447 448 /* §6.4.7 header-name lookahead: include-family keyword set, matched on the raw 449 * source bytes of the directive identifier following a `#`. */ 450 static int matches_include_kw(const char* s, size_t n) { 451 if (n == 7 && memcmp(s, "include", 7) == 0) return 1; 452 if (n == 12 && memcmp(s, "include_next", 12) == 0) return 1; 453 if (n == 6 && memcmp(s, "import", 6) == 0) return 1; 454 if (n == 5 && memcmp(s, "embed", 5) == 0) return 1; 455 return 0; 456 } 457 458 /* Fast whitespace + comment skip. The folded buffer is splice-free, so this is 459 * the single skip path. '\n' is never CC_SPACE (it surfaces as TOK_NEWLINE), so 460 * only a block comment can contain a newline here — and line tracking is lazy, 461 * so even those are just skipped. Sets had_space when anything is consumed. */ 462 static void skip_ws_and_comments(Lexer* l) { 463 const char* p = l->cur; 464 const char* end = l->end; 465 int adv = 0; 466 for (;;) { 467 const char* q = p; 468 while (p < end && (cclass[(unsigned char)*p] & CC_SPACE)) p++; 469 if (p != q) adv = 1; 470 if (p >= end) break; 471 if (*p == '/' && p + 1 < end && p[1] == '/') { 472 p += 2; 473 while (p < end && *p != '\n') p++; 474 adv = 1; 475 continue; 476 } 477 if (*p == '/' && p + 1 < end && p[1] == '*') { 478 p += 2; 479 while (p < end) { 480 if (*p == '*' && p + 1 < end && p[1] == '/') { 481 p += 2; 482 break; 483 } 484 p++; 485 } 486 adv = 1; 487 continue; 488 } 489 break; 490 } 491 l->cur = p; 492 if (adv) l->had_space = 1; 493 } 494 495 /* Consume a pp-number per §6.4.8; cursor at the leading digit (or `.`+digit). 496 */ 497 static void scan_pp_number(Lexer* l) { 498 const char* p = l->cur; 499 const char* end = l->end; 500 if (p < end && *p == '.') p++; 501 if (p < end) p++; /* first digit (dispatch guarantees it) */ 502 while (p < end) { 503 unsigned char c = (unsigned char)*p; 504 if ((c == 'e' || c == 'E' || c == 'p' || c == 'P') && p + 1 < end && 505 (p[1] == '+' || p[1] == '-')) { 506 p += 2; 507 } else if ((cclass[c] & CC_IDCONT) || c == '.') { 508 p++; 509 } else { 510 break; 511 } 512 } 513 l->cur = p; 514 } 515 516 /* 1 if the pp-number text is a floating constant (§6.4.4.2). */ 517 static int pp_number_is_float(const char* s, size_t n) { 518 int is_hex = 0; 519 size_t i = 0; 520 if (n >= 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) { 521 is_hex = 1; 522 i = 2; 523 } 524 for (; i < n; ++i) { 525 char c = s[i]; 526 if (c == '.') return 1; 527 if (is_hex && (c == 'p' || c == 'P')) return 1; 528 if (!is_hex && (c == 'e' || c == 'E')) { 529 if (i + 1 < n) { 530 char nx = s[i + 1]; 531 if (nx == '+' || nx == '-' || (nx >= '0' && nx <= '9')) return 1; 532 } 533 } 534 } 535 return 0; 536 } 537 538 /* Consume a quoted body — string ('"') or character ('\''); cursor at the open 539 * quote. Returns 1 on an unterminated/newline-broken literal. */ 540 static int scan_quoted(Lexer* l, int quote) { 541 const char* p = l->cur; 542 const char* end = l->end; 543 int bad = 0; 544 p++; /* opening quote */ 545 for (;;) { 546 if (p >= end) { 547 bad = 1; 548 break; 549 } 550 if (*p == quote) { 551 p++; 552 break; 553 } 554 if (*p == '\n') { 555 bad = 1; 556 break; 557 } 558 if (*p == '\\') { 559 p++; 560 if (p >= end) { 561 bad = 1; 562 break; 563 } 564 p++; /* the escaped char */ 565 continue; 566 } 567 p++; 568 } 569 l->cur = p; 570 return bad; 571 } 572 573 /* Finish a string/character literal: skip `sp_len` prefix bytes, consume the 574 * quoted body, classify, and record the full spelling span [tok_start, cur). */ 575 static void finish_str_lit(Lexer* l, Tok* t, const char* tok_start, int sp_len, 576 int is_char, u32 encf) { 577 l->cur += sp_len; 578 if (scan_quoted(l, is_char ? '\'' : '"')) t->flags |= TF_LITERAL_BAD; 579 t->kind = (u16)(is_char ? TOK_CHR : TOK_STR); 580 t->flags |= (u16)encf; 581 t->aux = 0; 582 t->text = text_src(l, tok_start, l->cur); 583 l->dstate = 0; 584 } 585 586 /* Finish an identifier token (§6.4.2). Cursor at the first identifier byte. */ 587 static void finish_ident(Lexer* l, Tok* t, const char* tok_start) { 588 int u = ucn_len(l, 0); 589 if (u) 590 l->cur += u; 591 else 592 l->cur += 1; 593 for (;;) { 594 scan_ident_run(l); 595 u = ucn_len(l, 0); 596 if (!u) break; 597 l->cur += u; 598 } 599 t->kind = TOK_IDENT; 600 t->aux = (u32)kit_sym_intern( 601 l->c, (KitSlice){.s = tok_start, .len = (size_t)(l->cur - tok_start)}); 602 t->text = text_src(l, tok_start, l->cur); 603 if (l->dstate == 1) { 604 l->dstate = 605 matches_include_kw(tok_start, (size_t)(l->cur - tok_start)) ? 2 : 0; 606 } else { 607 l->dstate = 0; 608 } 609 } 610 611 void lex_next(Lexer* l, Tok* out) { 612 Tok t; 613 const char* tok_start; 614 const char* end = l->end; 615 unsigned cc; 616 int ch; 617 618 t.flags = 0; 619 t.aux = 0; 620 621 /* Skip whitespace and comments; surface newlines per policy. */ 622 for (;;) { 623 skip_ws_and_comments(l); 624 if (l->cur >= end) { 625 t.kind = TOK_EOF; 626 t.loc = lex_here(l); 627 t.text = text_none_ref(); 628 *out = t; 629 return; 630 } 631 if (*l->cur == '\n') { 632 int emit_nl = l->emit_newlines || l->directive_line; 633 LocRef nloc = lex_here(l); 634 l->cur++; 635 l->at_bol = 1; 636 l->had_space = 0; 637 l->dstate = 0; 638 l->directive_line = 0; 639 if (!emit_nl) continue; 640 t.kind = TOK_NEWLINE; 641 t.loc = nloc; 642 t.text = text_none_ref(); 643 *out = t; 644 return; 645 } 646 break; 647 } 648 649 tok_start = l->cur; 650 ch = (unsigned char)*l->cur; 651 cc = cclass[(unsigned char)ch]; 652 653 if (l->at_bol) t.flags |= TF_AT_BOL; 654 if (l->had_space) t.flags |= TF_HAS_SPACE; 655 l->at_bol = 0; 656 l->had_space = 0; 657 t.loc = loc_at(l, tok_start); 658 659 /* §6.4.7 header-name in #include / #embed argument context. Takes precedence 660 * over the string-literal reading of a leading `"`. */ 661 if (l->dstate == 2 && (ch == '<' || ch == '"')) { 662 int closer = (ch == '<') ? '>' : '"'; 663 const char* p = l->cur + 1; 664 for (;;) { 665 if (p >= end || *p == '\n') { 666 t.flags |= TF_LITERAL_BAD; 667 break; 668 } 669 if (*p == closer) { 670 p++; 671 break; 672 } 673 p++; 674 } 675 l->cur = p; 676 t.kind = TOK_HEADER; 677 t.text = text_src(l, tok_start, l->cur); 678 l->dstate = 0; 679 *out = t; 680 return; 681 } 682 683 /* Identifier (§6.4.2), or an encoding-prefixed string/char literal. */ 684 if (cc & CC_IDST) { 685 if (ch == 'L') { 686 if (peek(l, 1) == '"') { 687 finish_str_lit(l, &t, tok_start, 1, 0, TF_STR_WIDE); 688 *out = t; 689 return; 690 } 691 if (peek(l, 1) == '\'') { 692 finish_str_lit(l, &t, tok_start, 1, 1, TF_STR_WIDE); 693 *out = t; 694 return; 695 } 696 } else if (ch == 'u') { 697 if (peek(l, 1) == '8' && peek(l, 2) == '"') { 698 finish_str_lit(l, &t, tok_start, 2, 0, TF_STR_U8); 699 *out = t; 700 return; 701 } 702 if (peek(l, 1) == '"') { 703 finish_str_lit(l, &t, tok_start, 1, 0, TF_STR_U16); 704 *out = t; 705 return; 706 } 707 if (peek(l, 1) == '\'') { 708 finish_str_lit(l, &t, tok_start, 1, 1, TF_STR_U16); 709 *out = t; 710 return; 711 } 712 } else if (ch == 'U') { 713 if (peek(l, 1) == '"') { 714 finish_str_lit(l, &t, tok_start, 1, 0, TF_STR_U32); 715 *out = t; 716 return; 717 } 718 if (peek(l, 1) == '\'') { 719 finish_str_lit(l, &t, tok_start, 1, 1, TF_STR_U32); 720 *out = t; 721 return; 722 } 723 } 724 finish_ident(l, &t, tok_start); 725 *out = t; 726 return; 727 } 728 729 /* Bare string / character literal (no encoding prefix). */ 730 if (ch == '"' || ch == '\'') { 731 finish_str_lit(l, &t, tok_start, 0, ch == '\'', 0); 732 *out = t; 733 return; 734 } 735 736 /* pp-number (§6.4.8), classified to TOK_NUM / TOK_FLT. */ 737 if ((cc & CC_DIGIT) || (ch == '.' && is_digit(peek(l, 1)))) { 738 const char* text; 739 size_t k; 740 scan_pp_number(l); 741 text = tok_start; 742 k = (size_t)(l->cur - tok_start); 743 t.kind = (u16)(pp_number_is_float(text, k) ? TOK_FLT : TOK_NUM); 744 if (t.kind == TOK_FLT) { 745 size_t j = k; 746 while (j > 0) { 747 char c = text[j - 1]; 748 if (c == 'f' || c == 'F') { 749 t.flags |= TF_FLT_F; 750 --j; 751 continue; 752 } 753 if (c == 'l' || c == 'L') { 754 t.flags |= TF_FLT_L; 755 --j; 756 continue; 757 } 758 break; 759 } 760 } else { 761 size_t j = k; 762 while (j > 0) { 763 char c = text[j - 1]; 764 if (c == 'u' || c == 'U') { 765 t.flags |= TF_INT_U; 766 --j; 767 continue; 768 } 769 if (c == 'l' || c == 'L') { 770 if (j >= 2 && (text[j - 2] == 'l' || text[j - 2] == 'L')) { 771 t.flags |= TF_INT_LL; 772 j -= 2; 773 } else { 774 t.flags |= TF_INT_L; 775 --j; 776 } 777 continue; 778 } 779 break; 780 } 781 } 782 t.aux = 0; 783 t.text = text_src(l, tok_start, l->cur); 784 l->dstate = 0; 785 *out = t; 786 return; 787 } 788 789 /* Identifier introduced by a UCN (§6.4.3). */ 790 if (ch == '\\' && ucn_len(l, 0)) { 791 finish_ident(l, &t, tok_start); 792 *out = t; 793 return; 794 } 795 796 /* Punctuator (§6.4.6) — longest match. `#`/`##` (and digraphs `%:`/`%:%:`) 797 * become TOK_PP_HASH / TOK_PP_PASTE. */ 798 { 799 int n0 = ch; 800 int n1 = peek(l, 1); 801 int n2 = peek(l, 2); 802 int n3 = peek(l, 3); 803 int adv = 1; 804 u32 punct = P_NONE; 805 u16 kind = TOK_PUNCT; 806 /* A digraph spells differently from its canonical punct, so it keeps an 807 * exact source span; canonical puncts reconstruct from the code. */ 808 int digraph = 0; 809 810 switch (n0) { 811 case '#': 812 if (n1 == '#') { 813 adv = 2; 814 kind = TOK_PP_PASTE; 815 punct = P_HASH_HASH; 816 } else { 817 adv = 1; 818 kind = TOK_PP_HASH; 819 punct = '#'; 820 } 821 break; 822 case '.': 823 if (n1 == '.' && n2 == '.') { 824 adv = 3; 825 punct = P_ELLIPSIS; 826 } else { 827 adv = 1; 828 punct = '.'; 829 } 830 break; 831 case '-': 832 if (n1 == '>') { 833 adv = 2; 834 punct = P_ARROW; 835 } else if (n1 == '-') { 836 adv = 2; 837 punct = P_DEC; 838 } else if (n1 == '=') { 839 adv = 2; 840 punct = P_SUB_ASSIGN; 841 } else { 842 adv = 1; 843 punct = '-'; 844 } 845 break; 846 case '+': 847 if (n1 == '+') { 848 adv = 2; 849 punct = P_INC; 850 } else if (n1 == '=') { 851 adv = 2; 852 punct = P_ADD_ASSIGN; 853 } else { 854 adv = 1; 855 punct = '+'; 856 } 857 break; 858 case '<': 859 if (n1 == '<' && n2 == '=') { 860 adv = 3; 861 punct = P_SHL_ASSIGN; 862 } else if (n1 == '<') { 863 adv = 2; 864 punct = P_SHL; 865 } else if (n1 == '=') { 866 adv = 2; 867 punct = P_LE; 868 } else if (n1 == ':') { 869 adv = 2; 870 punct = '['; 871 digraph = 1; 872 } else if (n1 == '%') { 873 adv = 2; 874 punct = '{'; 875 digraph = 1; 876 } else { 877 adv = 1; 878 punct = '<'; 879 } 880 break; 881 case '>': 882 if (n1 == '>' && n2 == '=') { 883 adv = 3; 884 punct = P_SHR_ASSIGN; 885 } else if (n1 == '>') { 886 adv = 2; 887 punct = P_SHR; 888 } else if (n1 == '=') { 889 adv = 2; 890 punct = P_GE; 891 } else { 892 adv = 1; 893 punct = '>'; 894 } 895 break; 896 case '=': 897 if (n1 == '=') { 898 adv = 2; 899 punct = P_EQ; 900 } else { 901 adv = 1; 902 punct = '='; 903 } 904 break; 905 case '!': 906 if (n1 == '=') { 907 adv = 2; 908 punct = P_NE; 909 } else { 910 adv = 1; 911 punct = '!'; 912 } 913 break; 914 case '&': 915 if (n1 == '&') { 916 adv = 2; 917 punct = P_AND; 918 } else if (n1 == '=') { 919 adv = 2; 920 punct = P_AND_ASSIGN; 921 } else { 922 adv = 1; 923 punct = '&'; 924 } 925 break; 926 case '|': 927 if (n1 == '|') { 928 adv = 2; 929 punct = P_OR; 930 } else if (n1 == '=') { 931 adv = 2; 932 punct = P_OR_ASSIGN; 933 } else { 934 adv = 1; 935 punct = '|'; 936 } 937 break; 938 case '^': 939 if (n1 == '=') { 940 adv = 2; 941 punct = P_XOR_ASSIGN; 942 } else { 943 adv = 1; 944 punct = '^'; 945 } 946 break; 947 case '*': 948 if (n1 == '=') { 949 adv = 2; 950 punct = P_MUL_ASSIGN; 951 } else { 952 adv = 1; 953 punct = '*'; 954 } 955 break; 956 case '/': 957 if (n1 == '=') { 958 adv = 2; 959 punct = P_DIV_ASSIGN; 960 } else { 961 adv = 1; 962 punct = '/'; 963 } 964 break; 965 case '%': 966 if (n1 == ':' && n2 == '%' && n3 == ':') { 967 adv = 4; 968 kind = TOK_PP_PASTE; 969 punct = P_HASH_HASH; 970 digraph = 1; 971 } else if (n1 == ':') { 972 adv = 2; 973 kind = TOK_PP_HASH; 974 punct = '#'; 975 digraph = 1; 976 } else if (n1 == '=') { 977 adv = 2; 978 punct = P_MOD_ASSIGN; 979 } else if (n1 == '>') { 980 adv = 2; 981 punct = '}'; 982 digraph = 1; 983 } else { 984 adv = 1; 985 punct = '%'; 986 } 987 break; 988 case ':': 989 if (n1 == '>') { 990 adv = 2; 991 punct = ']'; 992 digraph = 1; 993 } else { 994 adv = 1; 995 punct = ':'; 996 } 997 break; 998 case '(': 999 case ')': 1000 case '{': 1001 case '}': 1002 case '[': 1003 case ']': 1004 case ',': 1005 case ';': 1006 case '?': 1007 case '~': 1008 adv = 1; 1009 punct = (u32)n0; 1010 break; 1011 default: 1012 /* Unknown byte: surface as a single-char punct so the stream still 1013 * progresses; the digraph path keeps its exact source span. */ 1014 adv = 1; 1015 punct = (u32)n0; 1016 digraph = 1; 1017 break; 1018 } 1019 1020 l->cur += adv; 1021 t.kind = kind; 1022 t.aux = punct; 1023 t.text = digraph ? text_src(l, tok_start, l->cur) : text_none_ref(); 1024 if (kind == TOK_PP_HASH) { 1025 l->dstate = 1; 1026 if (t.flags & TF_AT_BOL) l->directive_line = 1; 1027 } else { 1028 l->dstate = 0; 1029 } 1030 *out = t; 1031 return; 1032 } 1033 }