lexer_test.c (13417B)
1 #include "features_lexer.h" 2 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 7 static int failures = 0; 8 9 typedef struct { 10 KitGramTokenKind kind; 11 const char *lexeme; 12 uint32_t line, col; 13 } WantTok; 14 15 static void fail(const char *msg) { 16 printf("FAIL %s\n", msg); 17 failures++; 18 } 19 20 static void check_status(const char *what, int ok) { 21 printf("%s %s\n", ok ? "ok " : "FAIL", what); 22 if (!ok) failures++; 23 } 24 25 static void expect_token(KitGramToken got, WantTok want, size_t idx) { 26 char label[160]; 27 int lex_ok = got.len == strlen(want.lexeme) && strncmp(got.lexeme, want.lexeme, got.len) == 0; 28 snprintf(label, sizeof label, "token[%zu] kind=%u lexeme=\"%.*s\" at %u:%u", 29 idx, (unsigned)got.kind, (int)got.len, got.lexeme, got.line, got.col); 30 check_status(label, got.kind == want.kind && lex_ok && got.line == want.line && got.col == want.col); 31 } 32 33 static void drain_expect(KitGramLexer *lx, const WantTok *want, size_t *idx, size_t nwant, int *saw_eof) { 34 for (;;) { 35 KitGramToken tok; 36 KitGramLexStatus st = kit_gram_lexer_next(lx, &tok); 37 if (st == KIT_GRAM_LEX_TOKEN) { 38 if (*idx >= nwant) { 39 fail("lexer emitted too many tokens"); 40 return; 41 } 42 expect_token(tok, want[*idx], *idx); 43 (*idx)++; 44 continue; 45 } 46 if (st == KIT_GRAM_LEX_NEED_MORE) return; 47 if (st == KIT_GRAM_LEX_EOF) { 48 *saw_eof = 1; 49 return; 50 } 51 fail("lexer returned unexpected error"); 52 return; 53 } 54 } 55 56 static void check_tokens_chunked(void) { 57 printf("== lexer tokens and streaming ==\n"); 58 59 const char *src = "while\n 12, for_each while1 for_each2;"; 60 WantTok want[] = { 61 { FEATURES_TOK_WHILE, "while", 1, 1 }, 62 { FEATURES_TOK_NUMBER, "12", 2, 3 }, 63 { FEATURES_TOK_COMMA, ",", 2, 5 }, 64 { FEATURES_TOK_FOR_EACH, "for_each", 2, 7 }, 65 { FEATURES_TOK_IDENT, "while1", 2, 16 }, 66 { FEATURES_TOK_IDENT, "for_each2", 2, 23 }, 67 { FEATURES_TOK_SEMI, ";", 2, 32 }, 68 }; 69 70 unsigned char carry[128]; 71 KitGramLexInput in; 72 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){ .carry = carry, .carry_cap = sizeof carry }); 73 KitGramLexer lx; 74 KitGramLexConfig cfg = {0}; 75 features_lexer_init(&lx, &in, &cfg); 76 77 size_t idx = 0; 78 int saw_eof = 0; 79 KitGramLexInputSpan spans[128]; 80 for (size_t i = 0; src[i]; i++) { 81 if (i >= sizeof spans / sizeof spans[0]) { 82 fail("test span array too small"); 83 return; 84 } 85 spans[i] = (KitGramLexInputSpan){ .bytes = (const unsigned char *)&src[i], .len = 1 }; 86 kit_gram_lex_input_push(&in, &spans[i]); 87 drain_expect(&lx, want, &idx, sizeof want / sizeof want[0], &saw_eof); 88 } 89 kit_gram_lex_input_finish(&in); 90 drain_expect(&lx, want, &idx, sizeof want / sizeof want[0], &saw_eof); 91 92 check_status("all expected tokens emitted", idx == sizeof want / sizeof want[0]); 93 check_status("EOF reported after finish", saw_eof); 94 } 95 96 static int parse_with_generated_lexer(const char *src, size_t chunk) { 97 unsigned char carry[64]; 98 KitGramLexInput in; 99 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){ .carry = carry, .carry_cap = sizeof carry }); 100 KitGramLexer lx; 101 KitGramLexConfig lcfg = {0}; 102 features_lexer_init(&lx, &in, &lcfg); 103 104 KitGramParser ps; 105 KitGramSlot ctl[128]; 106 KitGramSem vals[128]; 107 KitGramConfig pcfg = { .ctl_stack = ctl, .ctl_cap = 128, .val_stack = vals, .val_cap = 128 }; 108 features_parser_init(&ps, &pcfg); 109 110 size_t off = 0; 111 size_t nspans = 0; 112 KitGramLexInputSpan spans[256]; 113 int finished = 0; 114 for (;;) { 115 KitGramToken tok; 116 KitGramLexStatus st = kit_gram_lexer_next(&lx, &tok); 117 if (st == KIT_GRAM_LEX_TOKEN) { 118 if (kit_gram_parser_push(&ps, tok) == KIT_GRAM_PARSE_ERROR) return 0; 119 continue; 120 } 121 if (st == KIT_GRAM_LEX_ERROR) return 0; 122 if (st == KIT_GRAM_LEX_EOF) return kit_gram_parser_finish(&ps) == KIT_GRAM_PARSE_ACCEPT; 123 124 if (src[off]) { 125 size_t remain = strlen(src + off); 126 size_t take = remain < chunk ? remain : chunk; 127 if (nspans >= sizeof spans / sizeof spans[0]) return 0; 128 spans[nspans] = (KitGramLexInputSpan){ .bytes = (const unsigned char *)src + off, .len = take }; 129 kit_gram_lex_input_push(&in, &spans[nspans++]); 130 off += take; 131 } else if (!finished) { 132 kit_gram_lex_input_finish(&in); 133 finished = 1; 134 } else { 135 return 0; 136 } 137 } 138 } 139 140 static void check_parser_handoff(void) { 141 printf("\n== lexer to parser handoff ==\n"); 142 check_status("features accepts generated token stream", 143 parse_with_generated_lexer("for_each 1,2,3; a b c", 3)); 144 check_status("features rejects invalid grammar token stream", 145 !parse_with_generated_lexer("while 1,,; a", 2)); 146 } 147 148 static int tok_text_eq(KitGramToken tok, const char *s); 149 150 static void check_nested_lexer_composition(void) { 151 printf("\n== nested lexer composition ==\n"); 152 153 const char *src = "while \"abc\" while1"; 154 KitGramLexInput in; 155 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0}); 156 KitGramLexInputSpan span = { .bytes = (const unsigned char *)src, .len = strlen(src) }; 157 KitGramLexConfig cfg = {0}; 158 KitGramLexer main_lx; 159 features_lexer_init(&main_lx, &in, &cfg); 160 kit_gram_lex_input_push(&in, &span); 161 kit_gram_lex_input_finish(&in); 162 163 KitGramToken tok; 164 check_status("main lexer emits first token", 165 kit_gram_lexer_next(&main_lx, &tok) == KIT_GRAM_LEX_TOKEN && 166 tok.kind == FEATURES_TOK_WHILE && tok_text_eq(tok, "while")); 167 check_status("main lexer reaches mode boundary", 168 kit_gram_lexer_next(&main_lx, &tok) == KIT_GRAM_LEX_TOKEN && 169 tok.kind == FEATURES_TOK_DQUOTE && tok_text_eq(tok, "\"")); 170 171 KitGramLexer string_lx; 172 features_lexer_string_init(&string_lx, &in, &cfg); 173 check_status("string lexer starts at shared cursor", 174 kit_gram_lexer_next(&string_lx, &tok) == KIT_GRAM_LEX_TOKEN && 175 tok.kind == FEATURES_TOK_STR_TEXT && tok_text_eq(tok, "abc")); 176 check_status("string lexer consumes closing quote", 177 kit_gram_lexer_next(&string_lx, &tok) == KIT_GRAM_LEX_TOKEN && 178 tok.kind == FEATURES_TOK_STR_END && tok_text_eq(tok, "\"")); 179 check_status("main lexer resumes after nested lexer", 180 kit_gram_lexer_next(&main_lx, &tok) == KIT_GRAM_LEX_TOKEN && 181 tok.kind == FEATURES_TOK_IDENT && tok_text_eq(tok, "while1")); 182 check_status("composed input reaches EOF", kit_gram_lexer_next(&main_lx, &tok) == KIT_GRAM_LEX_EOF); 183 } 184 185 typedef struct { 186 int calls; 187 } HookUd; 188 189 typedef struct { 190 int calls; 191 const unsigned char *bytes[4]; 192 size_t len[4]; 193 } ReleaseUd; 194 195 static void release_cb(void *ud, const unsigned char *bytes, size_t len) { 196 ReleaseUd *r = ud; 197 if (r->calls < 4) { 198 r->bytes[r->calls] = bytes; 199 r->len[r->calls] = len; 200 } 201 r->calls++; 202 } 203 204 static int tok_text_eq(KitGramToken tok, const char *s) { 205 return tok.len == strlen(s) && strncmp(tok.lexeme, s, tok.len) == 0; 206 } 207 208 static KitGramLexHookResult feature_token_hook(KitGramLexer *lx, void *ud, KitGramToken *tok, KitGramLexError *err) { 209 (void)lx; 210 HookUd *hook = ud; 211 hook->calls++; 212 if (tok->kind == FEATURES_TOK_IDENT && tok_text_eq(*tok, "alias")) { 213 tok->kind = FEATURES_TOK_WHILE; 214 tok->lexeme = "while"; 215 tok->len = 5; 216 return KIT_GRAM_LEX_HOOK_KEEP; 217 } 218 if (tok->kind == FEATURES_TOK_NUMBER) { 219 return KIT_GRAM_LEX_HOOK_SKIP; 220 } 221 if (tok->kind == FEATURES_TOK_IDENT && tok_text_eq(*tok, "bad")) { 222 err->line = tok->line; 223 err->col = tok->col; 224 err->byte = 0; 225 err->message = "hook rejected token"; 226 return KIT_GRAM_LEX_HOOK_ERROR; 227 } 228 return KIT_GRAM_LEX_HOOK_KEEP; 229 } 230 231 static void check_token_hooks(void) { 232 printf("\n== lexer token hooks ==\n"); 233 234 { 235 HookUd hook = {0}; 236 KitGramLexInput in; 237 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0}); 238 KitGramLexer lx; 239 KitGramLexConfig cfg = { 240 .token_hook = feature_token_hook, 241 .hook_ud = &hook, 242 }; 243 features_lexer_init(&lx, &in, &cfg); 244 const char *src = "alias 123 name"; 245 KitGramLexInputSpan span = { .bytes = (const unsigned char *)src, .len = strlen(src) }; 246 kit_gram_lex_input_push(&in, &span); 247 kit_gram_lex_input_finish(&in); 248 249 KitGramToken tok; 250 check_status("hook rewrote first token", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_TOKEN && 251 tok.kind == FEATURES_TOK_WHILE && tok_text_eq(tok, "while") && 252 tok.line == 1 && tok.col == 1); 253 check_status("hook skipped number token", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_TOKEN && 254 tok.kind == FEATURES_TOK_IDENT && tok_text_eq(tok, "name") && 255 tok.line == 1 && tok.col == 11); 256 check_status("hooked lexer EOF", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_EOF); 257 check_status("hook saw non-skip tokens", hook.calls == 3); 258 } 259 260 { 261 HookUd hook = {0}; 262 KitGramLexInput in; 263 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0}); 264 KitGramLexer lx; 265 KitGramLexConfig cfg = { 266 .token_hook = feature_token_hook, 267 .hook_ud = &hook, 268 }; 269 features_lexer_init(&lx, &in, &cfg); 270 const char *src = "bad"; 271 KitGramLexInputSpan span = { .bytes = (const unsigned char *)src, .len = strlen(src) }; 272 kit_gram_lex_input_push(&in, &span); 273 kit_gram_lex_input_finish(&in); 274 275 KitGramToken tok; 276 check_status("hook can reject token", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_ERROR); 277 const KitGramLexError *e = kit_gram_lex_input_error(&in); 278 check_status("hook error location", e->line == 1 && e->col == 1); 279 check_status("hook error message", strcmp(e->message, "hook rejected token") == 0); 280 } 281 } 282 283 static void check_errors(void) { 284 printf("\n== lexer errors ==\n"); 285 286 { 287 KitGramLexInput in; 288 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0}); 289 KitGramLexer lx; 290 KitGramLexConfig cfg = {0}; 291 features_lexer_init(&lx, &in, &cfg); 292 const char *src = "ok @"; 293 KitGramLexInputSpan span = { .bytes = (const unsigned char *)src, .len = strlen(src) }; 294 kit_gram_lex_input_push(&in, &span); 295 296 KitGramToken tok; 297 check_status("first token before invalid byte", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_TOKEN); 298 check_status("invalid byte is reported", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_ERROR); 299 const KitGramLexError *e = kit_gram_lex_input_error(&in); 300 check_status("invalid byte location", e->line == 1 && e->col == 4 && e->byte == '@'); 301 check_status("invalid byte message", strcmp(e->message, "invalid token") == 0); 302 } 303 304 { 305 unsigned char carry[2]; 306 KitGramLexInput in; 307 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){ .carry = carry, .carry_cap = sizeof carry }); 308 KitGramLexer lx; 309 KitGramLexConfig cfg = {0}; 310 features_lexer_init(&lx, &in, &cfg); 311 static const unsigned char a[] = "ab"; 312 static const unsigned char b[] = "cd"; 313 ReleaseUd rel = {0}; 314 KitGramLexInputSpan spans[] = { 315 { .bytes = a, .len = 2, .on_consumed = release_cb, .ud = &rel }, 316 { .bytes = b, .len = 2, .on_consumed = release_cb, .ud = &rel }, 317 }; 318 kit_gram_lex_input_push(&in, &spans[0]); 319 kit_gram_lex_input_push(&in, &spans[1]); 320 kit_gram_lex_input_finish(&in); 321 322 KitGramToken tok; 323 check_status("large straddling token is segmented", 324 kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_TOKEN && 325 tok.kind == FEATURES_TOK_IDENT && tok.lexeme == NULL && tok.len == 4); 326 KitGramLexLexeme it; 327 kit_gram_lexer_lexeme(&lx, &it); 328 KitGramLexSegment seg; 329 check_status("first segment covers first span", 330 kit_gram_lex_lexeme_next(&it, &seg) && seg.bytes == a && seg.len == 2); 331 check_status("second segment covers second span", 332 kit_gram_lex_lexeme_next(&it, &seg) && seg.bytes == b && seg.len == 2); 333 check_status("segment iterator ends", !kit_gram_lex_lexeme_next(&it, &seg)); 334 check_status("segments retained until next lexer op", rel.calls == 0); 335 check_status("EOF commits segmented token", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_EOF); 336 check_status("release callbacks fired in order", 337 rel.calls == 2 && rel.bytes[0] == a && rel.len[0] == 2 && 338 rel.bytes[1] == b && rel.len[1] == 2); 339 } 340 } 341 342 int main(void) { 343 check_tokens_chunked(); 344 check_parser_handoff(); 345 check_nested_lexer_composition(); 346 check_token_hooks(); 347 check_errors(); 348 return failures ? 1 : 0; 349 }