lexer_standalone_test.c (21267B)
1 /* test_lexer_standalone.c — equivalence of the --lexer-standalone table-free 2 * tokenizer + match API with the table-driven runtime. 3 * 4 * For each grammar we hold the table lexer (compiled in memory: kit_gram_lexer_next 5 * over the dense DFA + extracted-keyword MPH) and the generated standalone 6 * surface (<prefix>lex_next: a self-contained scanner with keywords folded into 7 * the DFA, the accepted kind/skip baked in, and edge-anchor end context decided 8 * inline; no KitGramLexGrammar, no runtime-lexer link dependency). Both are the same 9 * grammar, so the resident token stream — kind, len, line, col — and terminal 10 * status (EOF vs ERROR) must be byte-identical. 11 * 12 * Covers: byte mode (json), %keywords via the default MPH and --fold-keywords 13 * (clike / clikesaf), UTF-8 mode + multi-byte keywords (kwutf8), start anchors 14 * \A/^ (anchors_start), end anchors \z/$ (anchors), UTF-8 scalar position over 15 * non-ASCII line breaks and CRLF (pos), keyword priority pinning (if/iffy/ifx), 16 * and the match API vs kit_gram_match_* (match). 17 * 18 * Standalone scanners take a uniform lex struct (identical layout for every 19 * grammar), so we forward-declare it once and bind the per-prefix entry points 20 * without pulling in the conflicting per-grammar TOK_* headers. The grammars are 21 * also compiled in memory for the table reference and the token-kind lookups. */ 22 #define _POSIX_C_SOURCE 200809L 23 24 #include <kit/gram.h> 25 #include <kit/gram_lex.h> 26 27 #include <stdio.h> 28 #include <stdlib.h> 29 #include <string.h> 30 31 #include "gram_test.h" 32 33 /* Every standalone grammar emits these exact struct layouts (lexer + find 34 * iterator). C linkage ignores the pointer types, so a local tag matching the 35 * generated layout is enough to call the externs. */ 36 typedef struct { const unsigned char *cur, *end; uint32_t line, col; bool after_cr; } sa_lex; 37 typedef struct { const unsigned char *buf; size_t len, from; int multiline; } sa_match_iter; 38 #define DECL_LEX(p) extern void p##_lex_init(sa_lex *, const unsigned char *, size_t); \ 39 extern KitGramLexStatus p##_lex_next(sa_lex *, KitGramToken *); 40 #define DECL_MATCH(p) \ 41 extern bool p##_match_anchored (const unsigned char *, size_t, const KitGramMatchOpts *, KitGramMatch *); \ 42 extern bool p##_match_full (const unsigned char *, size_t, const KitGramMatchOpts *, KitGramMatch *); \ 43 extern bool p##_match_find (const unsigned char *, size_t, const KitGramMatchOpts *, KitGramMatch *); \ 44 extern void p##_match_iter_init(sa_match_iter *, const unsigned char *, size_t, const KitGramMatchOpts *); \ 45 extern bool p##_match_iter_next(sa_match_iter *, KitGramMatch *); 46 DECL_LEX(jsonsa) DECL_LEX(clikesa) DECL_LEX(clikesaf) DECL_LEX(kwutf8sa) DECL_LEX(anchstartsa) DECL_LEX(anchsa) DECL_LEX(possa) 47 DECL_MATCH(matchsa) 48 49 /* Standalone + --parser-codegen variants: the fused parser drives the table-free 50 * scanner (rd_refill_lex). We check parse_fused(buf) against parse_rd fed by the 51 * standalone tokenizer's own token array — both run the same RD rule functions, 52 * so agreement isolates the fused refill, and the tokenizer is already pinned to 53 * the table lexer above. */ 54 #define DECL_FUSED(p) DECL_LEX(p) \ 55 extern KitGramSem p##_parse_rd(const KitGramToken *, size_t, const KitGramActions *, void *, KitGramError *, int *); \ 56 extern KitGramSem p##_parse_fused(const unsigned char *, size_t, const KitGramActions *, void *, KitGramError *, int *); 57 DECL_FUSED(jsonsr) DECL_FUSED(clikesr) 58 59 static int failures = 0; 60 static int checks = 0; 61 62 static void *xmalloc(size_t n) { 63 void *p = malloc(n ? n : 1); 64 if (!p) { fputs("oom\n", stderr); exit(2); } 65 return p; 66 } 67 68 static char *read_file(const char *path, size_t *len_out) { 69 FILE *f = fopen(path, "rb"); 70 if (!f) { fprintf(stderr, "cannot open %s\n", path); exit(2); } 71 fseek(f, 0, SEEK_END); 72 long n = ftell(f); 73 rewind(f); 74 char *buf = xmalloc((size_t)n + 1); 75 if (fread(buf, 1, (size_t)n, f) != (size_t)n) { fprintf(stderr, "read %s\n", path); exit(2); } 76 fclose(f); 77 buf[n] = '\0'; 78 *len_out = (size_t)n; 79 return buf; 80 } 81 82 /* Compile a grammar in memory; the returned object owns its tables (table-path 83 * lexer + token-kind lookups), so keep it alive for the program's life. */ 84 static KitGramCompiled *compile(const char *path, int multiline) { 85 size_t len = 0; 86 char *text = read_file(path, &len); 87 KitGramOptions opts = { .multiline = multiline != 0 }; 88 KitGramCompiled *c = NULL; 89 KitStatus st = kit_gram_compile_text(gram_test_ctx(), (KitSlice){.s=text,.len=len}, 90 kit_slice_cstr(path), &opts, &c); 91 free(text); 92 if (st != KIT_OK || !c) { fprintf(stderr, "compile %s failed (diagnostic on stderr)\n", path); exit(1); } 93 return c; 94 } 95 96 typedef struct { KitGramTokenKind kind; size_t len; uint32_t line, col; } Tok; 97 98 /* Tokenize a resident single-span buffer with the table lexer. */ 99 static size_t tbl_tokenize(const KitGramLexGrammar *g, const unsigned char *buf, size_t len, 100 Tok *out, size_t cap, KitGramLexStatus *status) { 101 KitGramLexInput in; 102 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0}); 103 KitGramLexer lx; 104 kit_gram_lexer_init(&lx, g, &in, &(KitGramLexConfig){0}); 105 KitGramLexInputSpan span = { .bytes = buf, .len = len }; 106 kit_gram_lex_input_push(&in, &span); 107 kit_gram_lex_input_finish(&in); 108 size_t n = 0; 109 KitGramLexStatus st; 110 for (;;) { 111 KitGramToken tok; 112 st = kit_gram_lexer_next(&lx, &tok); 113 if (st != KIT_GRAM_LEX_TOKEN) break; 114 if (n < cap) out[n] = (Tok){ tok.kind, tok.len, tok.line, tok.col }; 115 n++; 116 } 117 *status = st; 118 return n; 119 } 120 121 typedef void (*sa_init_fn)(sa_lex *, const unsigned char *, size_t); 122 typedef KitGramLexStatus (*sa_next_fn)(sa_lex *, KitGramToken *); 123 124 static size_t sa_tokenize(sa_init_fn init, sa_next_fn next, const unsigned char *buf, size_t len, 125 Tok *out, size_t cap, KitGramLexStatus *status) { 126 sa_lex lx; 127 init(&lx, buf, len); 128 size_t n = 0; 129 KitGramLexStatus st; 130 for (;;) { 131 KitGramToken tok; 132 st = next(&lx, &tok); 133 if (st != KIT_GRAM_LEX_TOKEN) break; 134 if (n < cap) out[n] = (Tok){ tok.kind, tok.len, tok.line, tok.col }; 135 n++; 136 } 137 *status = st; 138 return n; 139 } 140 141 /* Assert the standalone token stream equals the table token stream. */ 142 static void check_equiv(const char *name, const KitGramLexGrammar *tbl, 143 sa_init_fn init, sa_next_fn next, const char *input, size_t len) { 144 size_t cap = len + 16; 145 Tok *a = xmalloc(cap * sizeof *a); 146 Tok *b = xmalloc(cap * sizeof *b); 147 KitGramLexStatus sa, sb; 148 size_t na = tbl_tokenize(tbl, (const unsigned char *)input, len, a, cap, &sa); 149 size_t nb = sa_tokenize(init, next, (const unsigned char *)input, len, b, cap, &sb); 150 checks++; 151 int ok = (na == nb) && (sa == sb); 152 for (size_t i = 0; ok && i < na && i < cap; i++) 153 ok = a[i].kind == b[i].kind && a[i].len == b[i].len && 154 a[i].line == b[i].line && a[i].col == b[i].col; 155 if (!ok) { 156 failures++; 157 fprintf(stderr, "FAIL %s: table n=%zu st=%d, standalone n=%zu st=%d\n", 158 name, na, (int)sa, nb, (int)sb); 159 for (size_t i = 0; i < na && i < nb && i < cap; i++) 160 if (memcmp(&a[i], &b[i], sizeof(Tok)) != 0) { 161 fprintf(stderr, " tok %zu: table(k=%u l=%zu @%u:%u) vs standalone(k=%u l=%zu @%u:%u)\n", 162 i, a[i].kind, a[i].len, a[i].line, a[i].col, 163 b[i].kind, b[i].len, b[i].line, b[i].col); 164 break; 165 } 166 } else { 167 printf("ok %s: %zu tokens, status=%d\n", name, na, (int)sa); 168 } 169 free(a); 170 free(b); 171 } 172 173 /* ---- keyword folding priority: if -> IF, iffy/ifx -> IDENT --------------- */ 174 static void check_keyword_pins(KitGramCompiled *clike) { 175 KitGramTokenKind kif = 0, kident = 0; 176 if (!kit_gram_find_token(clike, "IF", &kif) || !kit_gram_find_token(clike, "IDENT", &kident)) { 177 fprintf(stderr, "FAIL keyword-pins: clike missing IF/IDENT tokens\n"); 178 failures++; 179 return; 180 } 181 struct { const char *s; KitGramTokenKind want; } cases[] = { 182 { "if", kif }, { "iffy", kident }, { "ifx", kident }, 183 { "i", kident }, { "if2", kident }, 184 }; 185 for (size_t i = 0; i < sizeof cases / sizeof cases[0]; i++) { 186 const char *s = cases[i].s; 187 sa_lex lx; 188 clikesa_lex_init(&lx, (const unsigned char *)s, strlen(s)); 189 KitGramToken tok; 190 KitGramLexStatus st = clikesa_lex_next(&lx, &tok); 191 checks++; 192 if (st != KIT_GRAM_LEX_TOKEN || tok.kind != cases[i].want || tok.len != strlen(s)) { 193 failures++; 194 fprintf(stderr, "FAIL keyword-pin %s: st=%d kind=%u (want %u) len=%zu\n", 195 s, (int)st, tok.kind, cases[i].want, tok.len); 196 } else { 197 printf("ok keyword-pin %s -> kind %u\n", s, tok.kind); 198 } 199 } 200 } 201 202 /* ---- match API equivalence vs kit_gram_match_* --------------------------------- */ 203 static int match_eq(const KitGramMatch *x, const KitGramMatch *y) { 204 return x->start == y->start && x->end == y->end && x->kind == y->kind; 205 } 206 207 static void check_match(const char *name, KitGramMatcher *m, const char *s, int multiline) { 208 size_t len = strlen(s); 209 KitGramMatchOpts opts = { .multiline = multiline != 0 }; 210 const KitGramMatchOpts *op = multiline ? &opts : NULL; 211 KitGramLexInput in; 212 KitGramLexInputSpan span; 213 KitGramMatch tm, am; 214 int ok = 1; 215 216 /* anchored */ 217 kit_gram_lex_input_init(&in, NULL); span = (KitGramLexInputSpan){ .bytes = (const unsigned char *)s, .len = len }; 218 kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 219 int rt = kit_gram_match_anchored(m, &in, op, &tm); 220 int ra = matchsa_match_anchored((const unsigned char *)s, len, op, &am); 221 ok &= (rt == ra) && (!rt || match_eq(&tm, &am)); 222 223 /* full */ 224 kit_gram_lex_input_init(&in, NULL); span = (KitGramLexInputSpan){ .bytes = (const unsigned char *)s, .len = len }; 225 kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 226 rt = kit_gram_match_full(m, &in, op, &tm); 227 ra = matchsa_match_full((const unsigned char *)s, len, op, &am); 228 ok &= (rt == ra) && (!rt || match_eq(&tm, &am)); 229 230 /* find */ 231 kit_gram_lex_input_init(&in, NULL); span = (KitGramLexInputSpan){ .bytes = (const unsigned char *)s, .len = len }; 232 kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 233 rt = kit_gram_match_find(m, &in, op, &tm); 234 ra = matchsa_match_find((const unsigned char *)s, len, op, &am); 235 ok &= (rt == ra) && (!rt || match_eq(&tm, &am)); 236 237 /* find iterator: drain both and compare the full match sequences */ 238 KitGramMatch tall[16], aall[16]; 239 kit_gram_lex_input_init(&in, NULL); span = (KitGramLexInputSpan){ .bytes = (const unsigned char *)s, .len = len }; 240 kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 241 KitGramMatchIter tit; kit_gram_match_iter_init(&tit, m, &in, op); 242 size_t nt = 0; while (nt < 16 && kit_gram_match_iter_next(&tit, &tall[nt])) nt++; 243 sa_match_iter ait; matchsa_match_iter_init(&ait, (const unsigned char *)s, len, op); 244 size_t naa = 0; while (naa < 16 && matchsa_match_iter_next(&ait, &aall[naa])) naa++; 245 ok &= (nt == naa); 246 for (size_t i = 0; ok && i < nt; i++) ok &= match_eq(&tall[i], &aall[i]); 247 248 checks++; 249 if (!ok) { failures++; fprintf(stderr, "FAIL match %s (ml=%d)\n", name, multiline); } 250 else printf("ok match %s (ml=%d)\n", name, multiline); 251 } 252 253 /* ---- fused parser: recording value-channel actions ----------------------- */ 254 typedef struct { int type; uint64_t a, b; } Ev; 255 typedef struct { Ev *v; size_t n, cap; uint64_t counter; } Rec; 256 static void rec_push(Rec *r, int type, uint64_t a, uint64_t b) { 257 if (r->n == r->cap) { r->cap = r->cap ? r->cap * 2 : 256; r->v = realloc(r->v, r->cap * sizeof *r->v); } 258 r->v[r->n++] = (Ev){ type, a, b }; 259 } 260 static KitGramSem rec_handle(Rec *r) { return (KitGramSem)(uintptr_t)(++r->counter); } 261 static KitGramSem rc_lift(void *ud, KitGramToken t) { 262 Rec *r = ud; KitGramSem h = rec_handle(r); rec_push(r, 1, t.kind, (uint64_t)(uintptr_t)h); return h; 263 } 264 static KitGramSem rc_reduce(void *ud, KitGramRuleId rule, int prod, KitGramSem *kids, size_t n) { 265 Rec *r = ud; uint64_t kh = 1469598103934665603ull; 266 for (size_t i = 0; i < n; i++) kh = (kh ^ (uint64_t)(uintptr_t)kids[i]) * 1099511628211ull; 267 rec_push(r, 2, ((uint64_t)rule << 8) | (unsigned)prod, kh); return rec_handle(r); 268 } 269 static KitGramSem rc_le(void *ud) { return rec_handle(ud); } 270 static KitGramSem rc_lp(void *ud, KitGramSem l, KitGramSem it) { 271 Rec *r = ud; rec_push(r, 3, (uint64_t)(uintptr_t)l, (uint64_t)(uintptr_t)it); return rec_handle(r); 272 } 273 static KitGramSem rc_os(void *ud, KitGramSem it) { Rec *r = ud; rec_push(r, 4, (uint64_t)(uintptr_t)it, 0); return rec_handle(r); } 274 static KitGramSem rc_on(void *ud) { return rec_handle(ud); } 275 static const KitGramActions REC_ACT = { 276 .lift_token = rc_lift, .reduce = rc_reduce, 277 .list_empty = rc_le, .list_push = rc_lp, .opt_some = rc_os, .opt_none = rc_on, 278 }; 279 static int ev_eq(const Rec *a, const Rec *b) { 280 if (a->n != b->n) return 0; 281 for (size_t i = 0; i < a->n; i++) 282 if (a->v[i].type != b->v[i].type || a->v[i].a != b->v[i].a || a->v[i].b != b->v[i].b) return 0; 283 return 1; 284 } 285 286 typedef KitGramSem (*parse_rd_fn)(const KitGramToken *, size_t, const KitGramActions *, void *, KitGramError *, int *); 287 typedef KitGramSem (*parse_fused_fn)(const unsigned char *, size_t, const KitGramActions *, void *, KitGramError *, int *); 288 289 static void check_fused(const char *name, sa_init_fn init, sa_next_fn next, 290 parse_rd_fn rd, parse_fused_fn fused, const char *s, int want_ok) { 291 size_t len = strlen(s); 292 /* tokenize with the standalone tokenizer (already proven == table lexer) */ 293 KitGramToken *toks = xmalloc((len + 1) * sizeof *toks); 294 sa_lex lx; init(&lx, (const unsigned char *)s, len); 295 size_t nt = 0; KitGramLexStatus lst; 296 for (;;) { KitGramToken t; lst = next(&lx, &t); if (lst != KIT_GRAM_LEX_TOKEN) break; toks[nt++] = t; } 297 298 Rec rr = {0}, rf = {0}; 299 int ok_rd = 0, ok_fused = 0; 300 KitGramError e_rd = {0}, e_fused = {0}; 301 if (lst == KIT_GRAM_LEX_EOF) rd(toks, nt, &REC_ACT, &rr, &e_rd, &ok_rd); 302 fused((const unsigned char *)s, len, &REC_ACT, &rf, &e_fused, &ok_fused); 303 304 checks++; 305 int ok = (ok_fused == want_ok); 306 if (lst == KIT_GRAM_LEX_EOF) ok = ok && (ok_rd == ok_fused) && (!ok_fused || ev_eq(&rr, &rf)); 307 if (!ok) { 308 failures++; 309 fprintf(stderr, "FAIL fused %s: lex_st=%d ok rd=%d fused=%d (want %d), ev rd=%zu fused=%zu\n", 310 name, (int)lst, ok_rd, ok_fused, want_ok, rr.n, rf.n); 311 } else { 312 printf("ok fused %s (ok=%d, %zu events)\n", name, ok_fused, rf.n); 313 } 314 free(toks); free(rr.v); free(rf.v); 315 } 316 317 int main(void) { 318 KitGramCompiled *json = compile("test/gram/realistic/json.ebnf", 0); 319 KitGramCompiled *clike = compile("test/gram/realistic/clike.ebnf", 0); 320 KitGramCompiled *kwutf8 = compile("test/gram/keywords_utf8.ebnf", 0); 321 KitGramCompiled *anch = compile("test/gram/anchors_start.ebnf", 1); 322 KitGramCompiled *anchz = compile("test/gram/anchors.ebnf", 1); 323 KitGramCompiled *pos = compile("test/gram/standalone/pos.ebnf", 1); 324 KitGramCompiled *match = compile("test/gram/standalone/match.ebnf", 1); 325 326 /* json: nesting, strings with escapes, numbers with exponent, newlines, then bad byte */ 327 const char *json_ok = 328 "[\n {\"id\": 12, \"name\": \"a\\nb\", \"x\": -3.5e2, \"ok\": true},\n" 329 " {\"y\": [1, 2, null, false]}\n]"; 330 const char *json_bad = "[1, @]"; 331 check_equiv("json/ok", kit_gram_lexer_grammar(json), jsonsa_lex_init, jsonsa_lex_next, json_ok, strlen(json_ok)); 332 check_equiv("json/bad", kit_gram_lexer_grammar(json), jsonsa_lex_init, jsonsa_lex_next, json_bad, strlen(json_bad)); 333 334 /* clike: keywords vs identifiers (iffy != if), punctuation, multi-line, then 335 * bad. clikesa uses the default keyword MPH; clikesaf folds keywords into 336 * the DFA (--fold-keywords). Both must agree with the table lexer. */ 337 const char *clike_ok = 338 "int main(int argc) {\n" 339 " int iffy = argc + 1;\n" 340 " while (iffy) { iffy = iffy - 1; }\n" 341 " if (argc < 2) return 0; else return iffy;\n" 342 "}\n"; 343 const char *clike_bad = "int x = `;"; 344 check_equiv("clike/ok", kit_gram_lexer_grammar(clike), clikesa_lex_init, clikesa_lex_next, clike_ok, strlen(clike_ok)); 345 check_equiv("clike/bad", kit_gram_lexer_grammar(clike), clikesa_lex_init, clikesa_lex_next, clike_bad, strlen(clike_bad)); 346 check_equiv("clike/fold-ok", kit_gram_lexer_grammar(clike), clikesaf_lex_init, clikesaf_lex_next, clike_ok, strlen(clike_ok)); 347 check_equiv("clike/fold-bad", kit_gram_lexer_grammar(clike), clikesaf_lex_init, clikesaf_lex_next, clike_bad, strlen(clike_bad)); 348 349 /* kwutf8: ASCII + multi-byte folded keywords (café, λ), keyword-prefixed ident, ints */ 350 const char *kw_ok = "if caf\xC3\xA9 \xCE\xBB while iffy 12 fn let \xCE\xBBx"; 351 const char *kw_bad = "ok \xFF bad"; /* invalid UTF-8 byte */ 352 check_equiv("kwutf8/ok", kit_gram_lexer_grammar(kwutf8), kwutf8sa_lex_init, kwutf8sa_lex_next, kw_ok, strlen(kw_ok)); 353 check_equiv("kwutf8/bad", kit_gram_lexer_grammar(kwutf8), kwutf8sa_lex_init, kwutf8sa_lex_next, kw_bad, strlen(kw_bad)); 354 355 /* anchors_start (multiline): \A"if" only at text start; ^"ln" only at line start */ 356 const char *anch_in = "if\nln\nlnx\nif x\nln"; 357 check_equiv("anchors_start", kit_gram_lexer_grammar(anch), anchstartsa_lex_init, anchstartsa_lex_next, anch_in, strlen(anch_in)); 358 359 /* anchors (multiline): end anchors \z/$ — the additive codegen scanner can't 360 * do these, so this is standalone-only coverage. No whitespace skip, so the 361 * stream ends in an error on the first newline; both runtimes must agree on 362 * the token before it and the error. */ 363 const char *anchz_a = "d"; /* "d" at end of text: DOL ("d" $) wins over ID */ 364 const char *anchz_b = "x"; /* "x" at end of text: END ("x" \z) wins over ID */ 365 const char *anchz_c = "lnx"; /* longest match: ID, not the ^"ln" prefix */ 366 const char *anchz_d = "if"; /* \A"if" at text start */ 367 check_equiv("anchors/dol", kit_gram_lexer_grammar(anchz), anchsa_lex_init, anchsa_lex_next, anchz_a, strlen(anchz_a)); 368 check_equiv("anchors/end", kit_gram_lexer_grammar(anchz), anchsa_lex_init, anchsa_lex_next, anchz_b, strlen(anchz_b)); 369 check_equiv("anchors/lng", kit_gram_lexer_grammar(anchz), anchsa_lex_init, anchsa_lex_next, anchz_c, strlen(anchz_c)); 370 check_equiv("anchors/kw", kit_gram_lexer_grammar(anchz), anchsa_lex_init, anchsa_lex_next, anchz_d, strlen(anchz_d)); 371 372 /* UTF-8 position: WORDs/NUMs separated by ASCII and non-ASCII line breaks. 373 * CRLF is one break (line +1, not +2); U+2028 (LS) and U+0085 (NEL) are 374 * breaks; scalar columns count code points, not bytes (the multi-byte breaks 375 * advance the column by one when they are not line breaks — here they always 376 * are). Compared scalar-for-scalar against the table lexer. */ 377 const char *pos_in = "ab12\r\ncd\xE2\x80\xA8" "e\xC2\x85" "fgh\nij9"; 378 check_equiv("utf8-pos", kit_gram_lexer_grammar(pos), possa_lex_init, possa_lex_next, pos_in, strlen(pos_in)); 379 /* CR alone, then LF that is NOT part of a preceding CR (after content) */ 380 const char *pos_cr = "a\rb\nc\r\nd"; 381 check_equiv("utf8-cr", kit_gram_lexer_grammar(pos), possa_lex_init, possa_lex_next, pos_cr, strlen(pos_cr)); 382 383 /* keyword folding priority pins */ 384 check_keyword_pins(clike); 385 386 /* match API vs kit_gram_match_* */ 387 KitGramMatcher mg; kit_gram_matcher_bind(&mg, kit_gram_lexer_grammar(match)); 388 check_match("digits/find", &mg, "abc123def45", 0); 389 check_match("digits/anch", &mg, "123abc", 0); 390 check_match("digits/full", &mg, "123", 0); 391 check_match("empty", &mg, "", 0); 392 check_match("word-end/noml", &mg, "abc\ndef", 0); 393 check_match("word-end/ml", &mg, "abc\ndef", 1); 394 check_match("mixed/ml", &mg, "x12\nword\n9", 1); 395 396 /* fused parser over the table-free scanner (rd_refill_lex) */ 397 check_fused("json/ok", jsonsr_lex_init, jsonsr_lex_next, jsonsr_parse_rd, jsonsr_parse_fused, 398 "[1, 2, {\"a\": true, \"b\": [null, -3.5e2]}, \"x\"]", 1); 399 check_fused("json/bad", jsonsr_lex_init, jsonsr_lex_next, jsonsr_parse_rd, jsonsr_parse_fused, 400 "[1, 2,]", 0); 401 check_fused("clike/ok", clikesr_lex_init, clikesr_lex_next, clikesr_parse_rd, clikesr_parse_fused, 402 "int main(int a){ int x = a + 1 * 2; if (x < 3) return x; else return 0; }", 1); 403 check_fused("clike/kw", clikesr_lex_init, clikesr_lex_next, clikesr_parse_rd, clikesr_parse_fused, 404 "int iffy = if;", 0); 405 406 fprintf(stderr, "%d checks, %d failures\n", checks, failures); 407 return failures ? 1 : 0; 408 }