sublexer_table_test.c (11575B)
1 /* test_sublexer_table.c — the sub-lexer idiom over the *table* runtime and the 2 * in-process kit_gram_lexer_* introspection API, the counterpart to the standalone 3 * test/standalone/test_sublexer.c. 4 * 5 * Same grammar (test/standalone/sublexer.ebnf), same realistic frontend shape: 6 * a main lexer yields whole NUMBER / STRING tokens, and a dedicated sub-lexer 7 * re-scans each token's *bytes* into its parts — NUMBER into sign/int/frac/exp, 8 * STRING into quotes/literal-runs/escape-atoms. Here the sub-lexers are reached 9 * through kit_gram_find_lexer / kit_gram_lexer_grammar_at (the named %lex blocks the 10 * in-process API now exposes) and driven with the ordinary kit_gram_lexer_* runtime 11 * plus the kit_gram_match_* API. The asserted results are byte-for-byte the ones the 12 * standalone test pins, so this proves the table path and the standalone path 13 * agree on sub-lexing. 14 */ 15 #include <kit/gram.h> 16 #include <kit/gram_lex.h> 17 18 #include <stdio.h> 19 #include <stdlib.h> 20 #include <string.h> 21 22 #include "gram_test.h" 23 24 static int failures = 0, checks = 0; 25 static void check(int cond, const char *msg) { 26 checks++; 27 printf("%s %s\n", cond ? "ok " : "FAIL", msg); 28 if (!cond) failures++; 29 } 30 31 /* Token kinds, resolved by name from the compiled grammar (the in-process 32 * counterpart of the generated TOK_* enum). */ 33 static KitGramTokenKind K_NUMBER, K_STRING, K_NSIGN, K_NINT, K_NFRAC, K_NEXP, 34 K_SQUOTE, K_SESC, K_SCHARS; 35 36 /* Bind a resident single span as a finished input. */ 37 static void input_set(KitGramLexInput *in, KitGramLexInputSpan *span, 38 const char *bytes, size_t len) { 39 kit_gram_lex_input_init(in, NULL); 40 *span = (KitGramLexInputSpan){ .bytes = (const unsigned char *)bytes, .len = len }; 41 kit_gram_lex_input_push(in, span); 42 kit_gram_lex_input_finish(in); 43 } 44 45 /* ---- NUMBER sub-lexer: split a numeric literal into its lexical parts ----- */ 46 typedef struct { 47 int neg; long int_part; int has_frac; int has_exp; char shape[64]; 48 } NumParts; 49 50 /* Drive the `number` sub-lexer over a NUMBER token's bytes with the table 51 * runtime. Returns 1 iff it ran to EOF having consumed every byte. */ 52 static int relex_number(const KitGramLexGrammar *g, const char *bytes, size_t len, 53 NumParts *np) { 54 memset(np, 0, sizeof *np); 55 KitGramLexInput in; KitGramLexInputSpan span; 56 input_set(&in, &span, bytes, len); 57 KitGramLexer lx; 58 kit_gram_lexer_init(&lx, g, &in, &(KitGramLexConfig){0}); 59 size_t covered = 0, sp = 0; 60 KitGramToken t; KitGramLexStatus st; 61 while ((st = kit_gram_lexer_next(&lx, &t)) == KIT_GRAM_LEX_TOKEN) { 62 covered += t.len; 63 for (size_t i = 0; i < t.len && sp < sizeof np->shape - 1; i++) 64 np->shape[sp++] = (char)t.lexeme[i]; 65 if (t.kind == K_NSIGN) np->neg = 1; 66 else if (t.kind == K_NINT) { 67 char b[32]; 68 size_t k = t.len < sizeof b - 1 ? t.len : sizeof b - 1; 69 memcpy(b, t.lexeme, k); b[k] = 0; np->int_part = atol(b); 70 } else if (t.kind == K_NFRAC) np->has_frac = 1; 71 else if (t.kind == K_NEXP) np->has_exp = 1; 72 } 73 np->shape[sp] = 0; 74 return st == KIT_GRAM_LEX_EOF && covered == len; 75 } 76 77 /* ---- STRING sub-lexer: unescape a quoted string into raw bytes ----------- */ 78 static int hexval(int c) { 79 if (c >= '0' && c <= '9') return c - '0'; 80 if (c >= 'a' && c <= 'f') return c - 'a' + 10; 81 if (c >= 'A' && c <= 'F') return c - 'A' + 10; 82 return 0; 83 } 84 static size_t decode_escape(const char *e, char *out) { 85 switch (e[1]) { 86 case '"': out[0] = '"'; return 1; 87 case '\\': out[0] = '\\'; return 1; 88 case '/': out[0] = '/'; return 1; 89 case 'b': out[0] = '\b'; return 1; 90 case 'f': out[0] = '\f'; return 1; 91 case 'n': out[0] = '\n'; return 1; 92 case 'r': out[0] = '\r'; return 1; 93 case 't': out[0] = '\t'; return 1; 94 case 'u': { 95 unsigned cp = 0; 96 for (int i = 0; i < 4; i++) 97 cp = cp * 16 + (unsigned)hexval((unsigned char)e[2 + i]); 98 if (cp < 0x80u) { out[0] = (char)cp; return 1; } 99 if (cp < 0x800u) { 100 out[0] = (char)(0xC0u | (cp >> 6)); 101 out[1] = (char)(0x80u | (cp & 0x3Fu)); 102 return 2; 103 } 104 out[0] = (char)(0xE0u | (cp >> 12)); 105 out[1] = (char)(0x80u | ((cp >> 6) & 0x3Fu)); 106 out[2] = (char)(0x80u | (cp & 0x3Fu)); 107 return 3; 108 } 109 default: return 0; 110 } 111 } 112 static int relex_string(const KitGramLexGrammar *g, const char *bytes, size_t len, 113 char *out, size_t *outn) { 114 KitGramLexInput in; KitGramLexInputSpan span; 115 input_set(&in, &span, bytes, len); 116 KitGramLexer lx; 117 kit_gram_lexer_init(&lx, g, &in, &(KitGramLexConfig){0}); 118 size_t covered = 0, on = 0; 119 int quotes = 0; 120 KitGramToken t; KitGramLexStatus st; 121 while ((st = kit_gram_lexer_next(&lx, &t)) == KIT_GRAM_LEX_TOKEN) { 122 covered += t.len; 123 if (t.kind == K_SQUOTE) quotes++; 124 else if (t.kind == K_SCHARS) { memcpy(out + on, t.lexeme, t.len); on += t.len; } 125 else if (t.kind == K_SESC) on += decode_escape((const char *)t.lexeme, out + on); 126 } 127 out[on] = 0; *outn = on; 128 return st == KIT_GRAM_LEX_EOF && covered == len && quotes == 2; 129 } 130 131 /* ---- per-lexer match API over the table kit_gram_match_* drivers ---------------- */ 132 static void test_match_api(KitGramCompiled *c) { 133 size_t inum, istr; 134 check(kit_gram_find_lexer(c, "number", &inum), "find_lexer number"); 135 check(kit_gram_find_lexer(c, "strescape", &istr), "find_lexer strescape"); 136 KitGramMatcher mmain; kit_gram_matcher_bind(&mmain, kit_gram_lexer_grammar(c)); 137 KitGramMatcher mnum; kit_gram_matcher_bind(&mnum, kit_gram_lexer_grammar_at(c, inum)); 138 KitGramMatcher mstr; kit_gram_matcher_bind(&mstr, kit_gram_lexer_grammar_at(c, istr)); 139 140 KitGramLexInput in; KitGramLexInputSpan span; KitGramMatch m; 141 142 /* main lexer: full-match a bare NUMBER. */ 143 input_set(&in, &span, "42", 2); 144 check(kit_gram_match_full(&mmain, &in, NULL, &m) && m.kind == K_NUMBER && 145 m.start == 0 && m.end == 2, 146 "main match_full: 42 -> NUMBER [0,2)"); 147 148 /* number sub-lexer: find the integer run inside a literal. */ 149 input_set(&in, &span, "42", 2); 150 check(kit_gram_match_find(&mnum, &in, NULL, &m) && m.kind == K_NINT && 151 m.start == 0 && m.end == 2, 152 "number match_find: 42 -> NINT [0,2)"); 153 154 /* number sub-lexer: iterate every non-overlapping part of -3.14e-2. */ 155 input_set(&in, &span, "-3.14e-2", 8); 156 KitGramMatchIter it; 157 kit_gram_match_iter_init(&it, &mnum, &in, NULL); 158 KitGramTokenKind parts[8]; int np = 0; 159 while (kit_gram_match_iter_next(&it, &m) && np < 8) parts[np++] = m.kind; 160 check(np == 4 && parts[0] == K_NSIGN && parts[1] == K_NINT && 161 parts[2] == K_NFRAC && parts[3] == K_NEXP, 162 "number match_iter: -3.14e-2 -> NSIGN NINT NFRAC NEXP"); 163 164 /* strescape sub-lexer: anchored match of the opening quote. */ 165 input_set(&in, &span, "\"hi\"", 4); 166 check(kit_gram_match_anchored(&mstr, &in, NULL, &m) && m.kind == K_SQUOTE && 167 m.start == 0 && m.end == 1, 168 "strescape match_anchored: leading quote -> SQUOTE [0,1)"); 169 } 170 171 int main(void) { 172 static char SRC[1 << 14]; 173 FILE *f = fopen("test/gram/standalone/sublexer.ebnf", "rb"); 174 if (!f) { fprintf(stderr, "cannot open sublexer.ebnf\n"); return 2; } 175 size_t n = fread(SRC, 1, sizeof SRC - 1, f); 176 fclose(f); SRC[n] = 0; 177 178 KitGramOptions opts = {0}; 179 KitGramCompiled *c = NULL; 180 KitStatus cst = kit_gram_compile_text(gram_test_ctx(), (KitSlice){.s=SRC,.len=n}, 181 KIT_SLICE_LIT("sublexer"), &opts, &c); 182 check(cst == KIT_OK && c != NULL, "compile sublexer.ebnf in-process"); 183 if (cst != KIT_OK || !c) return 1; 184 185 /* The grammar declares three independent lexers, main first. */ 186 check(kit_gram_lexer_count(c) == 3, "lexer_count == 3 (main + number + strescape)"); 187 check(kit_gram_lexer_name(c, 0) && strcmp(kit_gram_lexer_name(c, 0), "main") == 0, 188 "lexer[0] is the main block"); 189 /* kit_gram_lexer_grammar shorthand resolves to the main block. */ 190 size_t imain; kit_gram_find_lexer(c, "main", &imain); 191 check(kit_gram_lexer_grammar(c) == kit_gram_lexer_grammar_at(c, imain), 192 "kit_gram_lexer_grammar == grammar_at(main)"); 193 194 kit_gram_find_token(c, "NUMBER", &K_NUMBER); 195 kit_gram_find_token(c, "STRING", &K_STRING); 196 kit_gram_find_token(c, "NSIGN", &K_NSIGN); 197 kit_gram_find_token(c, "NINT", &K_NINT); 198 kit_gram_find_token(c, "NFRAC", &K_NFRAC); 199 kit_gram_find_token(c, "NEXP", &K_NEXP); 200 kit_gram_find_token(c, "SQUOTE", &K_SQUOTE); 201 kit_gram_find_token(c, "SESC", &K_SESC); 202 kit_gram_find_token(c, "SCHARS", &K_SCHARS); 203 204 test_match_api(c); 205 206 const KitGramLexGrammar *gmain = kit_gram_lexer_grammar(c); 207 size_t inum, istr; 208 kit_gram_find_lexer(c, "number", &inum); 209 kit_gram_find_lexer(c, "strescape", &istr); 210 const KitGramLexGrammar *gnum = kit_gram_lexer_grammar_at(c, inum); 211 const KitGramLexGrammar *gstr = kit_gram_lexer_grammar_at(c, istr); 212 213 /* Same corpus as the standalone test: drive the main lexer, sub-lex each 214 * NUMBER / STRING token's bytes, assert identical decoded results. */ 215 const char *src = 216 " -3.14e-2 42 0 \"a\\tb\" \"x\\u00e9y\" \"plain\" "; 217 KitGramLexInput in; KitGramLexInputSpan span; 218 input_set(&in, &span, src, strlen(src)); 219 KitGramLexer lx; 220 kit_gram_lexer_init(&lx, gmain, &in, &(KitGramLexConfig){0}); 221 222 int n_num = 0, n_str = 0; 223 KitGramToken tok; KitGramLexStatus st; 224 while ((st = kit_gram_lexer_next(&lx, &tok)) == KIT_GRAM_LEX_TOKEN) { 225 if (tok.kind == K_NUMBER) { 226 NumParts np; 227 int ok = relex_number(gnum, (const char *)tok.lexeme, tok.len, &np); 228 ok = ok && strlen(np.shape) == tok.len && 229 memcmp(np.shape, tok.lexeme, tok.len) == 0; 230 if (n_num == 0) 231 check(ok && np.neg && np.int_part == 3 && np.has_frac && np.has_exp, 232 "number sublex: -3.14e-2 -> -, int 3, frac, exp (full cover)"); 233 else if (n_num == 1) 234 check(ok && !np.neg && np.int_part == 42 && !np.has_frac && !np.has_exp, 235 "number sublex: 42 -> int 42 only"); 236 else if (n_num == 2) 237 check(ok && !np.neg && np.int_part == 0 && !np.has_frac && !np.has_exp, 238 "number sublex: 0 -> int 0 only"); 239 n_num++; 240 } else if (tok.kind == K_STRING) { 241 char out[256]; size_t on = 0; 242 int ok = relex_string(gstr, (const char *)tok.lexeme, tok.len, out, &on); 243 if (n_str == 0) 244 check(ok && on == 3 && memcmp(out, "a\tb", 3) == 0, 245 "string sublex: \"a\\tb\" -> a<TAB>b"); 246 else if (n_str == 1) 247 check(ok && on == 4 && memcmp(out, "x\xc3\xa9y", 4) == 0, 248 "string sublex: \"x\\u00e9y\" -> x U+00E9(UTF-8) y"); 249 else if (n_str == 2) 250 check(ok && on == 5 && memcmp(out, "plain", 5) == 0, 251 "string sublex: \"plain\" -> plain (no escapes)"); 252 n_str++; 253 } 254 } 255 check(st == KIT_GRAM_LEX_EOF, "main lexer reached EOF cleanly"); 256 check(n_num == 3 && n_str == 3, "main lexer produced 3 NUMBER + 3 STRING tokens"); 257 258 kit_gram_free(c); 259 printf("\n%d checks, %d failures\n", checks, failures); 260 return failures ? 1 : 0; 261 }