keywords_test.c (9006B)
1 /* test_keywords.c - explicit %keywords extraction with a minimal perfect hash. 2 * 3 * Part A drives the generated byte lexer from test/keywords.ebnf: the host 4 * (IDENT) stays in the DFA and keywords are recovered by the MPH, so keyword 5 * vs identifier, prefixes, and longer matches all tokenize correctly, and all 6 * three entry forms (NAME = "lit", bare "lit", bare NAME) work. 7 * 8 * Part B uses the in-process compiler for the parser-literal interaction, the 9 * re2::Set match API, and utf8-mode keywords (the MPH is byte-level, so it works 10 * unchanged in both modes). */ 11 #include "generated_keywords.h" 12 #include <kit/gram.h> 13 #include <kit/support/gram_lex_tables.h> 14 15 #include <stdio.h> 16 #include <stdlib.h> 17 #include <string.h> 18 19 #include "gram_test.h" 20 21 static int failures = 0; 22 23 static void ck(const char *what, int ok) { 24 printf("%s %s\n", ok ? "ok " : "FAIL", what); 25 if (!ok) failures++; 26 } 27 28 /* ---- Part A: the generated byte lexer ------------------------------------ */ 29 30 typedef struct { 31 KitGramTokenKind kind; 32 const char *lexeme; 33 } Want; 34 35 static void check_stream(const char *src, const Want *want, size_t nwant) { 36 KitGramLexInput in; 37 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0}); 38 KitGramLexer lx; 39 KitGramLexConfig cfg = {0}; 40 keywords_lexer_init(&lx, &in, &cfg); 41 KitGramLexInputSpan sp = { .bytes = (const unsigned char *)src, .len = strlen(src) }; 42 kit_gram_lex_input_push(&in, &sp); 43 kit_gram_lex_input_finish(&in); 44 45 size_t i = 0; 46 char label[256]; 47 for (;;) { 48 KitGramToken t; 49 KitGramLexStatus st = kit_gram_lexer_next(&lx, &t); 50 if (st == KIT_GRAM_LEX_TOKEN) { 51 if (i >= nwant) { ck("lexer emitted too many tokens", 0); return; } 52 int ok = t.kind == want[i].kind && t.len == strlen(want[i].lexeme) && 53 strncmp(t.lexeme, want[i].lexeme, t.len) == 0; 54 snprintf(label, sizeof label, "[%s] tok[%zu] kind=%u \"%.*s\" (want kind=%u \"%s\")", 55 src, i, (unsigned)t.kind, (int)t.len, t.lexeme, 56 (unsigned)want[i].kind, want[i].lexeme); 57 ck(label, ok); 58 i++; 59 continue; 60 } 61 if (st == KIT_GRAM_LEX_EOF) { ck("stream fully consumed", i == nwant); return; } 62 ck("unexpected lexer status", 0); 63 return; 64 } 65 } 66 67 static void test_generated_lexer(void) { 68 printf("== generated keyword lexer (byte) ==\n"); 69 { /* keywords win exact-lexeme ties; longer/embedded forms stay IDENT */ 70 Want w[] = { 71 { KEYWORDS_TOK_IF, "if" }, { KEYWORDS_TOK_IDENT, "iffy" }, { KEYWORDS_TOK_IDENT, "ifx" }, 72 { KEYWORDS_TOK_RETURN, "return" }, { KEYWORDS_TOK_IDENT, "returns" }, 73 { KEYWORDS_TOK_WHILE, "while" }, { KEYWORDS_TOK_IDENT, "_while" }, 74 }; 75 check_stream("if iffy ifx return returns while _while", w, 7); 76 } 77 { /* the three entry forms: full (STRUCT), bare string (SIZEOF), bare name (TYPEDEF) */ 78 Want w[] = { { KEYWORDS_TOK_STRUCT, "struct" }, { KEYWORDS_TOK_SIZEOF, "sizeof" }, { KEYWORDS_TOK_TYPEDEF, "typedef" }, 79 { KEYWORDS_TOK_IDENT, "typedefs" } }; 80 check_stream("struct sizeof typedef typedefs", w, 4); 81 } 82 { /* numbers, strings, comments, operators around keywords */ 83 Want w[] = { 84 { KEYWORDS_TOK_FOR, "for" }, { KEYWORDS_TOK_INT, "0" }, { KEYWORDS_TOK_HEX, "0x1f" }, { KEYWORDS_TOK_FLOAT, "3.5" }, 85 { KEYWORDS_TOK_BREAK, "break" }, { KEYWORDS_TOK_IDENT, "x" }, 86 }; 87 check_stream("for 0 0x1f 3.5 // a comment\n break x", w, 6); 88 } 89 } 90 91 /* ---- Part B: the in-process compiler ------------------------------------- */ 92 93 static KitGramCompiled *compile(const char *grammar) { 94 KitGramOptions opts = {0}; 95 KitGramCompiled *c = NULL; 96 KitStatus st = kit_gram_compile_text(gram_test_ctx(), 97 (KitSlice){.s=grammar,.len=strlen(grammar)}, 98 KIT_SLICE_LIT("<kw>"), &opts, &c); 99 if (st != KIT_OK || !c) printf("compile failed (diagnostic on stderr)\n"); 100 return c; 101 } 102 103 static size_t tokenize(const KitGramLexGrammar *g, const char *src, KitGramTokenKind *out, size_t cap) { 104 KitGramLexInput in; 105 kit_gram_lex_input_init(&in, NULL); 106 KitGramLexInputSpan sp = { .bytes = (const unsigned char *)src, .len = strlen(src) }; 107 kit_gram_lex_input_push(&in, &sp); 108 kit_gram_lex_input_finish(&in); 109 KitGramLexer lx; 110 kit_gram_lexer_init(&lx, g, &in, NULL); 111 size_t n = 0; 112 for (;;) { 113 KitGramToken t; 114 if (kit_gram_lexer_next(&lx, &t) != KIT_GRAM_LEX_TOKEN) break; 115 if (n < cap) out[n] = t.kind; 116 n++; 117 } 118 return n; 119 } 120 121 /* A keyword referenced as a parser string literal resolves to the keyword token 122 * and is NOT given its own DFA recognizer (the host DFA stays small). */ 123 static void test_parser_literal(void) { 124 printf("== parser-literal keyword ==\n"); 125 static const char *G = 126 "%lex {\n" 127 " %skip WS = [ \\t]+;\n" 128 " IDENT = [A-Za-z_][A-Za-z0-9_]*;\n" 129 " %keywords IDENT { IF = \"if\"; ELSE = \"else\"; }\n" 130 "}\n" 131 "stmt = \"if\" IDENT | ELSE | IDENT;\n"; /* "if" literal == keyword IF */ 132 KitGramCompiled *c = compile(G); 133 if (!c) { ck("parser-literal grammar compiles", 0); return; } 134 const KitGramLexGrammar *g = kit_gram_lexer_grammar(c); 135 ck("host DFA stays tiny (keywords not merged)", g->nstates <= 4); 136 ck("one keyword table", g->nkeyword_tables == 1); 137 KitGramTokenKind tif = 0, tident = 0; 138 kit_gram_find_token(c, "IF", &tif); 139 kit_gram_find_token(c, "IDENT", &tident); 140 KitGramTokenKind a[8]; 141 size_t n = tokenize(g, "if x else", a, 8); 142 ck("\"if x else\" -> IF IDENT ELSE", n == 3 && a[0] == tif && a[1] == tident); 143 kit_gram_free(c); 144 } 145 146 /* The match API rewrites extracted keywords too, so a Set match reports the 147 * keyword kind. */ 148 static void test_match_api(void) { 149 printf("== match API ==\n"); 150 static const char *G = 151 "%lex {\n IDENT = [A-Za-z_][A-Za-z0-9_]*;\n" 152 " %keywords IDENT { IF = \"if\"; WHILE = \"while\"; }\n}\n" 153 "p = IF | WHILE | IDENT;\n"; 154 KitGramCompiled *c = compile(G); 155 if (!c) { ck("match grammar compiles", 0); return; } 156 KitGramMatcher mt; kit_gram_matcher_bind(&mt, kit_gram_lexer_grammar(c)); 157 KitGramTokenKind tif = 0, tident = 0; 158 kit_gram_find_token(c, "IF", &tif); 159 kit_gram_find_token(c, "IDENT", &tident); 160 KitGramLexInput in; 161 KitGramLexInputSpan sp; 162 KitGramMatch m; 163 kit_gram_lex_input_init(&in, NULL); 164 sp = (KitGramLexInputSpan){ .bytes = (const unsigned char *)"if", .len = 2 }; 165 kit_gram_lex_input_push(&in, &sp); 166 kit_gram_lex_input_finish(&in); 167 ck("match \"if\" -> IF", kit_gram_match_anchored(&mt, &in, NULL, &m) && m.end == 2 && m.kind == tif); 168 kit_gram_lex_input_init(&in, NULL); 169 sp = (KitGramLexInputSpan){ .bytes = (const unsigned char *)"iffy", .len = 4 }; 170 kit_gram_lex_input_push(&in, &sp); 171 kit_gram_lex_input_finish(&in); 172 ck("match \"iffy\" -> IDENT", kit_gram_match_anchored(&mt, &in, NULL, &m) && m.end == 4 && m.kind == tident); 173 kit_gram_free(c); 174 } 175 176 #ifndef KIT_GRAM_NO_UNICODE 177 /* The MPH is byte-level, so %keywords work in utf8 mode too, including 178 * multi-byte keywords and identifiers. */ 179 static void test_utf8(void) { 180 printf("== utf8 keywords ==\n"); 181 static const char *G = 182 "%lex :utf8 {\n" 183 " %skip WS = \\s+;\n" 184 " IDENT = [\\p{XID_Start}][\\p{XID_Continue}]*;\n" 185 " %keywords IDENT { IF = \"if\"; LET = \"let\"; CAFE = \"caf\xc3\xa9\"; LAMBDA = \"\xce\xbb\"; }\n" 186 "}\n" 187 "p = IF | LET | CAFE | LAMBDA | IDENT;\n"; 188 KitGramCompiled *c = compile(G); 189 if (!c) { ck("utf8 grammar compiles", 0); return; } 190 const KitGramLexGrammar *g = kit_gram_lexer_grammar(c); 191 ck("utf8 input mode", g->input == KIT_GRAM_LEX_INPUT_UTF8); 192 ck("utf8 keyword table built", g->nkeyword_tables == 1); 193 KitGramTokenKind t_if = 0, t_let = 0, t_cafe = 0, t_lambda = 0, t_id = 0; 194 kit_gram_find_token(c, "IF", &t_if); 195 kit_gram_find_token(c, "LET", &t_let); 196 kit_gram_find_token(c, "CAFE", &t_cafe); 197 kit_gram_find_token(c, "LAMBDA", &t_lambda); 198 kit_gram_find_token(c, "IDENT", &t_id); 199 KitGramTokenKind a[16]; 200 size_t n = tokenize(g, "if iffy let caf\xc3\xa9 caf\xc3\xa9s \xce\xbb \xce\xbbx", a, 16); 201 int ok = n == 7 && a[0] == t_if && a[1] == t_id && a[2] == t_let && 202 a[3] == t_cafe && a[4] == t_id && a[5] == t_lambda && a[6] == t_id; 203 ck("utf8 keywords/idents tokenize (incl. multi-byte café, λ)", ok); 204 kit_gram_free(c); 205 } 206 #endif /* !KIT_GRAM_NO_UNICODE */ 207 208 int main(void) { 209 test_generated_lexer(); 210 test_parser_literal(); 211 test_match_api(); 212 #ifndef KIT_GRAM_NO_UNICODE 213 test_utf8(); 214 #endif 215 printf(failures ? "\nKEYWORD FAILURES=%d\n" : "\nKEYWORD ALL OK\n", failures); 216 return failures ? 1 : 0; 217 }