gramgen_api_test.c (9841B)
1 #include <kit/gram.h> 2 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 7 #include "gram_test.h" 8 9 static int parse_source(KitGramCompiled *cg, const char *src) { 10 const KitGramLexGrammar *lg = kit_gram_lexer_grammar(cg); 11 const KitGramGrammar *pg = kit_gram_parser_grammar(cg); 12 if (!lg || !pg) return 0; 13 14 unsigned char carry[64]; 15 KitGramLexInput in; 16 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){ .carry = carry, .carry_cap = sizeof carry }); 17 KitGramLexer lx; 18 KitGramLexConfig lcfg = {0}; 19 kit_gram_lexer_init(&lx, lg, &in, &lcfg); 20 21 KitGramParser ps; 22 KitGramSlot ctl[128]; 23 KitGramSem vals[128]; 24 KitGramConfig pcfg = { .ctl_stack = ctl, .ctl_cap = 128, 25 .val_stack = vals, .val_cap = 128 }; 26 kit_gram_parser_init(&ps, pg, &pcfg); 27 28 size_t off = 0; 29 size_t nspans = 0; 30 KitGramLexInputSpan spans[256]; 31 int finished = 0; 32 for (;;) { 33 KitGramToken tok; 34 KitGramLexStatus ls = kit_gram_lexer_next(&lx, &tok); 35 if (ls == KIT_GRAM_LEX_TOKEN) { 36 if (kit_gram_parser_push(&ps, tok) == KIT_GRAM_PARSE_ERROR) return 0; 37 continue; 38 } 39 if (ls == KIT_GRAM_LEX_ERROR) return 0; 40 if (ls == KIT_GRAM_LEX_EOF) return kit_gram_parser_finish(&ps) == KIT_GRAM_PARSE_ACCEPT; 41 42 if (src[off]) { 43 if (nspans >= sizeof spans / sizeof spans[0]) return 0; 44 spans[nspans] = (KitGramLexInputSpan){ .bytes = (const unsigned char *)src + off, .len = 1 }; 45 kit_gram_lex_input_push(&in, &spans[nspans++]); 46 off++; 47 } else if (!finished) { 48 kit_gram_lex_input_finish(&in); 49 finished = 1; 50 } else { 51 return 0; 52 } 53 } 54 } 55 56 static int parse_tokens(KitGramCompiled *cg, const KitGramToken *tokens, size_t ntokens) { 57 const KitGramGrammar *pg = kit_gram_parser_grammar(cg); 58 if (!pg) return 0; 59 60 KitGramParser ps; 61 KitGramSlot ctl[512]; 62 KitGramSem vals[512]; 63 KitGramConfig pcfg = { .ctl_stack = ctl, .ctl_cap = 512, 64 .val_stack = vals, .val_cap = 512 }; 65 kit_gram_parser_init(&ps, pg, &pcfg); 66 67 for (size_t i = 0; i < ntokens; i++) 68 if (kit_gram_parser_push(&ps, tokens[i]) == KIT_GRAM_PARSE_ERROR) return 0; 69 return kit_gram_parser_finish(&ps) == KIT_GRAM_PARSE_ACCEPT; 70 } 71 72 static int generate_tokens(KitGramCompiled *cg, uint64_t seed, size_t max_depth, 73 KitGramToken *tokens, size_t cap, size_t *ntokens) { 74 const KitGramGrammar *pg = kit_gram_parser_grammar(cg); 75 if (!pg) return 0; 76 77 KitGramParser ps; 78 KitGramSlot ctl[512]; 79 KitGramSem vals[512]; 80 KitGramConfig pcfg = { .ctl_stack = ctl, .ctl_cap = 512, 81 .val_stack = vals, .val_cap = 512 }; 82 kit_gram_parser_init(&ps, pg, &pcfg); 83 84 size_t scratch[128]; 85 size_t need = kit_gram_parser_generate_scratch_count(pg); 86 if (need > sizeof scratch / sizeof scratch[0]) return 0; 87 KitGramGenConfig gcfg = { 88 .seed = seed, 89 .max_depth = max_depth, 90 .max_repeat = 3, 91 .max_tokens = cap, 92 .scratch = scratch, 93 .scratch_cap = need, 94 }; 95 96 size_t n = 0; 97 for (;;) { 98 KitGramToken tok; 99 KitGramGenStatus st = kit_gram_parser_generate_next(&ps, &gcfg, &tok); 100 if (st == KIT_GRAM_GEN_TOKEN) { 101 if (n == cap) return 0; 102 tokens[n++] = tok; 103 continue; 104 } 105 if (st != KIT_GRAM_GEN_DONE) return 0; 106 *ntokens = n; 107 return 1; 108 } 109 } 110 111 static int generate_after_prefix(KitGramCompiled *cg) { 112 const KitGramGrammar *pg = kit_gram_parser_grammar(cg); 113 if (!pg) return 0; 114 KitGramTokenKind if_tok = 0; 115 if (!kit_gram_find_token(cg, "IF", &if_tok)) return 0; 116 117 KitGramParser ps; 118 KitGramSlot ctl[128]; 119 KitGramSem vals[128]; 120 KitGramConfig pcfg = { .ctl_stack = ctl, .ctl_cap = 128, 121 .val_stack = vals, .val_cap = 128 }; 122 kit_gram_parser_init(&ps, pg, &pcfg); 123 if (kit_gram_parser_push(&ps, (KitGramToken){ .kind = if_tok }) == KIT_GRAM_PARSE_ERROR) return 0; 124 125 size_t scratch[16]; 126 size_t need = kit_gram_parser_generate_scratch_count(pg); 127 if (need > sizeof scratch / sizeof scratch[0]) return 0; 128 KitGramGenConfig gcfg = { 129 .seed = 42, 130 .max_depth = 8, 131 .max_repeat = 3, 132 .max_tokens = 8, 133 .emitted = 1, 134 .scratch = scratch, 135 .scratch_cap = need, 136 }; 137 138 size_t generated = 0; 139 for (;;) { 140 KitGramToken tok; 141 KitGramGenStatus st = kit_gram_parser_generate_next(&ps, &gcfg, &tok); 142 if (st == KIT_GRAM_GEN_TOKEN) { 143 generated++; 144 continue; 145 } 146 return st == KIT_GRAM_GEN_DONE && generated == 2; 147 } 148 } 149 150 static int generate_rejects_small_scratch(KitGramCompiled *cg) { 151 const KitGramGrammar *pg = kit_gram_parser_grammar(cg); 152 if (!pg) return 0; 153 154 KitGramParser ps; 155 KitGramSlot ctl[128]; 156 KitGramSem vals[128]; 157 KitGramConfig pcfg = { .ctl_stack = ctl, .ctl_cap = 128, 158 .val_stack = vals, .val_cap = 128 }; 159 kit_gram_parser_init(&ps, pg, &pcfg); 160 161 size_t scratch[128]; 162 size_t need = kit_gram_parser_generate_scratch_count(pg); 163 if (need == 0 || need > sizeof scratch / sizeof scratch[0]) return 0; 164 KitGramGenConfig gcfg = { 165 .seed = 7, 166 .max_depth = 8, 167 .max_repeat = 3, 168 .max_tokens = 8, 169 .scratch = scratch, 170 .scratch_cap = need - 1, 171 }; 172 return kit_gram_parser_generate_next(&ps, &gcfg, NULL) == KIT_GRAM_GEN_ERROR; 173 } 174 175 static int generate_round_trips(KitGramCompiled *cg, size_t samples) { 176 KitGramToken tokens[96]; 177 for (size_t i = 0; i < samples; i++) { 178 size_t ntokens = 0; 179 if (!generate_tokens(cg, i, 8, tokens, sizeof tokens / sizeof tokens[0], &ntokens)) 180 return 0; 181 if (!parse_tokens(cg, tokens, ntokens)) return 0; 182 } 183 return 1; 184 } 185 186 int main(void) { 187 static const char grammar[] = 188 "%token IF = \"if\";\n" 189 "%lex {\n" 190 " %skip WS = [ \\t\\r\\n] [ \\t\\r\\n]*;\n" 191 " NAME = [A-Za-z_] [A-Za-z0-9_]*;\n" 192 "}\n" 193 "stmt = \"if\" NAME \";\";\n"; 194 195 KitGramOptions opts = { .name = "direct" }; 196 KitGramCompiled *cg = NULL; 197 KitStatus cst = kit_gram_compile_text(gram_test_ctx(), 198 (KitSlice){.s=grammar,.len=strlen(grammar)}, 199 KIT_SLICE_LIT("direct.ebnf"), &opts, &cg); 200 if (cst != KIT_OK || !cg) { 201 printf("FAIL compile (a diagnostic was printed to stderr)\n"); 202 return 1; 203 } 204 205 KitGramTokenKind if_tok = 0; 206 KitGramRuleId stmt_rule = 0; 207 int ok = 1; 208 ok &= kit_gram_find_token(cg, "IF", &if_tok); 209 ok &= kit_gram_find_rule(cg, "stmt", &stmt_rule); 210 ok &= if_tok != 0 && stmt_rule == 0; 211 ok &= parse_source(cg, "if alpha;"); 212 ok &= !parse_source(cg, "if ;"); 213 214 KitWriter *header_writer = NULL, *source_writer = NULL; 215 kit_writer_mem(gram_test_heap(), &header_writer); 216 kit_writer_mem(gram_test_heap(), &source_writer); 217 KitGramEmitOptions emit_opts = { .header_path = "direct.h", 218 .source_path = "direct.c", 219 .prefix = "direct_" }; 220 ok &= kit_gram_emit_c(cg, &emit_opts, header_writer, source_writer) == KIT_OK; 221 size_t hlen = 0, slen = 0; 222 const char *hbytes = (const char *)kit_writer_mem_bytes(header_writer, &hlen); 223 const char *sbytes = (const char *)kit_writer_mem_bytes(source_writer, &slen); 224 ok &= hbytes && memmem(hbytes, hlen, "direct_parser_init", strlen("direct_parser_init")) != NULL; 225 ok &= sbytes && memmem(sbytes, slen, "const KitGramGrammar direct_grammar", 226 strlen("const KitGramGrammar direct_grammar")) != NULL; 227 kit_writer_close(header_writer); 228 kit_writer_close(source_writer); 229 ok &= generate_round_trips(cg, 16); 230 ok &= generate_after_prefix(cg); 231 ok &= generate_rejects_small_scratch(cg); 232 233 static const char pratt_grammar[] = 234 "%pratt expr {\n" 235 " primary primary;\n" 236 " infixl \"+\" \"-\";\n" 237 " infixl \"*\" \"/\";\n" 238 " prefix \"-\" \"+\";\n" 239 " postfix \"!\";\n" 240 "}\n" 241 "primary = NUMBER | \"(\" expr \")\";\n"; 242 KitGramOptions popts = { .name = "synth_pratt" }; 243 KitGramCompiled *pcg = NULL; 244 ok &= kit_gram_compile_text(gram_test_ctx(), 245 (KitSlice){.s=pratt_grammar,.len=strlen(pratt_grammar)}, 246 KIT_SLICE_LIT("synth_pratt.ebnf"), &popts, &pcg) == KIT_OK; 247 ok &= pcg != NULL; 248 if (pcg) { 249 KitGramTokenKind plus_tok = 0, star_tok = 0, bang_tok = 0; 250 ok &= kit_gram_find_token(pcg, "PLUS", &plus_tok); 251 ok &= kit_gram_find_token(pcg, "STAR", &star_tok); 252 ok &= kit_gram_find_token(pcg, "BANG", &bang_tok); 253 int saw_pratt_op = 0; 254 KitGramToken tokens[96]; 255 for (size_t i = 0; i < 64; i++) { 256 size_t ntokens = 0; 257 ok &= generate_tokens(pcg, 1000 + i, 10, tokens, 258 sizeof tokens / sizeof tokens[0], &ntokens); 259 if (ntokens) { 260 ok &= parse_tokens(pcg, tokens, ntokens); 261 for (size_t ti = 0; ti < ntokens; ti++) 262 if (tokens[ti].kind == plus_tok || 263 tokens[ti].kind == star_tok || 264 tokens[ti].kind == bang_tok) 265 saw_pratt_op = 1; 266 } 267 } 268 ok &= saw_pratt_op; 269 kit_gram_free(pcg); 270 } 271 272 printf("%s gramgen direct API compile/lex/parse/codegen\n", ok ? "ok " : "FAIL"); 273 kit_gram_free(cg); 274 return ok ? 0 : 1; 275 }