test_json_frontend.c (15486B)
1 /* test_json_frontend.c — an end-to-end JSON front end: parser semantic actions 2 * that invoke sub-lexers to decode the leaf tokens, the way a real language 3 * frontend does. This is the composition the other tests cover only in 4 * isolation — parser actions (test_rjson) and sub-lexers (test_sublexer*) — wired 5 * together through one in-process compile. 6 * 7 * Pipeline (grammar: test/realistic/json_frontend.ebnf): 8 * - the main lexer yields whole STRING / NUMBER tokens; 9 * - the parser reduces the value tree through the value channel; 10 * - in that channel, a NUMBER lexeme is re-scanned by the `number` sub-lexer 11 * (classify int vs float from its NFRAC/NEXP parts, read the integer value 12 * from NINT/NSIGN), and a STRING lexeme by the `strescape` sub-lexer 13 * (unescape SESC atoms into raw UTF-8). 14 * The sub-lexers are reached purely through the in-process introspection API 15 * (kit_gram_find_lexer / kit_gram_lexer_grammar_at), so this also exercises that API 16 * on a realistic grammar. The result is a decoded canonical rendering compared 17 * to an expected string. 18 */ 19 #include <kit/gram.h> 20 #include <kit/gram_lex.h> 21 #include <kit/gram_parse.h> 22 23 #include <stdarg.h> 24 #include <stdio.h> 25 #include <stdlib.h> 26 #include <string.h> 27 28 #include "gram_test.h" 29 30 static int failures = 0; 31 static void okfail(int ok, const char *fmt, ...) { 32 va_list ap; va_start(ap, fmt); 33 char buf[512]; vsnprintf(buf, sizeof buf, fmt, ap); va_end(ap); 34 printf("%s %s\n", ok ? "ok " : "FAIL", buf); 35 if (!ok) failures++; 36 } 37 38 /* ------------------------------------------------------------------ arena -- */ 39 typedef struct { char **mem; size_t n, cap; } Bag; 40 static char *areg(Bag *a, char *s) { 41 if (a->n == a->cap) { a->cap = a->cap ? a->cap * 2 : 64; a->mem = realloc(a->mem, a->cap * sizeof *a->mem); } 42 a->mem[a->n++] = s; return s; 43 } 44 static char *adup(Bag *a, const char *s, size_t len) { 45 char *p = malloc(len + 1); memcpy(p, s, len); p[len] = 0; return areg(a, p); 46 } 47 static char *afmt(Bag *a, const char *fmt, ...) { 48 va_list ap; va_start(ap, fmt); 49 char buf[1024]; vsnprintf(buf, sizeof buf, fmt, ap); va_end(ap); 50 return adup(a, buf, strlen(buf)); 51 } 52 static void bag_free(Bag *a) { for (size_t i = 0; i < a->n; i++) free(a->mem[i]); free(a->mem); a->mem = 0; a->n = a->cap = 0; } 53 54 /* The frontend context threaded through the value channel as `ud`: the arena for 55 * results plus the two sub-lexer grammars and the token/rule ids resolved once. */ 56 typedef struct { 57 Bag *bag; 58 const KitGramLexGrammar *gnum, *gstr; 59 KitGramTokenKind STRING, NUMBER, NSIGN, NINT, NFRAC, NEXP, SQUOTE, SESC, SCHARS; 60 KitGramRuleId R_value, R_member, R_member_tail, R_members, R_object, R_array, 61 R_elements, R_element_tail; 62 } FE; 63 64 /* ---- NUMBER decode: drive the `number` sub-lexer over the lexeme ---------- */ 65 /* Classify int vs float from the presence of NFRAC/NEXP, and for an integer read 66 * the value straight from the NINT digits (with the NSIGN part). Returns a freshly 67 * formatted decimal ("42", "-7") for integers, or "%g" of the value for floats. */ 68 static char *decode_number(FE *fe, const char *bytes, size_t len) { 69 KitGramLexInput in; kit_gram_lex_input_init(&in, NULL); 70 KitGramLexInputSpan span = { .bytes = (const unsigned char *)bytes, .len = len }; 71 kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 72 KitGramLexer lx; kit_gram_lexer_init(&lx, fe->gnum, &in, &(KitGramLexConfig){0}); 73 74 int neg = 0, has_frac = 0, has_exp = 0; 75 long long int_val = 0; 76 KitGramToken t; KitGramLexStatus st; 77 while ((st = kit_gram_lexer_next(&lx, &t)) == KIT_GRAM_LEX_TOKEN) { 78 if (t.kind == fe->NSIGN) neg = 1; 79 else if (t.kind == fe->NINT) { 80 char b[32]; size_t k = t.len < sizeof b - 1 ? t.len : sizeof b - 1; 81 memcpy(b, t.lexeme, k); b[k] = 0; int_val = atoll(b); 82 } else if (t.kind == fe->NFRAC) has_frac = 1; 83 else if (t.kind == fe->NEXP) has_exp = 1; 84 } 85 if (st != KIT_GRAM_LEX_EOF) return afmt(fe->bag, "<numerr>"); 86 if (!has_frac && !has_exp) 87 return afmt(fe->bag, "%lld", neg ? -int_val : int_val); 88 /* float: the sub-lexer confirmed the shape; read the actual value with strtod */ 89 char tmp[64]; size_t k = len < sizeof tmp - 1 ? len : sizeof tmp - 1; 90 memcpy(tmp, bytes, k); tmp[k] = 0; 91 return afmt(fe->bag, "%g", strtod(tmp, NULL)); 92 } 93 94 /* ---- STRING decode: drive the `strescape` sub-lexer over the lexeme ------- */ 95 static int hexval(int c) { 96 if (c >= '0' && c <= '9') return c - '0'; 97 if (c >= 'a' && c <= 'f') return c - 'a' + 10; 98 if (c >= 'A' && c <= 'F') return c - 'A' + 10; 99 return 0; 100 } 101 static size_t decode_escape(const char *e, char *out) { 102 switch (e[1]) { 103 case '"': out[0] = '"'; return 1; 104 case '\\': out[0] = '\\'; return 1; 105 case '/': out[0] = '/'; return 1; 106 case 'b': out[0] = '\b'; return 1; 107 case 'f': out[0] = '\f'; return 1; 108 case 'n': out[0] = '\n'; return 1; 109 case 'r': out[0] = '\r'; return 1; 110 case 't': out[0] = '\t'; return 1; 111 case 'u': { 112 unsigned cp = 0; 113 for (int i = 0; i < 4; i++) cp = cp * 16 + (unsigned)hexval((unsigned char)e[2 + i]); 114 if (cp < 0x80u) { out[0] = (char)cp; return 1; } 115 if (cp < 0x800u) { 116 out[0] = (char)(0xC0u | (cp >> 6)); out[1] = (char)(0x80u | (cp & 0x3Fu)); return 2; 117 } 118 out[0] = (char)(0xE0u | (cp >> 12)); 119 out[1] = (char)(0x80u | ((cp >> 6) & 0x3Fu)); 120 out[2] = (char)(0x80u | (cp & 0x3Fu)); 121 return 3; 122 } 123 default: return 0; 124 } 125 } 126 /* Returns the unescaped raw bytes (no surrounding quotes) as an arena string. */ 127 static char *decode_string(FE *fe, const char *bytes, size_t len) { 128 KitGramLexInput in; kit_gram_lex_input_init(&in, NULL); 129 KitGramLexInputSpan span = { .bytes = (const unsigned char *)bytes, .len = len }; 130 kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 131 KitGramLexer lx; kit_gram_lexer_init(&lx, fe->gstr, &in, &(KitGramLexConfig){0}); 132 133 char *out = malloc(len + 1); size_t on = 0; 134 KitGramToken t; KitGramLexStatus st; 135 while ((st = kit_gram_lexer_next(&lx, &t)) == KIT_GRAM_LEX_TOKEN) { 136 if (t.kind == fe->SCHARS) { memcpy(out + on, t.lexeme, t.len); on += t.len; } 137 else if (t.kind == fe->SESC) on += decode_escape((const char *)t.lexeme, out + on); 138 /* SQUOTE: drop */ 139 } 140 out[on] = 0; 141 return areg(fe->bag, out); 142 } 143 144 /* ----------------------------------------------------------- value channel -- */ 145 static KitGramSem fe_lift(void *ud, KitGramToken t) { 146 FE *fe = ud; 147 if (t.kind == fe->STRING) return decode_string(fe, t.lexeme, t.len); 148 if (t.kind == fe->NUMBER) return decode_number(fe, t.lexeme, t.len); 149 return adup(fe->bag, (const char *)t.lexeme, t.len); /* true / false / null */ 150 } 151 static KitGramSem fe_list_empty(void *ud) { return adup(((FE *)ud)->bag, "", 0); } 152 static KitGramSem fe_list_push(void *ud, KitGramSem list, KitGramSem item) { 153 FE *fe = ud; const char *l = list; 154 return *l ? afmt(fe->bag, "%s,%s", l, (const char *)item) : (char *)item; 155 } 156 static KitGramSem fe_opt_none(void *ud) { return adup(((FE *)ud)->bag, "", 0); } 157 static KitGramSem fe_opt_some(void *ud, KitGramSem item) { (void)ud; return item; } 158 159 static KitGramSem fe_reduce(void *ud, KitGramRuleId r, int prod, KitGramSem *k, size_t n) { 160 FE *fe = ud; Bag *a = fe->bag; (void)prod; (void)n; 161 if (r == fe->R_value) return k[0]; 162 if (r == fe->R_member) return afmt(a, "%s:%s", (char *)k[0], (char *)k[2]); /* key:value */ 163 if (r == fe->R_member_tail) return k[1]; 164 if (r == fe->R_members) return *(char *)k[1] ? afmt(a, "%s,%s", (char *)k[0], (char *)k[1]) : (char *)k[0]; 165 if (r == fe->R_object) return afmt(a, "{%s}", (char *)k[1]); 166 if (r == fe->R_array) return afmt(a, "[%s]", (char *)k[1]); 167 if (r == fe->R_elements) return *(char *)k[1] ? afmt(a, "%s,%s", (char *)k[0], (char *)k[1]) : (char *)k[0]; 168 if (r == fe->R_element_tail) return k[1]; 169 return NULL; 170 } 171 172 /* ----------------------------------------------------------------- driver --- */ 173 /* Parse `src` and return the decoded canonical rendering, or NULL on reject. */ 174 static const char *parse_decode(KitGramCompiled *c, FE *fe, const char *src) { 175 KitGramActions acts = { 176 .reduce = fe_reduce, .lift_token = fe_lift, 177 .list_empty = fe_list_empty, .list_push = fe_list_push, 178 .opt_none = fe_opt_none, .opt_some = fe_opt_some, 179 }; 180 const KitGramGrammar *g = kit_gram_parser_grammar(c); 181 const KitGramLexGrammar *lg = kit_gram_lexer_grammar(c); 182 183 KitGramLexInput in; kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0}); 184 KitGramLexer lx; kit_gram_lexer_init(&lx, lg, &in, &(KitGramLexConfig){0}); 185 size_t cc = 0, vc = 0; 186 kit_gram_stack_bounds(g, 128, &cc, &vc); 187 size_t cap = cc > vc ? cc : vc; 188 KitGramSlot *ctl = malloc(cap * sizeof *ctl); 189 KitGramSem *val = malloc(cap * sizeof *val); 190 KitGramParser ps; 191 kit_gram_parser_init(&ps, g, &(KitGramConfig){ .actions = &acts, .ud = fe, 192 .ctl_stack = ctl, .ctl_cap = cap, .val_stack = val, .val_cap = cap }); 193 194 KitGramLexInputSpan span = { .bytes = (const unsigned char *)src, .len = strlen(src) }; 195 kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 196 197 const char *out = NULL; 198 for (;;) { 199 KitGramToken tok; KitGramLexStatus st = kit_gram_lexer_next(&lx, &tok); 200 if (st == KIT_GRAM_LEX_TOKEN) { if (kit_gram_parser_push(&ps, tok) == KIT_GRAM_PARSE_ERROR) break; continue; } 201 if (st != KIT_GRAM_LEX_EOF) break; 202 if (kit_gram_parser_finish(&ps) == KIT_GRAM_PARSE_ACCEPT) out = kit_gram_parser_result(&ps); 203 break; 204 } 205 free(ctl); free(val); 206 return out; 207 } 208 209 /* Regression for literal scoping: the parser's inline literals ("[", "{", ":", 210 * "true", ...) must stay in the parser-feeding main lexer and NOT leak into the 211 * strescape sub-lexer. If they leaked, the catch-all SCHARS = [^"\\]+ would be a 212 * compile-time shadow conflict; even past that, the sub-lexer would split "[a]" 213 * into "[" / "a" / "]" instead of one SCHARS run. So: the main lexer tokenizes a 214 * bare "[" as a one-byte structural token, while the strescape sub-lexer scans 215 * "[a]{}:," as a single SCHARS run covering every byte. */ 216 static void test_literal_scoping(KitGramCompiled *c, FE *fe) { 217 KitGramMatcher mmain; kit_gram_matcher_bind(&mmain, kit_gram_lexer_grammar(c)); 218 KitGramMatcher mstr; kit_gram_matcher_bind(&mstr, fe->gstr); 219 KitGramLexInput in; KitGramLexInputSpan span; KitGramMatch m; 220 221 span = (KitGramLexInputSpan){ .bytes = (const unsigned char *)"[", .len = 1 }; 222 kit_gram_lex_input_init(&in, NULL); kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 223 int main_ok = kit_gram_match_anchored(&mmain, &in, NULL, &m) && m.start == 0 && m.end == 1 && 224 m.kind != fe->SCHARS && m.kind != fe->STRING; 225 okfail(main_ok, "main lexer: \"[\" is a structural token (kind=%u)", (unsigned)m.kind); 226 227 const char *s = "[a]{}:,"; 228 span = (KitGramLexInputSpan){ .bytes = (const unsigned char *)s, .len = strlen(s) }; 229 kit_gram_lex_input_init(&in, NULL); kit_gram_lex_input_push(&in, &span); kit_gram_lex_input_finish(&in); 230 int sub_ok = kit_gram_match_find(&mstr, &in, NULL, &m) && m.kind == fe->SCHARS && 231 m.start == 0 && m.end == strlen(s); 232 okfail(sub_ok, "strescape sub-lexer: \"%s\" is ONE SCHARS run (literals not leaked)", s); 233 } 234 235 static void check(KitGramCompiled *c, FE *fe, const char *src, const char *want) { 236 Bag bag = {0}; 237 fe->bag = &bag; 238 const char *got = parse_decode(c, fe, src); 239 int ok = got && strcmp(got, want) == 0; 240 /* The TAB in some expectations would be invisible in the log; render it. */ 241 char shown[256]; size_t si = 0; 242 for (const char *p = got ? got : "(reject)"; *p && si < sizeof shown - 3; p++) { 243 if (*p == '\t') { shown[si++] = '\\'; shown[si++] = 't'; } 244 else shown[si++] = *p; 245 } 246 shown[si] = 0; 247 okfail(ok, "%-40s -> %s", src, shown); 248 bag_free(&bag); 249 } 250 251 int main(void) { 252 static char SRC[1 << 14]; 253 FILE *f = fopen("test/gram/realistic/json_frontend.ebnf", "rb"); 254 if (!f) { fprintf(stderr, "cannot open json_frontend.ebnf\n"); return 2; } 255 size_t n = fread(SRC, 1, sizeof SRC - 1, f); fclose(f); SRC[n] = 0; 256 257 KitGramOptions opts = {0}; 258 KitGramCompiled *c = NULL; 259 KitStatus st = kit_gram_compile_text(gram_test_ctx(), (KitSlice){.s=SRC,.len=n}, 260 KIT_SLICE_LIT("json_frontend"), &opts, &c); 261 okfail(st == KIT_OK && c != NULL, "compile json_frontend.ebnf in-process"); 262 if (st != KIT_OK || !c) return 1; 263 264 /* Reach the sub-lexers purely through the in-process introspection API. */ 265 size_t inum = 0, istr = 0; 266 okfail(kit_gram_find_lexer(c, "number", &inum) && kit_gram_find_lexer(c, "strescape", &istr), 267 "find number + strescape sub-lexers via the in-process API"); 268 269 FE fe = {0}; 270 fe.gnum = kit_gram_lexer_grammar_at(c, inum); 271 fe.gstr = kit_gram_lexer_grammar_at(c, istr); 272 kit_gram_find_token(c, "STRING", &fe.STRING); 273 kit_gram_find_token(c, "NUMBER", &fe.NUMBER); 274 kit_gram_find_token(c, "NSIGN", &fe.NSIGN); 275 kit_gram_find_token(c, "NINT", &fe.NINT); 276 kit_gram_find_token(c, "NFRAC", &fe.NFRAC); 277 kit_gram_find_token(c, "NEXP", &fe.NEXP); 278 kit_gram_find_token(c, "SQUOTE", &fe.SQUOTE); 279 kit_gram_find_token(c, "SESC", &fe.SESC); 280 kit_gram_find_token(c, "SCHARS", &fe.SCHARS); 281 kit_gram_find_rule(c, "value", &fe.R_value); 282 kit_gram_find_rule(c, "member", &fe.R_member); 283 kit_gram_find_rule(c, "member_tail", &fe.R_member_tail); 284 kit_gram_find_rule(c, "members", &fe.R_members); 285 kit_gram_find_rule(c, "object", &fe.R_object); 286 kit_gram_find_rule(c, "array", &fe.R_array); 287 kit_gram_find_rule(c, "elements", &fe.R_elements); 288 kit_gram_find_rule(c, "element_tail", &fe.R_element_tail); 289 290 printf("== literal scoping: parser literals stay out of the sub-lexer ==\n"); 291 test_literal_scoping(c, &fe); 292 293 printf("\n== NUMBER sub-lexer decode (int vs float classification + value) ==\n"); 294 check(c, &fe, "0", "0"); 295 check(c, &fe, "42", "42"); 296 check(c, &fe, "-7", "-7"); 297 check(c, &fe, "3.14", "3.14"); /* NFRAC present -> float */ 298 check(c, &fe, "-3.14e2", "-314"); /* NEXP present -> float, value -314 */ 299 check(c, &fe, "1e3", "1000"); /* NEXP present -> float */ 300 check(c, &fe, "100", "100"); /* plain integer */ 301 302 printf("\n== STRING sub-lexer decode (unescape into raw UTF-8) ==\n"); 303 check(c, &fe, "\"plain\"", "plain"); 304 check(c, &fe, "\"a\\tb\"", "a\tb"); /* \t -> real TAB */ 305 check(c, &fe, "\"x\\u00e9y\"", "x\xc3\xa9y"); /* é -> U+00E9 UTF-8 */ 306 check(c, &fe, "\"a\\\"b\"", "a\"b"); /* \" -> quote */ 307 308 printf("\n== end-to-end documents: parser actions drive both sub-lexers ==\n"); 309 check(c, &fe, "[1, -3.14e2, \"a\\tb\", true, null]", "[1,-314,a\tb,true,null]"); 310 check(c, &fe, "{\"k\\u0041\": 7, \"s\": \"x\\u00e9y\"}", "{kA:7,s:x\xc3\xa9y}"); 311 check(c, &fe, "{\"nums\": [0, 42, -7], \"f\": 1e3}", "{nums:[0,42,-7],f:1000}"); 312 check(c, &fe, "[]", "[]"); 313 check(c, &fe, "{}", "{}"); 314 315 kit_gram_free(c); 316 printf("\n%d failures\n", failures); 317 return failures ? 1 : 0; 318 }