utf8_runtime_test.c (10742B)
1 #include "generated_utf8_runtime.h" 2 #include <kit/support/gram_lex_tables.h> 3 extern const KitGramLexGrammar utf8_runtime_lex_grammar; 4 5 #include <stdio.h> 6 #include <string.h> 7 8 static int failures = 0; 9 10 typedef struct { 11 KitGramTokenKind kind; 12 const unsigned char *lexeme; 13 size_t len; 14 uint32_t line, col; 15 } WantTok; 16 17 static void check_status(const char *what, int ok) { 18 printf("%s %s\n", ok ? "ok " : "FAIL", what); 19 if (!ok) failures++; 20 } 21 22 static uint16_t table_step(const KitGramLexGrammar *g, uint16_t state, unsigned char byte) { 23 if (state >= g->nstates) return KIT_GRAM_LEX_DEAD; 24 uint8_t cls = g->class_of[byte]; 25 if (cls >= g->nclasses) return KIT_GRAM_LEX_DEAD; 26 uint16_t stride = g->class_stride ? g->class_stride : g->nclasses; 27 return g->trans[(size_t)state * stride + cls]; 28 } 29 30 static uint16_t table_run(const KitGramLexGrammar *g, const unsigned char *bytes, size_t len) { 31 uint16_t state = 0; 32 for (size_t i = 0; i < len; i++) { 33 state = table_step(g, state, bytes[i]); 34 if (state == KIT_GRAM_LEX_DEAD) return state; 35 } 36 return state; 37 } 38 39 static int table_accepts(const unsigned char *bytes, size_t len, KitGramTokenKind want) { 40 const KitGramLexGrammar *g = &utf8_runtime_lex_grammar; 41 uint16_t state = table_run(g, bytes, len); 42 if (state == KIT_GRAM_LEX_DEAD) return 0; 43 uint16_t acc = g->accept[state]; 44 return acc != KIT_GRAM_LEX_ACCEPT_NONE && acc < g->naccepts && 45 !g->accepts[acc].skip && g->accepts[acc].tok == want; 46 } 47 48 static void check_lowered_tables(void) { 49 printf("== utf8 lowered byte DFA ==\n"); 50 51 static const unsigned char ident[] = { 52 0xCE, 0xB1, 0xCC, 0x81, 0xCE, 0xB2 53 }; 54 static const unsigned char lambda[] = { 0xCE, 0xBB }; 55 static const unsigned char e9[] = { 0xC3, 0xA9 }; 56 static const unsigned char acute[] = { 0xCC, 0x81 }; 57 static const unsigned char copyright[] = { 0xC2, 0xA9 }; 58 static const unsigned char latin_a[] = { 'A' }; 59 static const unsigned char copyright_acute[] = { 0xC2, 0xA9, 0xCC, 0x81 }; 60 static const unsigned char emoji[] = { 0xF0, 0x9F, 0x99, 0x82 }; 61 static const unsigned char partial_alpha[] = { 0xCE }; 62 static const unsigned char invalid_alpha[] = { 0xCE, 0x28 }; 63 64 check_status("grammar is marked UTF-8", 65 utf8_runtime_lex_grammar.input == KIT_GRAM_LEX_INPUT_UTF8); 66 check_status("IDENT byte path accepts alpha acute beta", 67 table_accepts(ident, sizeof ident, UTF8_RUNTIME_TOK_IDENT)); 68 check_status("GREEK intersection accepts lambda", 69 table_accepts(lambda, sizeof lambda, UTF8_RUNTIME_TOK_GREEK)); 70 check_status("HEX_E9 treats hex escape as scalar U+00E9", 71 table_accepts(e9, sizeof e9, UTF8_RUNTIME_TOK_HEX_E9)); 72 check_status("MARK difference accepts combining acute", 73 table_accepts(acute, sizeof acute, UTF8_RUNTIME_TOK_MARK)); 74 check_status("RANGE accepts non-ASCII scalar", 75 table_accepts(copyright, sizeof copyright, UTF8_RUNTIME_TOK_RANGE)); 76 check_status("LATIN difference accepts ASCII letter", 77 table_accepts(latin_a, sizeof latin_a, UTF8_RUNTIME_TOK_LATIN)); 78 check_status("NONASCII_PAIR complement accepts copyright acute", 79 table_accepts(copyright_acute, sizeof copyright_acute, UTF8_RUNTIME_TOK_NONASCII_PAIR)); 80 check_status("EMOJI byte path accepts U+1F642", 81 table_accepts(emoji, sizeof emoji, UTF8_RUNTIME_TOK_EMOJI)); 82 83 uint16_t partial = table_run(&utf8_runtime_lex_grammar, 84 partial_alpha, sizeof partial_alpha); 85 check_status("partial UTF-8 prefix is a live DFA path", 86 partial != KIT_GRAM_LEX_DEAD && 87 utf8_runtime_lex_grammar.accept[partial] == KIT_GRAM_LEX_ACCEPT_NONE); 88 check_status("invalid UTF-8 continuation is not a DFA path", 89 table_run(&utf8_runtime_lex_grammar, 90 invalid_alpha, sizeof invalid_alpha) == KIT_GRAM_LEX_DEAD); 91 } 92 93 static void expect_token(KitGramToken got, WantTok want, size_t idx) { 94 char label[160]; 95 int lex_ok = got.len == want.len && 96 memcmp(got.lexeme, want.lexeme, want.len) == 0; 97 snprintf(label, sizeof label, "utf8 token[%zu] kind=%u len=%zu at %u:%u", 98 idx, (unsigned)got.kind, got.len, got.line, got.col); 99 check_status(label, got.kind == want.kind && lex_ok && 100 got.line == want.line && got.col == want.col); 101 } 102 103 static int push_one_byte_chunks(KitGramLexInput *in, KitGramLexInputSpan *spans, 104 const unsigned char *src, size_t len) { 105 for (size_t i = 0; i < len; i++) { 106 spans[i] = (KitGramLexInputSpan){ .bytes = src + i, .len = 1 }; 107 kit_gram_lex_input_push(in, &spans[i]); 108 } 109 return 1; 110 } 111 112 static void check_utf8_tokens(void) { 113 printf("\n== utf8 runtime tokens ==\n"); 114 115 static const unsigned char ident1[] = { 116 0xCE, 0xB1, 0xCC, 0x81, 0xCE, 0xB2 117 }; 118 static const unsigned char emoji[] = { 0xF0, 0x9F, 0x99, 0x82 }; 119 static const unsigned char greek[] = { 0xCE, 0xBB }; 120 static const unsigned char e9[] = { 0xC3, 0xA9 }; 121 static const unsigned char mark[] = { 0xCC, 0x81 }; 122 static const unsigned char range[] = { 0xC2, 0xA9 }; 123 static const unsigned char latin[] = { 'A' }; 124 static const unsigned char nonascii_pair[] = { 0xC2, 0xA9, 0xCC, 0x81 }; 125 static const unsigned char ident2[] = { 'f', 'o', 'o' }; 126 static const unsigned char src[] = { 127 0xCE, 0xB1, 0xCC, 0x81, 0xCE, 0xB2, 128 ' ', 129 0xF0, 0x9F, 0x99, 0x82, 130 0xE2, 0x80, 0xA8, 131 0xCE, 0xBB, 132 ' ', 133 0xC3, 0xA9, 134 ' ', 135 0xCC, 0x81, 136 ' ', 137 0xC2, 0xA9, 138 ' ', 139 'A', 140 ' ', 141 0xC2, 0xA9, 0xCC, 0x81, 142 ' ', 143 'f', 'o', 'o' 144 }; 145 const WantTok want[] = { 146 { UTF8_RUNTIME_TOK_IDENT, ident1, sizeof ident1, 1, 1 }, 147 { UTF8_RUNTIME_TOK_EMOJI, emoji, sizeof emoji, 1, 5 }, 148 { UTF8_RUNTIME_TOK_GREEK, greek, sizeof greek, 2, 1 }, 149 { UTF8_RUNTIME_TOK_HEX_E9, e9, sizeof e9, 2, 3 }, 150 { UTF8_RUNTIME_TOK_MARK, mark, sizeof mark, 2, 5 }, 151 { UTF8_RUNTIME_TOK_RANGE, range, sizeof range, 2, 7 }, 152 { UTF8_RUNTIME_TOK_LATIN, latin, sizeof latin, 2, 9 }, 153 { UTF8_RUNTIME_TOK_NONASCII_PAIR, nonascii_pair, sizeof nonascii_pair, 2, 11 }, 154 { UTF8_RUNTIME_TOK_IDENT, ident2, sizeof ident2, 2, 14 }, 155 }; 156 157 unsigned char carry[64]; 158 KitGramLexInput in; 159 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){ .carry = carry, .carry_cap = sizeof carry }); 160 KitGramLexer lx; 161 KitGramLexConfig cfg = {0}; 162 utf8_runtime_lexer_init(&lx, &in, &cfg); 163 164 KitGramLexInputSpan spans[sizeof src]; 165 check_status("utf8 input pushed in one-byte chunks", 166 push_one_byte_chunks(&in, spans, src, sizeof src)); 167 kit_gram_lex_input_finish(&in); 168 169 for (size_t i = 0; i < sizeof want / sizeof want[0]; i++) { 170 KitGramToken tok; 171 check_status("utf8 token available", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_TOKEN); 172 expect_token(tok, want[i], i); 173 } 174 175 KitGramToken tok; 176 check_status("utf8 lexer EOF", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_EOF); 177 } 178 179 static void check_parser_handoff(void) { 180 printf("\n== utf8 parser handoff ==\n"); 181 182 static const unsigned char src[] = { 183 0xCE, 0xB1, 0xCC, 0x81, 0xCE, 0xB2, 184 ' ', 185 0xF0, 0x9F, 0x99, 0x82, 186 0xE2, 0x80, 0xA8, 187 0xCE, 0xBB, 188 ' ', 189 0xC3, 0xA9, 190 ' ', 191 0xCC, 0x81, 192 ' ', 193 0xC2, 0xA9, 194 ' ', 195 'A', 196 ' ', 197 0xC2, 0xA9, 0xCC, 0x81, 198 ' ', 199 'f', 'o', 'o' 200 }; 201 202 unsigned char carry[64]; 203 KitGramLexInput in; 204 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){ .carry = carry, .carry_cap = sizeof carry }); 205 KitGramLexer lx; 206 KitGramLexConfig lcfg = {0}; 207 utf8_runtime_lexer_init(&lx, &in, &lcfg); 208 209 KitGramParser ps; 210 KitGramSlot ctl[64]; 211 KitGramSem vals[64]; 212 KitGramConfig pcfg = { 213 .ctl_stack = ctl, .ctl_cap = 64, 214 .val_stack = vals, .val_cap = 64, 215 }; 216 utf8_runtime_parser_init(&ps, &pcfg); 217 218 KitGramLexInputSpan spans[sizeof src]; 219 check_status("utf8 parse input pushed in one-byte chunks", 220 push_one_byte_chunks(&in, spans, src, sizeof src)); 221 kit_gram_lex_input_finish(&in); 222 223 for (;;) { 224 KitGramToken tok; 225 KitGramLexStatus st = kit_gram_lexer_next(&lx, &tok); 226 if (st == KIT_GRAM_LEX_TOKEN) { 227 check_status("utf8 parser accepted token", 228 kit_gram_parser_push(&ps, tok) != KIT_GRAM_PARSE_ERROR); 229 continue; 230 } 231 check_status("utf8 lexer finished cleanly", st == KIT_GRAM_LEX_EOF); 232 break; 233 } 234 235 check_status("utf8 parser accepted stream", 236 kit_gram_parser_finish(&ps) == KIT_GRAM_PARSE_ACCEPT); 237 } 238 239 static void expect_error(const char *what, const unsigned char *src, size_t len, 240 const char *message, unsigned char byte) { 241 KitGramLexInput in; 242 kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0}); 243 KitGramLexer lx; 244 KitGramLexConfig cfg = {0}; 245 utf8_runtime_lexer_init(&lx, &in, &cfg); 246 247 KitGramLexInputSpan span = { .bytes = src, .len = len }; 248 kit_gram_lex_input_push(&in, &span); 249 kit_gram_lex_input_finish(&in); 250 251 KitGramToken tok; 252 KitGramLexStatus st = kit_gram_lexer_next(&lx, &tok); 253 const KitGramLexError *err = kit_gram_lex_input_error(&in); 254 char label[120]; 255 snprintf(label, sizeof label, "%s reports %s", what, message); 256 check_status(label, st == KIT_GRAM_LEX_ERROR && 257 strcmp(err->message, message) == 0 && 258 err->line == 1 && err->col == 1 && 259 err->byte == byte); 260 } 261 262 static void check_utf8_errors(void) { 263 printf("\n== utf8 runtime errors ==\n"); 264 265 static const unsigned char bad_cont[] = { 0xCE, 0x28 }; 266 static const unsigned char partial[] = { 0xCE }; 267 static const unsigned char valid_miss[] = { '#' }; 268 269 expect_error("bad continuation", bad_cont, sizeof bad_cont, 270 "invalid UTF-8", 0xCE); 271 expect_error("partial code point at EOF", partial, sizeof partial, 272 "incomplete UTF-8", 0xCE); 273 expect_error("valid scalar outside token language", valid_miss, sizeof valid_miss, 274 "invalid token", '#'); 275 } 276 277 int main(void) { 278 check_lowered_tables(); 279 check_utf8_tokens(); 280 check_parser_handoff(); 281 check_utf8_errors(); 282 return failures ? 1 : 0; 283 }