test_sublexer.c (8955B)
1 /* Sub-lexer coverage for --lexer-standalone (grammar: sublexer.ebnf). 2 * 3 * The standalone generator now emits one resident tokenizer per %lex mode in a 4 * single file. This drives the realistic frontend pattern: the main lexer 5 * yields whole NUMBER / STRING tokens, then a dedicated sub-lexer re-scans each 6 * token's *bytes* into its parts — 7 * - NUMBER -> sign / integer / fraction / exponent (a value extractor), and 8 * - STRING -> quotes / literal runs / escape atoms (an unescaper). 9 * 10 * The sub-lexers are pointed straight at the parent token's lexeme/len: no 11 * shared cursor, no kit_gram_lex_stack (a table-runtime construct the standalone 12 * lexer deliberately does without). Built and linked with NO libgram. 13 */ 14 #include "sublexersa.h" 15 16 #include <stdio.h> 17 #include <stdlib.h> 18 #include <string.h> 19 20 static int failures = 0; 21 static int checks = 0; 22 static void check(int cond, const char *msg) { 23 checks++; 24 if (cond) { 25 printf("ok %s\n", msg); 26 } else { 27 printf("FAIL %s\n", msg); 28 failures++; 29 } 30 } 31 32 /* ---- NUMBER sub-lexer: split a numeric literal into its lexical parts ----- */ 33 typedef struct { 34 int neg; /* an NSIGN part was present */ 35 long int_part; /* value of the integer-digits (NINT) part */ 36 int has_frac; /* an NFRAC part was present */ 37 int has_exp; /* an NEXP part was present */ 38 char shape[64]; /* concatenated part lexemes (== whole token iff */ 39 /* the sub-lexer covered every byte) */ 40 } NumParts; 41 42 /* Returns 1 iff the sub-lexer ran to EOF having consumed every byte. */ 43 static int relex_number(const char *bytes, size_t len, NumParts *np) { 44 memset(np, 0, sizeof *np); 45 sublex_lex sub; 46 sublex_number_lex_init(&sub, (const unsigned char *)bytes, len); 47 size_t covered = 0, sp = 0; 48 KitGramToken t; 49 KitGramLexStatus st; 50 while ((st = sublex_number_lex_next(&sub, &t)) == KIT_GRAM_LEX_TOKEN) { 51 covered += t.len; 52 for (size_t i = 0; i < t.len && sp < sizeof np->shape - 1; i++) 53 np->shape[sp++] = t.lexeme[i]; 54 switch (t.kind) { 55 case SUBLEX_TOK_NSIGN: np->neg = 1; break; 56 case SUBLEX_TOK_NINT: { 57 char b[32]; 58 size_t n = t.len < sizeof b - 1 ? t.len : sizeof b - 1; 59 memcpy(b, t.lexeme, n); 60 b[n] = 0; 61 np->int_part = atol(b); 62 break; 63 } 64 case SUBLEX_TOK_NFRAC: np->has_frac = 1; break; 65 case SUBLEX_TOK_NEXP: np->has_exp = 1; break; 66 default: break; 67 } 68 } 69 np->shape[sp] = 0; 70 return st == KIT_GRAM_LEX_EOF && covered == len; 71 } 72 73 /* ---- STRING sub-lexer: unescape a quoted string into raw bytes ----------- */ 74 static int hexval(int c) { 75 if (c >= '0' && c <= '9') return c - '0'; 76 if (c >= 'a' && c <= 'f') return c - 'a' + 10; 77 if (c >= 'A' && c <= 'F') return c - 'A' + 10; 78 return 0; 79 } 80 81 /* Decode one SESC atom (e[0]=='\\') into out; returns bytes written. \uXXXX is 82 * decoded to its scalar and re-encoded as UTF-8, the way a real frontend would. */ 83 static size_t decode_escape(const char *e, char *out) { 84 switch (e[1]) { 85 case '"': out[0] = '"'; return 1; 86 case '\\': out[0] = '\\'; return 1; 87 case '/': out[0] = '/'; return 1; 88 case 'b': out[0] = '\b'; return 1; 89 case 'f': out[0] = '\f'; return 1; 90 case 'n': out[0] = '\n'; return 1; 91 case 'r': out[0] = '\r'; return 1; 92 case 't': out[0] = '\t'; return 1; 93 case 'u': { 94 unsigned cp = 0; 95 for (int i = 0; i < 4; i++) 96 cp = cp * 16 + (unsigned)hexval((unsigned char)e[2 + i]); 97 if (cp < 0x80u) { 98 out[0] = (char)cp; 99 return 1; 100 } 101 if (cp < 0x800u) { 102 out[0] = (char)(0xC0u | (cp >> 6)); 103 out[1] = (char)(0x80u | (cp & 0x3Fu)); 104 return 2; 105 } 106 out[0] = (char)(0xE0u | (cp >> 12)); 107 out[1] = (char)(0x80u | ((cp >> 6) & 0x3Fu)); 108 out[2] = (char)(0x80u | (cp & 0x3Fu)); 109 return 3; 110 } 111 default: return 0; 112 } 113 } 114 115 /* Returns 1 iff the sub-lexer covered every byte and saw exactly two quotes. */ 116 static int relex_string(const char *bytes, size_t len, char *out, size_t *outn) { 117 sublex_lex sub; 118 sublex_strescape_lex_init(&sub, (const unsigned char *)bytes, len); 119 size_t covered = 0, on = 0; 120 int quotes = 0; 121 KitGramToken t; 122 KitGramLexStatus st; 123 while ((st = sublex_strescape_lex_next(&sub, &t)) == KIT_GRAM_LEX_TOKEN) { 124 covered += t.len; 125 switch (t.kind) { 126 case SUBLEX_TOK_SQUOTE: quotes++; break; 127 case SUBLEX_TOK_SCHARS: 128 memcpy(out + on, t.lexeme, t.len); 129 on += t.len; 130 break; 131 case SUBLEX_TOK_SESC: on += decode_escape(t.lexeme, out + on); break; 132 default: break; 133 } 134 } 135 out[on] = 0; 136 *outn = on; 137 return st == KIT_GRAM_LEX_EOF && covered == len && quotes == 2; 138 } 139 140 /* ---- per-lexer match API: each %lex mode exposes its own match/iter fns ---- */ 141 static void test_match_api(void) { 142 KitGramMatch m; 143 144 /* main lexer: full-match a bare NUMBER (no WS, no sub-lexer involved). */ 145 check(sublex_match_full((const unsigned char *)"42", 2, NULL, &m) && 146 m.kind == SUBLEX_TOK_NUMBER && m.start == 0 && m.end == 2, 147 "main match_full: 42 -> NUMBER [0,2)"); 148 149 /* number sub-lexer: find the integer run inside a literal. */ 150 check(sublex_number_match_find((const unsigned char *)"42", 2, NULL, &m) && 151 m.kind == SUBLEX_TOK_NINT && m.start == 0 && m.end == 2, 152 "number match_find: 42 -> NINT [0,2)"); 153 154 /* number sub-lexer: iterate every non-overlapping part of -3.14e-2. */ 155 const unsigned char *full = (const unsigned char *)"-3.14e-2"; 156 sublex_number_match_iter it; 157 sublex_number_match_iter_init(&it, full, 8, NULL); 158 KitGramTokenKind parts[8]; 159 int np = 0; 160 while (sublex_number_match_iter_next(&it, &m) && np < 8) parts[np++] = m.kind; 161 check(np == 4 && parts[0] == SUBLEX_TOK_NSIGN && parts[1] == SUBLEX_TOK_NINT && 162 parts[2] == SUBLEX_TOK_NFRAC && parts[3] == SUBLEX_TOK_NEXP, 163 "number match_iter: -3.14e-2 -> NSIGN NINT NFRAC NEXP"); 164 165 /* strescape sub-lexer: anchored match of the opening quote. */ 166 check(sublex_strescape_match_anchored((const unsigned char *)"\"hi\"", 4, 167 NULL, &m) && 168 m.kind == SUBLEX_TOK_SQUOTE && m.start == 0 && m.end == 1, 169 "strescape match_anchored: leading quote -> SQUOTE [0,1)"); 170 } 171 172 int main(void) { 173 test_match_api(); 174 175 /* -3.14e-2 42 0 "a\tb" "xéy" "plain" */ 176 const char *src = 177 " -3.14e-2 42 0 \"a\\tb\" \"x\\u00e9y\" \"plain\" "; 178 179 sublex_lex lx; 180 sublex_lex_init(&lx, (const unsigned char *)src, strlen(src)); 181 182 int n_num = 0, n_str = 0; 183 KitGramToken tok; 184 KitGramLexStatus st; 185 while ((st = sublex_lex_next(&lx, &tok)) == KIT_GRAM_LEX_TOKEN) { 186 if (tok.kind == SUBLEX_TOK_NUMBER) { 187 NumParts np; 188 int ok = relex_number(tok.lexeme, tok.len, &np); 189 ok = ok && strlen(np.shape) == tok.len && 190 memcmp(np.shape, tok.lexeme, tok.len) == 0; 191 if (n_num == 0) 192 check(ok && np.neg && np.int_part == 3 && np.has_frac && 193 np.has_exp, 194 "number sublex: -3.14e-2 -> -, int 3, frac, exp (full cover)"); 195 else if (n_num == 1) 196 check(ok && !np.neg && np.int_part == 42 && !np.has_frac && 197 !np.has_exp, 198 "number sublex: 42 -> int 42 only"); 199 else if (n_num == 2) 200 check(ok && !np.neg && np.int_part == 0 && !np.has_frac && 201 !np.has_exp, 202 "number sublex: 0 -> int 0 only"); 203 n_num++; 204 } else if (tok.kind == SUBLEX_TOK_STRING) { 205 char out[256]; 206 size_t on = 0; 207 int ok = relex_string(tok.lexeme, tok.len, out, &on); 208 if (n_str == 0) 209 check(ok && on == 3 && memcmp(out, "a\tb", 3) == 0, 210 "string sublex: \"a\\tb\" -> a<TAB>b"); 211 else if (n_str == 1) 212 check(ok && on == 4 && memcmp(out, "x\xc3\xa9y", 4) == 0, 213 "string sublex: \"x\\u00e9y\" -> x U+00E9(UTF-8) y"); 214 else if (n_str == 2) 215 check(ok && on == 5 && memcmp(out, "plain", 5) == 0, 216 "string sublex: \"plain\" -> plain (no escapes)"); 217 n_str++; 218 } 219 } 220 check(st == KIT_GRAM_LEX_EOF, "main lexer reached EOF cleanly"); 221 check(n_num == 3 && n_str == 3, 222 "main lexer produced 3 NUMBER + 3 STRING tokens"); 223 224 printf("\n%d checks, %d failures\n", checks, failures); 225 return failures ? 1 : 0; 226 }