kit

kit
git clone https://git.ryansepassi.com/git/kit.git
Log | Files | Refs | README

unicode_lexer_test.c (3724B)

      1 #include "generated_unicode.h"
      2 
      3 #include <stdio.h>
      4 #include <string.h>
      5 
      6 static int failures = 0;
      7 
      8 typedef struct {
      9     KitGramTokenKind kind;
     10     const unsigned char *lexeme;
     11     size_t len;
     12     uint32_t line, col;
     13 } WantTok;
     14 
     15 static void check_status(const char *what, int ok) {
     16     printf("%s  %s\n", ok ? "ok  " : "FAIL", what);
     17     if (!ok) failures++;
     18 }
     19 
     20 static void expect_token(KitGramToken got, WantTok want, size_t idx) {
     21     char label[160];
     22     int lex_ok = got.len == want.len &&
     23                  memcmp(got.lexeme, want.lexeme, want.len) == 0;
     24     snprintf(label, sizeof label, "unicode token[%zu] kind=%u len=%zu at %u:%u",
     25              idx, (unsigned)got.kind, got.len, got.line, got.col);
     26     check_status(label, got.kind == want.kind && lex_ok &&
     27                         got.line == want.line && got.col == want.col);
     28 }
     29 
     30 static void check_unicode_tokens(void) {
     31     printf("== unicode lexer bytes ==\n");
     32 
     33     static const unsigned char src[] = {
     34         0xC3, 0xA9, ' ', 0xCE, 0xBB, ' ', 0xE9, ' ', 0xA0,
     35         ' ', 0xCF, 0x80
     36     };
     37     static const unsigned char e_acute[] = { 0xC3, 0xA9 };
     38     static const unsigned char lambda[] = { 0xCE, 0xBB };
     39     static const unsigned char pi[] = { 0xCF, 0x80 };
     40     static const unsigned char raw_e9[] = { 0xE9 };
     41     static const unsigned char byte_a0[] = { 0xA0 };
     42     const WantTok want[] = {
     43         { UNICODE_TOK_E_ACUTE, e_acute, sizeof e_acute, 1, 1 },
     44         { UNICODE_TOK_LAMBDA,  lambda,  sizeof lambda,  1, 4 },
     45         { UNICODE_TOK_RAW_E9,  raw_e9,  sizeof raw_e9,  1, 7 },
     46         { UNICODE_TOK_BYTE_A0, byte_a0, sizeof byte_a0, 1, 9 },
     47         { UNICODE_TOK_DIRECT_PI, pi, sizeof pi, 1, 11 },
     48     };
     49 
     50     KitGramLexInput in;
     51     kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0});
     52     KitGramLexer lx;
     53     KitGramLexConfig cfg = {0};
     54     unicode_lexer_init(&lx, &in, &cfg);
     55 
     56     KitGramLexInputSpan span = { .bytes = src, .len = sizeof src };
     57     kit_gram_lex_input_push(&in, &span);
     58     kit_gram_lex_input_finish(&in);
     59 
     60     for (size_t i = 0; i < sizeof want / sizeof want[0]; i++) {
     61         KitGramToken tok;
     62         check_status("unicode token available", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_TOKEN);
     63         expect_token(tok, want[i], i);
     64     }
     65 
     66     KitGramToken tok;
     67     check_status("unicode lexer EOF", kit_gram_lexer_next(&lx, &tok) == KIT_GRAM_LEX_EOF);
     68 }
     69 
     70 static void check_parser_handoff(void) {
     71     printf("\n== unicode parser handoff ==\n");
     72 
     73     static const unsigned char src[] = {
     74         0xC3, 0xA9, ' ', 0xCE, 0xBB, ' ', 0xE9, ' ', 0xA0,
     75         ' ', 0xCF, 0x80
     76     };
     77 
     78     KitGramLexInput in;
     79     kit_gram_lex_input_init(&in, &(KitGramLexInputConfig){0});
     80     KitGramLexer lx;
     81     KitGramLexConfig lcfg = {0};
     82     unicode_lexer_init(&lx, &in, &lcfg);
     83 
     84     KitGramParser ps;
     85     KitGramSlot ctl[64];
     86     KitGramSem vals[64];
     87     KitGramConfig pcfg = { .ctl_stack = ctl, .ctl_cap = 64, .val_stack = vals, .val_cap = 64 };
     88     unicode_parser_init(&ps, &pcfg);
     89 
     90     KitGramLexInputSpan span = { .bytes = src, .len = sizeof src };
     91     kit_gram_lex_input_push(&in, &span);
     92     kit_gram_lex_input_finish(&in);
     93 
     94     for (;;) {
     95         KitGramToken tok;
     96         KitGramLexStatus st = kit_gram_lexer_next(&lx, &tok);
     97         if (st == KIT_GRAM_LEX_TOKEN) {
     98             check_status("unicode parser accepted token", kit_gram_parser_push(&ps, tok) != KIT_GRAM_PARSE_ERROR);
     99             continue;
    100         }
    101         check_status("unicode lexer finished cleanly", st == KIT_GRAM_LEX_EOF);
    102         break;
    103     }
    104 
    105     check_status("unicode parser accepted stream", kit_gram_parser_finish(&ps) == KIT_GRAM_PARSE_ACCEPT);
    106 }
    107 
    108 int main(void) {
    109     check_unicode_tokens();
    110     check_parser_handoff();
    111     return failures ? 1 : 0;
    112 }