position_test.c (6092B)
1 /* test_position.c — grampos.h line index + the --position-lazy codegen flag. 2 * 3 * For byte and utf8 modes, against a torture input exercising every line-break 4 * kind and (utf8) multibyte scalar columns, this checks three things: 5 * 1. the normal standalone lexer's per-token (line,col) is reproduced exactly 6 * by grampos's offset -> (line,col) query; 7 * 2. the --position-lazy lexer emits the identical (kind,len,offset) stream 8 * but reports line==0 && col==0 (positions deferred to grampos); 9 * 3. grampos over the lazy lexer's offsets recovers the real positions. 10 */ 11 #define _POSIX_C_SOURCE 200809L 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 16 #include <kit/gram_pos.h> 17 #include "posb.h" /* byte, normal */ 18 #include "posbl.h" /* byte, --position-lazy */ 19 #include "posu.h" /* utf8, normal */ 20 #include "posul.h" /* utf8, --position-lazy */ 21 22 #define MAXTOK 4096 23 typedef struct { KitGramTokenKind kind; size_t off, len; uint32_t line, col; } Rec; 24 25 static int fails = 0; 26 static void fail(const char *what, const char *detail) { 27 printf(" FAIL %s: %s\n", what, detail); 28 fails++; 29 } 30 31 /* ---- byte mode ---- */ 32 static void test_byte(const unsigned char *buf, size_t len) { 33 Rec rec[MAXTOK]; size_t n = 0; 34 posb_lex lx; posb_lex_init(&lx, buf, len); 35 for (;;) { 36 KitGramToken t; KitGramLexStatus st = posb_lex_next(&lx, &t); 37 if (st == KIT_GRAM_LEX_EOF) break; 38 if (st != KIT_GRAM_LEX_TOKEN) { fail("byte", "normal lexer error"); return; } 39 if (n == MAXTOK) { fail("byte", "too many tokens"); return; } 40 rec[n].kind = t.kind; rec[n].len = t.len; rec[n].line = t.line; rec[n].col = t.col; 41 rec[n].off = (size_t)((const unsigned char *)t.lexeme - buf); 42 n++; 43 } 44 45 /* lazy lexer: same kind/len/offset, line==col==0 */ 46 posbl_lex lz; posbl_lex_init(&lz, buf, len); size_t i = 0; 47 for (;;) { 48 KitGramToken t; KitGramLexStatus st = posbl_lex_next(&lz, &t); 49 if (st == KIT_GRAM_LEX_EOF) break; 50 if (st != KIT_GRAM_LEX_TOKEN) { fail("byte", "lazy lexer error"); return; } 51 size_t off = (size_t)((const unsigned char *)t.lexeme - buf); 52 if (i >= n) { fail("byte", "lazy emitted more tokens"); return; } 53 if (t.kind != rec[i].kind || t.len != rec[i].len || off != rec[i].off) 54 fail("byte", "lazy token (kind/len/offset) differs from normal"); 55 if (t.line != 0 || t.col != 0) 56 fail("byte", "lazy token line/col not 0"); 57 i++; 58 } 59 if (i != n) fail("byte", "lazy emitted fewer tokens"); 60 61 /* grampos reproduces the normal lexer's line/col */ 62 size_t nl = kit_gram_line_count(buf, len); 63 uint32_t *idx = malloc((nl ? nl : 1) * sizeof *idx); 64 KitGramLineIndex ix; kit_gram_line_index_build(&ix, buf, len, idx, nl); 65 for (size_t k = 0; k < n; k++) { 66 uint32_t L, C; kit_gram_position(&ix, rec[k].off, &L, &C); 67 if (L != rec[k].line || C != rec[k].col) fail("byte", "grampos != lexer line/col"); 68 } 69 free(idx); 70 printf(" byte : %zu tokens, %zu lines, %d fail(s)\n", n, nl + 1, fails); 71 } 72 73 /* ---- utf8 mode ---- */ 74 static void test_utf8(const unsigned char *buf, size_t len) { 75 int before = fails; 76 Rec rec[MAXTOK]; size_t n = 0; 77 posu_lex lx; posu_lex_init(&lx, buf, len); 78 for (;;) { 79 KitGramToken t; KitGramLexStatus st = posu_lex_next(&lx, &t); 80 if (st == KIT_GRAM_LEX_EOF) break; 81 if (st != KIT_GRAM_LEX_TOKEN) { fail("utf8", "normal lexer error"); return; } 82 if (n == MAXTOK) { fail("utf8", "too many tokens"); return; } 83 rec[n].kind = t.kind; rec[n].len = t.len; rec[n].line = t.line; rec[n].col = t.col; 84 rec[n].off = (size_t)((const unsigned char *)t.lexeme - buf); 85 n++; 86 } 87 88 posul_lex lz; posul_lex_init(&lz, buf, len); size_t i = 0; 89 for (;;) { 90 KitGramToken t; KitGramLexStatus st = posul_lex_next(&lz, &t); 91 if (st == KIT_GRAM_LEX_EOF) break; 92 if (st != KIT_GRAM_LEX_TOKEN) { fail("utf8", "lazy lexer error"); return; } 93 size_t off = (size_t)((const unsigned char *)t.lexeme - buf); 94 if (i >= n) { fail("utf8", "lazy emitted more tokens"); return; } 95 if (t.kind != rec[i].kind || t.len != rec[i].len || off != rec[i].off) 96 fail("utf8", "lazy token (kind/len/offset) differs from normal"); 97 if (t.line != 0 || t.col != 0) fail("utf8", "lazy token line/col not 0"); 98 i++; 99 } 100 if (i != n) fail("utf8", "lazy emitted fewer tokens"); 101 102 size_t nl = kit_gram_uline_count(buf, len); 103 uint32_t *idx = malloc((nl ? nl : 1) * sizeof *idx); 104 KitGramULineIndex ix; kit_gram_uline_index_build(&ix, buf, len, idx, nl); 105 for (size_t k = 0; k < n; k++) { 106 uint32_t L, C; kit_gram_uposition(&ix, rec[k].off, &L, &C); 107 if (L != rec[k].line || C != rec[k].col) fail("utf8", "grampos != lexer line/col"); 108 } 109 free(idx); 110 printf(" utf8 : %zu tokens, %zu lines, %d fail(s)\n", n, nl + 1, fails - before); 111 } 112 113 int main(void) { 114 printf("test_position: grampos.h + --position-lazy\n"); 115 116 /* byte torture: '\r' is a column byte in byte mode; only '\n' breaks lines */ 117 const char *bt = 118 "alpha beta\tgamma\n" 119 "delta\r\n" /* CRLF */ 120 "epsilon\rzeta\n" /* lone CR */ 121 "\n\n eta theta\n" /* blank lines + leading indent */ 122 "iota"; 123 test_byte((const unsigned char *)bt, strlen(bt)); 124 125 /* utf8 torture: every break kind + multibyte WORD scalars, as raw bytes */ 126 static const unsigned char ut[] = { 127 'a','b','c', 0x0A, /* LF */ 128 'd','e','f', 0x0D,0x0A, /* CRLF (coalesced) */ 129 'g','h','i', 0x0D, /* lone CR */ 130 'j','k','l', 0xC2,0x85, /* NEL U+0085 */ 131 'm','n','o', 0xE2,0x80,0xA8, /* LS U+2028 */ 132 ' ','p','q','r', 0xE2,0x80,0xA9, /* PS U+2029 */ 133 ' ','s','t','u', 0x0B, /* VT */ 134 'v','w','x', 0x0C, /* FF */ 135 'y','z', ' ', 136 0xC3,0xA9, 0xCE,0xBB, 0xF0,0x9F,0x99,0x82, /* é λ 🙂 (multibyte WORD) */ 137 'A','9', 138 0x0A, 'l','a','s','t' 139 }; 140 test_utf8(ut, sizeof ut); 141 142 if (fails) { printf("FAILED (%d)\n", fails); return 1; } 143 printf("OK\n"); 144 return 0; 145 }