unicode_support_test.c (9909B)
1 #include <kit/gram_unicode.h> 2 #include "unicode_props.h" 3 4 #include <stdio.h> 5 #include <string.h> 6 7 static int failures = 0; 8 9 static void check_status(const char *what, int ok) { 10 printf("%s %s\n", ok ? "ok " : "FAIL", what); 11 if (!ok) failures++; 12 } 13 14 static void check_decode(const char *what, const unsigned char *s, size_t len, 15 KitGramUtf8Status want_st, uint32_t want_cp, 16 size_t want_n) { 17 uint32_t cp = 999; 18 size_t n = 999; 19 KitGramUtf8Status st = kit_gram_utf8_decode_one(s, len, &cp, &n); 20 check_status(what, st == want_st && 21 (st != KIT_GRAM_UTF8_OK || (cp == want_cp && n == want_n))); 22 } 23 24 static void check_encode(uint32_t cp, const unsigned char *want, size_t want_n) { 25 unsigned char out[4] = {0, 0, 0, 0}; 26 size_t n = kit_gram_utf8_encode(cp, out); 27 char label[80]; 28 snprintf(label, sizeof label, "encode U+%04X", (unsigned)cp); 29 check_status(label, n == want_n && memcmp(out, want, want_n) == 0); 30 } 31 32 static void check_utf8(void) { 33 printf("== unicode utf-8 helpers ==\n"); 34 35 static const unsigned char ascii[] = { 0x24 }; 36 static const unsigned char min2[] = { 0xC2, 0x80 }; 37 static const unsigned char max2[] = { 0xDF, 0xBF }; 38 static const unsigned char min3[] = { 0xE0, 0xA0, 0x80 }; 39 static const unsigned char before_surrogate[] = { 0xED, 0x9F, 0xBF }; 40 static const unsigned char after_surrogate[] = { 0xEE, 0x80, 0x80 }; 41 static const unsigned char min4[] = { 0xF0, 0x90, 0x80, 0x80 }; 42 static const unsigned char max4[] = { 0xF4, 0x8F, 0xBF, 0xBF }; 43 44 check_decode("decode ASCII", ascii, sizeof ascii, KIT_GRAM_UTF8_OK, 0x24, 1); 45 check_decode("decode U+0080", min2, sizeof min2, KIT_GRAM_UTF8_OK, 0x80, 2); 46 check_decode("decode U+07FF", max2, sizeof max2, KIT_GRAM_UTF8_OK, 0x7FF, 2); 47 check_decode("decode U+0800", min3, sizeof min3, KIT_GRAM_UTF8_OK, 0x800, 3); 48 check_decode("decode U+D7FF", before_surrogate, sizeof before_surrogate, 49 KIT_GRAM_UTF8_OK, 0xD7FF, 3); 50 check_decode("decode U+E000", after_surrogate, sizeof after_surrogate, 51 KIT_GRAM_UTF8_OK, 0xE000, 3); 52 check_decode("decode U+10000", min4, sizeof min4, KIT_GRAM_UTF8_OK, 0x10000, 4); 53 check_decode("decode U+10FFFF", max4, sizeof max4, KIT_GRAM_UTF8_OK, 0x10FFFF, 4); 54 55 static const unsigned char cont[] = { 0x80 }; 56 static const unsigned char overlong2[] = { 0xC0, 0xAF }; 57 static const unsigned char overlong3[] = { 0xE0, 0x9F, 0x80 }; 58 static const unsigned char surrogate[] = { 0xED, 0xA0, 0x80 }; 59 static const unsigned char overlong4[] = { 0xF0, 0x8F, 0x80, 0x80 }; 60 static const unsigned char too_large[] = { 0xF4, 0x90, 0x80, 0x80 }; 61 static const unsigned char bad_lead[] = { 0xF5, 0x80, 0x80, 0x80 }; 62 63 check_decode("reject continuation lead", cont, sizeof cont, 64 KIT_GRAM_UTF8_INVALID, 0, 0); 65 check_decode("reject overlong 2-byte", overlong2, sizeof overlong2, 66 KIT_GRAM_UTF8_INVALID, 0, 0); 67 check_decode("reject overlong 3-byte", overlong3, sizeof overlong3, 68 KIT_GRAM_UTF8_INVALID, 0, 0); 69 check_decode("reject UTF-8 surrogate", surrogate, sizeof surrogate, 70 KIT_GRAM_UTF8_INVALID, 0, 0); 71 check_decode("reject overlong 4-byte", overlong4, sizeof overlong4, 72 KIT_GRAM_UTF8_INVALID, 0, 0); 73 check_decode("reject code point above max", too_large, sizeof too_large, 74 KIT_GRAM_UTF8_INVALID, 0, 0); 75 check_decode("reject invalid lead", bad_lead, sizeof bad_lead, 76 KIT_GRAM_UTF8_INVALID, 0, 0); 77 78 static const unsigned char empty[] = { 0 }; 79 static const unsigned char partial3[] = { 0xE2, 0x82 }; 80 static const unsigned char partial4[] = { 0xF0, 0x90, 0x80 }; 81 check_decode("need more for empty input", empty, 0, KIT_GRAM_UTF8_NEED_MORE, 0, 0); 82 check_decode("need more for null empty input", NULL, 0, KIT_GRAM_UTF8_NEED_MORE, 0, 0); 83 check_decode("need more for partial 3-byte", partial3, sizeof partial3, 84 KIT_GRAM_UTF8_NEED_MORE, 0, 0); 85 check_decode("need more for partial 4-byte", partial4, sizeof partial4, 86 KIT_GRAM_UTF8_NEED_MORE, 0, 0); 87 88 check_encode(0x24, ascii, sizeof ascii); 89 check_encode(0x80, min2, sizeof min2); 90 check_encode(0x800, min3, sizeof min3); 91 check_encode(0x10000, min4, sizeof min4); 92 check_encode(0x10FFFF, max4, sizeof max4); 93 } 94 95 static void check_scalars(void) { 96 printf("\n== unicode scalar helpers ==\n"); 97 98 check_status("ASCII is scalar", kit_gram_unicode_is_scalar('a')); 99 check_status("U+D7FF is scalar", kit_gram_unicode_is_scalar(0xD7FF)); 100 check_status("surrogate is not scalar", !kit_gram_unicode_is_scalar(0xD800)); 101 check_status("max scalar is scalar", kit_gram_unicode_is_scalar(0x10FFFF)); 102 check_status("above max is not scalar", !kit_gram_unicode_is_scalar(0x110000)); 103 check_status("U+FDD0 is noncharacter", kit_gram_unicode_is_noncharacter(0xFDD0)); 104 check_status("U+10FFFF is noncharacter", kit_gram_unicode_is_noncharacter(0x10FFFF)); 105 check_status("U+10FFFD is not noncharacter", !kit_gram_unicode_is_noncharacter(0x10FFFD)); 106 } 107 108 static void check_positions(void) { 109 printf("\n== unicode position helpers ==\n"); 110 111 KitGramUnicodePos pos; 112 kit_gram_unicode_pos_init(&pos); 113 kit_gram_unicode_pos_advance(&pos, 'a'); 114 check_status("ordinary scalar increments column", pos.line == 1 && pos.col == 2); 115 kit_gram_unicode_pos_advance(&pos, '\n'); 116 check_status("LF increments line", pos.line == 2 && pos.col == 1); 117 kit_gram_unicode_pos_advance(&pos, 'b'); 118 check_status("column after LF", pos.line == 2 && pos.col == 2); 119 120 kit_gram_unicode_pos_init(&pos); 121 kit_gram_unicode_pos_advance(&pos, 'a'); 122 kit_gram_unicode_pos_advance(&pos, '\r'); 123 kit_gram_unicode_pos_advance(&pos, '\n'); 124 kit_gram_unicode_pos_advance(&pos, 'b'); 125 check_status("CRLF counts as one newline", pos.line == 2 && pos.col == 2); 126 127 kit_gram_unicode_pos_init(&pos); 128 kit_gram_unicode_pos_advance(&pos, 0x0085); 129 kit_gram_unicode_pos_advance(&pos, 0x2028); 130 kit_gram_unicode_pos_advance(&pos, 0x2029); 131 check_status("Unicode line separators increment lines", 132 pos.line == 4 && pos.col == 1); 133 } 134 135 static void check_property_keys(void) { 136 printf("\n== unicode property key folding ==\n"); 137 138 char out[32]; 139 size_t n = kit_gram_unicode_fold_property_key(" Script-Extensions ", out, sizeof out); 140 check_status("fold property key", n == 16 && strcmp(out, "scriptextensions") == 0); 141 n = kit_gram_unicode_fold_property_key("Line_Break", out, 5); 142 check_status("fold key truncates safely", n == 9 && strcmp(out, "line") == 0); 143 } 144 145 static void check_property_set(const char *spec, uint32_t cp, int want) { 146 KitGramUnicodeSet set; 147 KitGramUnicodePropStatus st = kit_gram_unicode_resolve_property_spec(spec, &set); 148 int has = st == KIT_GRAM_UNICODE_PROP_OK && kit_gram_unicode_set_contains(&set, cp); 149 char label[128]; 150 snprintf(label, sizeof label, "%s %s U+%04X", 151 spec, want ? "contains" : "excludes", (unsigned)cp); 152 check_status(label, st == KIT_GRAM_UNICODE_PROP_OK && has == want); 153 } 154 155 static void check_unicode_props(void) { 156 printf("\n== unicode property tables ==\n"); 157 158 check_status("UCD version is 17.0.0", 159 strcmp(kit_gram_unicode_props_version(), "17.0.0") == 0); 160 161 check_property_set("Any", 0x0000, 1); 162 check_property_set("Any", 0x10FFFF, 1); 163 check_property_set("Any", 0xD800, 0); 164 165 check_property_set("ASCII", 0x007F, 1); 166 check_property_set("ASCII", 0x0080, 0); 167 168 check_property_set("Assigned", 0x0041, 1); 169 check_property_set("Assigned", 0x0378, 0); 170 check_property_set("Assigned", 0xFDD0, 0); 171 172 check_property_set("gc=Lu", 'A', 1); 173 check_property_set("General Category = uppercase-letter", 'A', 1); 174 check_property_set("gc=Lu", 'a', 0); 175 check_property_set("L", 'A', 1); 176 check_property_set("L", 0x03BB, 1); 177 check_property_set("L", '0', 0); 178 179 check_property_set("Script=Greek", 0x03BB, 1); 180 check_property_set("Greek", 0x03BB, 1); 181 check_property_set("Script=Greek", 'A', 0); 182 check_property_set("Script_Extensions=Greek", 0x0301, 1); 183 check_property_set("scx:Grek", 0x0301, 1); 184 check_property_set("Script_Extensions=Greek", 'A', 0); 185 186 check_property_set("White_Space", ' ', 1); 187 check_property_set("space", 0x00A0, 1); 188 check_property_set("White_Space", 'A', 0); 189 check_property_set("Default_Ignorable_Code_Point", 0x00AD, 1); 190 check_property_set("Default_Ignorable_Code_Point", 'A', 0); 191 check_property_set("Noncharacter_Code_Point", 0xFDD0, 1); 192 check_property_set("Noncharacter_Code_Point", 0x10FFFF, 1); 193 check_property_set("Noncharacter_Code_Point", 'A', 0); 194 195 check_property_set("XID_Start", 'A', 1); 196 check_property_set("XID_Start", 0x03BB, 1); 197 check_property_set("XID_Start", 0x0301, 0); 198 check_property_set("XID_Continue", 0x0301, 1); 199 check_property_set("XID_Continue", 'A', 1); 200 201 check_property_set("Pattern_White_Space", '\t', 1); 202 check_property_set("Pattern_White_Space", 'A', 0); 203 check_property_set("Pattern_Syntax", '<', 1); 204 check_property_set("Pattern_Syntax", 'A', 0); 205 206 KitGramUnicodeSet set; 207 check_status("unknown bare property status", 208 kit_gram_unicode_resolve_property_spec("No_Such_Property", &set) == 209 KIT_GRAM_UNICODE_PROP_UNKNOWN_PROPERTY); 210 check_status("unknown property value status", 211 kit_gram_unicode_resolve_property_spec("General_Category=NoSuchValue", &set) == 212 KIT_GRAM_UNICODE_PROP_UNKNOWN_VALUE); 213 } 214 215 int main(void) { 216 check_utf8(); 217 check_scalars(); 218 check_positions(); 219 check_property_keys(); 220 check_unicode_props(); 221 return failures ? 1 : 0; 222 }