kit

unicode.c (5700B)

---

 /* gramunicode.c - UTF-8 and scalar helpers for Unicode-aware lexing. */
 #include <kit/gram_unicode.h>
 
 static bool is_cont(unsigned char b) { return kit_gram_utf8_is_cont(b); }
 
 static bool is_ignored_property_key_char(unsigned char c) {
   return c == '-' || c == '_' || c == ' ' || c == '\t' || c == '\n' ||
          c == '\r' || c == '\f' || c == '\v';
 }
 
 static unsigned char lower_ascii(unsigned char c) {
   return (c >= 'A' && c <= 'Z') ? (unsigned char)(c - 'A' + 'a') : c;
 }
 
 bool kit_gram_unicode_is_surrogate(uint32_t cp) {
   return cp >= 0xD800u && cp <= 0xDFFFu;
 }
 
 bool kit_gram_unicode_is_scalar(uint32_t cp) {
   return cp <= KIT_GRAM_UNICODE_MAX_SCALAR && !kit_gram_unicode_is_surrogate(cp);
 }
 
 bool kit_gram_unicode_is_noncharacter(uint32_t cp) {
   if (cp >= 0xFDD0u && cp <= 0xFDEFu) return true;
   return cp <= KIT_GRAM_UNICODE_MAX_SCALAR && (cp & 0xFFFEu) == 0xFFFEu;
 }
 
 bool kit_gram_unicode_is_line_break(uint32_t cp) {
   return cp == 0x000Au || cp == 0x000Bu || cp == 0x000Cu || cp == 0x000Du ||
          cp == 0x0085u || cp == 0x2028u || cp == 0x2029u;
 }
 
 size_t kit_gram_utf8_encoded_len(uint32_t cp) {
   if (!kit_gram_unicode_is_scalar(cp)) return 0;
   if (cp <= 0x7Fu) return 1;
   if (cp <= 0x7FFu) return 2;
   if (cp <= 0xFFFFu) return 3;
   return 4;
 }
 
 size_t kit_gram_utf8_encode(uint32_t cp, unsigned char out[4]) {
   size_t n = kit_gram_utf8_encoded_len(cp);
   if (!n || !out) return 0;
   if (n == 1) {
     out[0] = (unsigned char)cp;
   } else if (n == 2) {
     out[0] = (unsigned char)(0xC0u | (cp >> 6));
     out[1] = (unsigned char)(0x80u | (cp & 0x3Fu));
   } else if (n == 3) {
     out[0] = (unsigned char)(0xE0u | (cp >> 12));
     out[1] = (unsigned char)(0x80u | ((cp >> 6) & 0x3Fu));
     out[2] = (unsigned char)(0x80u | (cp & 0x3Fu));
   } else {
     out[0] = (unsigned char)(0xF0u | (cp >> 18));
     out[1] = (unsigned char)(0x80u | ((cp >> 12) & 0x3Fu));
     out[2] = (unsigned char)(0x80u | ((cp >> 6) & 0x3Fu));
     out[3] = (unsigned char)(0x80u | (cp & 0x3Fu));
   }
   return n;
 }
 
 KitGramUtf8Status kit_gram_utf8_decode_one(const unsigned char* bytes, size_t len,
                                       uint32_t* cp, size_t* nbytes) {
   if (!cp || !nbytes) return KIT_GRAM_UTF8_INVALID;
   *cp = 0;
   *nbytes = 0;
   if (!len) return KIT_GRAM_UTF8_NEED_MORE;
   if (!bytes) return KIT_GRAM_UTF8_INVALID;
 
   unsigned char b0 = bytes[0];
   if (b0 <= 0x7Fu) {
     *cp = b0;
     *nbytes = 1;
     return KIT_GRAM_UTF8_OK;
   }
 
   if (b0 >= 0xC2u && b0 <= 0xDFu) {
     if (len < 2) return KIT_GRAM_UTF8_NEED_MORE;
     unsigned char b1 = bytes[1];
     if (!is_cont(b1)) return KIT_GRAM_UTF8_INVALID;
     *cp = ((uint32_t)(b0 & 0x1Fu) << 6) | (uint32_t)(b1 & 0x3Fu);
     *nbytes = 2;
     return KIT_GRAM_UTF8_OK;
   }
 
   if (b0 >= 0xE0u && b0 <= 0xEFu) {
     if (len < 2) return KIT_GRAM_UTF8_NEED_MORE;
     unsigned char b1 = bytes[1];
     if (!is_cont(b1)) return KIT_GRAM_UTF8_INVALID;
     if (b0 == 0xE0u && b1 < 0xA0u) return KIT_GRAM_UTF8_INVALID;
     if (b0 == 0xEDu && b1 > 0x9Fu) return KIT_GRAM_UTF8_INVALID;
     if (len < 3) return KIT_GRAM_UTF8_NEED_MORE;
     unsigned char b2 = bytes[2];
     if (!is_cont(b2)) return KIT_GRAM_UTF8_INVALID;
     *cp = ((uint32_t)(b0 & 0x0Fu) << 12) | ((uint32_t)(b1 & 0x3Fu) << 6) |
           (uint32_t)(b2 & 0x3Fu);
     *nbytes = 3;
     return KIT_GRAM_UTF8_OK;
   }
 
   if (b0 >= 0xF0u && b0 <= 0xF4u) {
     if (len < 2) return KIT_GRAM_UTF8_NEED_MORE;
     unsigned char b1 = bytes[1];
     if (!is_cont(b1)) return KIT_GRAM_UTF8_INVALID;
     if (b0 == 0xF0u && b1 < 0x90u) return KIT_GRAM_UTF8_INVALID;
     if (b0 == 0xF4u && b1 > 0x8Fu) return KIT_GRAM_UTF8_INVALID;
     if (len < 3) return KIT_GRAM_UTF8_NEED_MORE;
     unsigned char b2 = bytes[2];
     if (!is_cont(b2)) return KIT_GRAM_UTF8_INVALID;
     if (len < 4) return KIT_GRAM_UTF8_NEED_MORE;
     unsigned char b3 = bytes[3];
     if (!is_cont(b3)) return KIT_GRAM_UTF8_INVALID;
     *cp = ((uint32_t)(b0 & 0x07u) << 18) | ((uint32_t)(b1 & 0x3Fu) << 12) |
           ((uint32_t)(b2 & 0x3Fu) << 6) | (uint32_t)(b3 & 0x3Fu);
     *nbytes = 4;
     return KIT_GRAM_UTF8_OK;
   }
 
   return KIT_GRAM_UTF8_INVALID;
 }
 
 void kit_gram_unicode_pos_init(KitGramUnicodePos* pos) {
   if (!pos) return;
   *pos = (KitGramUnicodePos){.line = 1, .col = 1, .after_cr = false};
 }
 
 void kit_gram_unicode_pos_advance(KitGramUnicodePos* pos, uint32_t cp) {
   if (!pos) return;
   if (cp == 0x000Au && pos->after_cr) {
     pos->after_cr = false;
     return;
   }
   if (cp == 0x000Du) {
     pos->line++;
     pos->col = 1;
     pos->after_cr = true;
     return;
   }
   pos->after_cr = false;
   if (kit_gram_unicode_is_line_break(cp)) {
     pos->line++;
     pos->col = 1;
   } else {
     pos->col++;
   }
 }
 
 KitGramUtf8Status kit_gram_unicode_pos_advance_utf8(KitGramUnicodePos* pos,
                                                const unsigned char* bytes,
                                                size_t len, size_t* nbytes) {
   uint32_t cp = 0;
   size_t n = 0;
   KitGramUtf8Status st = kit_gram_utf8_decode_one(bytes, len, &cp, &n);
   if (nbytes) *nbytes = n;
   if (st != KIT_GRAM_UTF8_OK) return st;
   kit_gram_unicode_pos_advance(pos, cp);
   return KIT_GRAM_UTF8_OK;
 }
 
 size_t kit_gram_unicode_fold_property_key(const char* in, char* out,
                                       size_t out_cap) {
   size_t n = 0;
   if (out_cap) out[0] = '\0';
   if (!in) return 0;
   for (const unsigned char* p = (const unsigned char*)in; *p; p++) {
     if (is_ignored_property_key_char(*p)) continue;
     unsigned char c = lower_ascii(*p);
     if (out_cap && n + 1 < out_cap) out[n] = (char)c;
     n++;
   }
   if (out_cap) {
     size_t end = n < out_cap ? n : out_cap - 1;
     out[end] = '\0';
   }
   return n;
 }