gen_gram_unicode_props.py (16683B)
1 #!/usr/bin/env python3 2 """Generate gramgen Unicode property tables from ICU ppucd.txt.""" 3 4 from __future__ import annotations 5 6 import argparse 7 from collections import defaultdict 8 from pathlib import Path 9 10 MAX_CP = 0x10FFFF 11 SURROGATE_FIRST = 0xD800 12 SURROGATE_LAST = 0xDFFF 13 14 BINARY_PROPS = { 15 "Alpha", 16 "Upper", 17 "Lower", 18 "WSpace", 19 "NChar", 20 "DI", 21 "XIDS", 22 "XIDC", 23 "IDS", 24 "IDC", 25 "Join_C", 26 "Pat_Syn", 27 "Pat_WS", 28 } 29 30 REQUIRED_PROPS = { 31 "gc", 32 "sc", 33 "scx", 34 *BINARY_PROPS, 35 } 36 37 GC_MAJOR = { 38 "C": {"Cc", "Cf", "Cn", "Co", "Cs"}, 39 "L": {"Ll", "Lm", "Lo", "Lt", "Lu"}, 40 "LC": {"Ll", "Lt", "Lu"}, 41 "M": {"Mc", "Me", "Mn"}, 42 "N": {"Nd", "Nl", "No"}, 43 "P": {"Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps"}, 44 "S": {"Sc", "Sk", "Sm", "So"}, 45 "Z": {"Zl", "Zp", "Zs"}, 46 } 47 48 SPECIAL_SETS = {"Any", "ASCII", "Assigned"} 49 50 51 def fold_key(s: str) -> str: 52 return "".join(ch.lower() for ch in s if ch not in "-_ \t\r\n\f\v") 53 54 55 def parse_range(s: str) -> tuple[int, int]: 56 if ".." in s: 57 a, b = s.split("..", 1) 58 return int(a, 16), int(b, 16) 59 cp = int(s, 16) 60 return cp, cp 61 62 63 def parse_props(fields: list[str]) -> tuple[dict[str, str], set[str], set[str]]: 64 values: dict[str, str] = {} 65 yes: set[str] = set() 66 no: set[str] = set() 67 for field in fields: 68 if not field: 69 continue 70 if field[0] == "-": 71 no.add(field[1:]) 72 elif "=" in field: 73 key, value = field.split("=", 1) 74 values[key] = value 75 else: 76 yes.add(field) 77 return values, yes, no 78 79 80 def apply_props( 81 lo: int, 82 hi: int, 83 values: dict[str, str], 84 yes: set[str], 85 no: set[str], 86 gc: list[str], 87 sc: list[str], 88 scx: list[tuple[str, ...] | None], 89 binary: dict[str, bytearray], 90 reset_unassigned: bool = False, 91 ) -> None: 92 lo = max(lo, 0) 93 hi = min(hi, MAX_CP) 94 if lo > hi: 95 return 96 97 gc_value = values.get("gc") 98 sc_value = values.get("sc") 99 scx_value = values.get("scx") 100 scx_tuple = tuple(scx_value.split()) if scx_value is not None else None 101 102 for cp in range(lo, hi + 1): 103 if reset_unassigned: 104 gc[cp] = "Cn" 105 sc[cp] = "Zzzz" 106 scx[cp] = None 107 for arr in binary.values(): 108 arr[cp] = 0 109 if gc_value is not None: 110 gc[cp] = gc_value 111 if sc_value is not None: 112 sc[cp] = sc_value 113 if scx_tuple is not None: 114 scx[cp] = scx_tuple 115 for prop in yes: 116 arr = binary.get(prop) 117 if arr is not None: 118 arr[cp] = 1 119 for prop in no: 120 arr = binary.get(prop) 121 if arr is not None: 122 arr[cp] = 0 123 124 125 def add_range_point(ranges: list[tuple[int, int]], cp: int) -> None: 126 if ranges and ranges[-1][1] + 1 == cp: 127 ranges[-1] = (ranges[-1][0], cp) 128 else: 129 ranges.append((cp, cp)) 130 131 132 def c_string(s: str) -> str: 133 out = ['"'] 134 for ch in s: 135 o = ord(ch) 136 if ch == "\\": 137 out.append("\\\\") 138 elif ch == '"': 139 out.append('\\"') 140 elif ch == "\n": 141 out.append("\\n") 142 elif 32 <= o <= 126: 143 out.append(ch) 144 else: 145 out.append(f"\\x{o:02x}") 146 out.append('"') 147 return "".join(out) 148 149 150 def parse_ppucd(path: Path): 151 version = "" 152 prop_aliases: dict[str, list[str]] = {} 153 prop_types: dict[str, str] = {} 154 value_aliases: dict[str, dict[str, list[str]]] = defaultdict(dict) 155 156 gc = ["Cn"] * (MAX_CP + 1) 157 sc = ["Zzzz"] * (MAX_CP + 1) 158 scx: list[tuple[str, ...] | None] = [None] * (MAX_CP + 1) 159 binary = {prop: bytearray(MAX_CP + 1) for prop in BINARY_PROPS} 160 161 for raw in path.read_text(encoding="utf-8").splitlines(): 162 if not raw or raw.startswith("#"): 163 continue 164 fields = raw.split(";") 165 kind = fields[0] 166 if kind == "ucd": 167 version = fields[1] 168 elif kind == "property": 169 ptype = fields[1] 170 aliases = [a for a in fields[2:] if a] 171 if aliases: 172 canon = aliases[0] 173 prop_aliases[canon] = aliases 174 prop_types[canon] = ptype 175 elif kind == "value": 176 prop = fields[1] 177 aliases = [a for a in fields[2:] if a] 178 if aliases: 179 value_aliases[prop][aliases[0]] = aliases 180 elif kind in {"block", "cp", "unassigned"}: 181 lo, hi = parse_range(fields[1]) 182 values, yes, no = parse_props(fields[2:]) 183 apply_props( 184 lo, 185 hi, 186 values, 187 yes, 188 no, 189 gc, 190 sc, 191 scx, 192 binary, 193 reset_unassigned=(kind == "unassigned"), 194 ) 195 196 if not version: 197 raise SystemExit("ppucd is missing ucd version line") 198 199 missing = sorted(prop for prop in REQUIRED_PROPS if prop not in prop_aliases) 200 if missing: 201 raise SystemExit(f"ppucd is missing required properties: {', '.join(missing)}") 202 for prop in ("gc", "sc"): 203 if prop not in value_aliases: 204 raise SystemExit(f"ppucd is missing values for {prop}") 205 206 return version, prop_aliases, prop_types, value_aliases, gc, sc, scx, binary 207 208 209 def build_sets(gc, sc, scx, binary): 210 sets: dict[str, list[tuple[int, int]]] = defaultdict(list) 211 212 for cp in range(MAX_CP + 1): 213 if SURROGATE_FIRST <= cp <= SURROGATE_LAST: 214 continue 215 gc_value = gc[cp] 216 sc_value = sc[cp] 217 218 add_range_point(sets["Any"], cp) 219 if cp <= 0x7F: 220 add_range_point(sets["ASCII"], cp) 221 if gc_value != "Cn": 222 add_range_point(sets["Assigned"], cp) 223 224 add_range_point(sets[f"gc={gc_value}"], cp) 225 for major, members in GC_MAJOR.items(): 226 if gc_value in members: 227 add_range_point(sets[f"gc={major}"], cp) 228 229 add_range_point(sets[f"sc={sc_value}"], cp) 230 scx_values = scx[cp] if scx[cp] is not None else (sc_value,) 231 for value in scx_values: 232 add_range_point(sets[f"scx={value}"], cp) 233 234 for prop, arr in binary.items(): 235 if arr[cp]: 236 add_range_point(sets[prop], cp) 237 238 return dict(sets) 239 240 241 def build_aliases(prop_aliases, value_aliases, sets): 242 bare_aliases: dict[str, str] = {} 243 value_spec_aliases: dict[tuple[str, str], str] = {} 244 245 def add_bare(alias: str, set_name: str) -> None: 246 key = fold_key(alias) 247 old = bare_aliases.get(key) 248 if old is None: 249 bare_aliases[key] = set_name 250 251 def add_value(prop_alias: str, value_alias: str, set_name: str) -> None: 252 value_spec_aliases[(fold_key(prop_alias), fold_key(value_alias))] = set_name 253 254 for special in sorted(SPECIAL_SETS): 255 add_bare(special, special) 256 257 for prop in sorted(BINARY_PROPS): 258 if prop not in sets: 259 continue 260 for alias in prop_aliases[prop]: 261 add_bare(alias, prop) 262 263 for value, aliases in value_aliases["gc"].items(): 264 set_name = f"gc={value}" 265 if set_name not in sets: 266 continue 267 for alias in aliases: 268 add_bare(alias, set_name) 269 for prop_alias in prop_aliases["gc"]: 270 add_value(prop_alias, alias, set_name) 271 272 for value, aliases in value_aliases["sc"].items(): 273 sc_set = f"sc={value}" 274 scx_set = f"scx={value}" 275 for alias in aliases: 276 if sc_set in sets: 277 add_bare(alias, sc_set) 278 for prop_alias in prop_aliases["sc"]: 279 add_value(prop_alias, alias, sc_set) 280 if scx_set in sets: 281 for prop_alias in prop_aliases["scx"]: 282 add_value(prop_alias, alias, scx_set) 283 284 for special in sorted(SPECIAL_SETS): 285 for alias in (special, special.upper(), special.lower()): 286 add_bare(alias, special) 287 288 return bare_aliases, value_spec_aliases 289 290 291 def emit_header(path: Path) -> None: 292 path.write_text( 293 """/* gramunicode_props.h - generated Unicode property table interface. */ 294 #ifndef KIT_GRAM_UNICODE_PROPS_H 295 #define KIT_GRAM_UNICODE_PROPS_H 296 297 #include <kit/core.h> 298 299 #include <stdbool.h> 300 #include <stddef.h> 301 #include <stdint.h> 302 303 typedef struct { 304 uint32_t first; 305 uint32_t last; 306 } KitGramUnicodeRange; 307 308 typedef struct { 309 const KitGramUnicodeRange *ranges; 310 size_t count; 311 } KitGramUnicodeSet; 312 313 typedef enum { 314 KIT_GRAM_UNICODE_PROP_OK, 315 KIT_GRAM_UNICODE_PROP_UNKNOWN_PROPERTY, 316 KIT_GRAM_UNICODE_PROP_UNKNOWN_VALUE, 317 } KitGramUnicodePropStatus; 318 319 KIT_API const char *kit_gram_unicode_props_version(void); 320 321 KIT_API bool kit_gram_unicode_set_contains(const KitGramUnicodeSet *set, uint32_t cp); 322 323 KIT_API KitGramUnicodePropStatus kit_gram_unicode_resolve_property(const char *property, 324 const char *value, 325 KitGramUnicodeSet *out); 326 KIT_API KitGramUnicodePropStatus kit_gram_unicode_resolve_property_spec(const char *spec, 327 KitGramUnicodeSet *out); 328 329 #endif /* KIT_GRAM_UNICODE_PROPS_H */ 330 """, 331 encoding="utf-8", 332 ) 333 334 335 def emit_source(path: Path, version: str, sets, bare_aliases, value_spec_aliases) -> None: 336 set_names = sorted(sets) 337 set_index = {name: i for i, name in enumerate(set_names)} 338 flat_ranges: list[tuple[int, int]] = [] 339 set_entries: list[tuple[int, int]] = [] 340 for name in set_names: 341 ranges = sets[name] 342 off = len(flat_ranges) 343 flat_ranges.extend(ranges) 344 set_entries.append((off, len(ranges))) 345 346 lines: list[str] = [] 347 lines.append("/* Generated by tools/gen_unicode_props.py from ICU ppucd.txt. */") 348 lines.append('#include "unicode_props.h"') 349 lines.append('#include <kit/gram_unicode.h>') 350 lines.append("") 351 lines.append("#include <string.h>") 352 lines.append("") 353 lines.append("typedef struct { uint32_t off, len; } SetEntry;") 354 lines.append("typedef struct { const char *key; uint16_t set; } BareAlias;") 355 lines.append("typedef struct { const char *prop; const char *value; uint16_t set; } ValueAlias;") 356 lines.append("") 357 lines.append(f'static const char ucd_version[] = {c_string(version)};') 358 lines.append("") 359 lines.append("static const KitGramUnicodeRange range_pool[] = {") 360 for lo, hi in flat_ranges: 361 lines.append(f" {{ 0x{lo:04X}u, 0x{hi:04X}u }},") 362 lines.append("};") 363 lines.append("") 364 lines.append("static const SetEntry sets[] = {") 365 for off, length in set_entries: 366 lines.append(f" {{ {off}u, {length}u }},") 367 lines.append("};") 368 lines.append("") 369 lines.append("static const BareAlias bare_aliases[] = {") 370 for key, name in sorted(bare_aliases.items()): 371 lines.append(f" {{ {c_string(key)}, {set_index[name]}u }},") 372 lines.append("};") 373 lines.append("") 374 lines.append("static const ValueAlias value_aliases[] = {") 375 for (prop, value), name in sorted(value_spec_aliases.items()): 376 lines.append(f" {{ {c_string(prop)}, {c_string(value)}, {set_index[name]}u }},") 377 lines.append("};") 378 lines.append("") 379 lines.append("const char *kit_gram_unicode_props_version(void) { return ucd_version; }") 380 lines.append("") 381 lines.append("static KitGramUnicodeSet make_set(uint16_t id) {") 382 lines.append(" SetEntry entry = sets[id];") 383 lines.append(" return (KitGramUnicodeSet){ .ranges = range_pool + entry.off, .count = entry.len };") 384 lines.append("}") 385 lines.append("") 386 lines.append("bool kit_gram_unicode_set_contains(const KitGramUnicodeSet *set, uint32_t cp) {") 387 lines.append(" if (!set) return false;") 388 lines.append(" size_t lo = 0, hi = set->count;") 389 lines.append(" while (lo < hi) {") 390 lines.append(" size_t mid = lo + (hi - lo) / 2;") 391 lines.append(" KitGramUnicodeRange r = set->ranges[mid];") 392 lines.append(" if (cp < r.first) hi = mid;") 393 lines.append(" else if (cp > r.last) lo = mid + 1;") 394 lines.append(" else return true;") 395 lines.append(" }") 396 lines.append(" return false;") 397 lines.append("}") 398 lines.append("") 399 lines.append("static bool find_bare(const char *key, KitGramUnicodeSet *out) {") 400 lines.append(" for (size_t i = 0; i < sizeof bare_aliases / sizeof bare_aliases[0]; i++) {") 401 lines.append(" if (strcmp(key, bare_aliases[i].key) == 0) {") 402 lines.append(" if (out) *out = make_set(bare_aliases[i].set);") 403 lines.append(" return true;") 404 lines.append(" }") 405 lines.append(" }") 406 lines.append(" return false;") 407 lines.append("}") 408 lines.append("") 409 lines.append("KitGramUnicodePropStatus kit_gram_unicode_resolve_property(const char *property,") 410 lines.append(" const char *value,") 411 lines.append(" KitGramUnicodeSet *out) {") 412 lines.append(" char prop_key[96];") 413 lines.append(" char value_key[96];") 414 lines.append(" if (!property) return KIT_GRAM_UNICODE_PROP_UNKNOWN_PROPERTY;") 415 lines.append(" kit_gram_unicode_fold_property_key(property, prop_key, sizeof prop_key);") 416 lines.append(" if (!value || !value[0]) {") 417 lines.append(" return find_bare(prop_key, out) ? KIT_GRAM_UNICODE_PROP_OK : KIT_GRAM_UNICODE_PROP_UNKNOWN_PROPERTY;") 418 lines.append(" }") 419 lines.append(" kit_gram_unicode_fold_property_key(value, value_key, sizeof value_key);") 420 lines.append(" bool saw_prop = false;") 421 lines.append(" for (size_t i = 0; i < sizeof value_aliases / sizeof value_aliases[0]; i++) {") 422 lines.append(" if (strcmp(prop_key, value_aliases[i].prop) != 0) continue;") 423 lines.append(" saw_prop = true;") 424 lines.append(" if (strcmp(value_key, value_aliases[i].value) == 0) {") 425 lines.append(" if (out) *out = make_set(value_aliases[i].set);") 426 lines.append(" return KIT_GRAM_UNICODE_PROP_OK;") 427 lines.append(" }") 428 lines.append(" }") 429 lines.append(" return saw_prop ? KIT_GRAM_UNICODE_PROP_UNKNOWN_VALUE : KIT_GRAM_UNICODE_PROP_UNKNOWN_PROPERTY;") 430 lines.append("}") 431 lines.append("") 432 lines.append("KitGramUnicodePropStatus kit_gram_unicode_resolve_property_spec(const char *spec,") 433 lines.append(" KitGramUnicodeSet *out) {") 434 lines.append(" if (!spec) return KIT_GRAM_UNICODE_PROP_UNKNOWN_PROPERTY;") 435 lines.append(" const char *eq = strchr(spec, '=');") 436 lines.append(" const char *colon = strchr(spec, ':');") 437 lines.append(" const char *sep = eq && colon ? (eq < colon ? eq : colon) : (eq ? eq : colon);") 438 lines.append(" if (!sep) return kit_gram_unicode_resolve_property(spec, NULL, out);") 439 lines.append(" char prop[96];") 440 lines.append(" char value[96];") 441 lines.append(" size_t pn = (size_t)(sep - spec);") 442 lines.append(" if (pn >= sizeof prop) pn = sizeof prop - 1;") 443 lines.append(" memcpy(prop, spec, pn);") 444 lines.append(" prop[pn] = '\\0';") 445 lines.append(" size_t vn = strlen(sep + 1);") 446 lines.append(" if (vn >= sizeof value) vn = sizeof value - 1;") 447 lines.append(" memcpy(value, sep + 1, vn);") 448 lines.append(" value[vn] = '\\0';") 449 lines.append(" return kit_gram_unicode_resolve_property(prop, value, out);") 450 lines.append("}") 451 lines.append("") 452 453 path.write_text("\n".join(lines), encoding="utf-8") 454 455 456 def main() -> None: 457 ap = argparse.ArgumentParser() 458 ap.add_argument("ppucd", type=Path) 459 ap.add_argument("--header", type=Path, default=Path("src/gram/unicode_props.h")) 460 ap.add_argument("--source", type=Path, default=Path("src/gram/unicode_props.c")) 461 args = ap.parse_args() 462 463 version, prop_aliases, _prop_types, value_aliases, gc, sc, scx, binary = parse_ppucd(args.ppucd) 464 sets = build_sets(gc, sc, scx, binary) 465 bare_aliases, value_spec_aliases = build_aliases(prop_aliases, value_aliases, sets) 466 467 args.header.parent.mkdir(parents=True, exist_ok=True) 468 args.source.parent.mkdir(parents=True, exist_ok=True) 469 emit_header(args.header) 470 emit_source(args.source, version, sets, bare_aliases, value_spec_aliases) 471 472 print(f"generated {len(sets)} sets from UCD {version}") 473 474 475 if __name__ == "__main__": 476 main()