kit

internal.h (23045B)

---

 #ifndef KIT_GRAM_INTERNAL_H
 #define KIT_GRAM_INTERNAL_H
 
 #include <ctype.h>
 #include <limits.h>
 #include <setjmp.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include <kit/gram.h>
 #include <kit/support/gram_lex_tables.h>
 #include <kit/support/gram_parse_tables.h>
 #include <kit/gram_unicode.h>
 
 #define diag_vset kit_gram_diag_vset
 #define diag_set kit_gram_diag_set
 #define die_oom kit_gram_die_oom
 #define xmalloc kit_gram_xmalloc
 #define xrealloc kit_gram_xrealloc
 #define xcalloc kit_gram_xcalloc
 #define xfree kit_gram_xfree
 #define mem_release kit_gram_mem_release
 #define scratch_enter kit_gram_scratch_enter
 #define scratch_leave kit_gram_scratch_leave
 #define scratch_release kit_gram_scratch_release
 #define xstrndup kit_gram_xstrndup
 #define xstrdup kit_gram_xstrdup
 #define xasprintf kit_gram_xasprintf
 #define buf_init kit_gram_buf_init
 #define buf_init_writer kit_gram_buf_init_writer
 #define buf_reserve kit_gram_buf_reserve
 #define buf_appendn kit_gram_buf_appendn
 #define buf_append kit_gram_buf_append
 #define buf_append_uint kit_gram_buf_append_uint
 #define buf_take kit_gram_buf_take
 #define str_from_owned kit_gram_str_from_owned
 #define str_dup_len kit_gram_str_dup_len
 #define str_dup_c kit_gram_str_dup_c
 #define str_eq kit_gram_str_eq
 #define str_eq_c kit_gram_str_eq_c
 #define str_to_c kit_gram_str_to_c
 #define intern_len kit_gram_intern_len
 #define intern_c kit_gram_intern_c
 #define name_index_find kit_gram_name_index_find
 #define name_index_put kit_gram_name_index_put
 #define ast_seq_new kit_gram_ast_seq_new
 #define ast_alt_new kit_gram_ast_alt_new
 #define ast_node_new kit_gram_ast_node_new
 #define ast_lex_seq_new kit_gram_ast_lex_seq_new
 #define ast_lex_alt_new kit_gram_ast_lex_alt_new
 #define ast_lex_node_new kit_gram_ast_lex_node_new
 #define ast_seq_append kit_gram_ast_seq_append
 #define ast_alt_append kit_gram_ast_alt_append
 #define ast_lex_seq_append kit_gram_ast_lex_seq_append
 #define ast_lex_alt_append kit_gram_ast_lex_alt_append
 #define hex_val kit_gram_hex_val
 #define is_c_ident_str kit_gram_is_c_ident_str
 #define is_rule_name kit_gram_is_rule_name
 #define is_token_name kit_gram_is_token_name
 #define intset_contains kit_gram_intset_contains
 #define intset_add kit_gram_intset_add
 #define intset_union kit_gram_intset_union
 #define intset_copy kit_gram_intset_copy
 #define intset_sorted kit_gram_intset_sorted
 #define intset_equal kit_gram_intset_equal
 #define intset_intersection_min kit_gram_intset_intersection_min
 #define c_string_str kit_gram_c_string_str
 
 typedef struct GramgenContext GramgenContext;
 typedef struct ScalarSetCacheEntry ScalarSetCacheEntry;
 
 typedef struct {
   const char* path;
   uint32_t line, col;
 } Loc;
 
 /* Pending diagnostic, captured into the GramgenContext and flushed to the
  * caller's KitContext.diag sink at the public-API boundary. (Replaces the
  * upstream gramgen_diagnostic out-param; kit routes diagnostics through
  * KitContext.diag.) */
 typedef struct {
   const char* path;
   uint32_t line, col;
   char message[512];
 } GramDiag;
 
 typedef struct {
   char* s;
   size_t len;
 } Str;
 
 typedef struct {
   GramgenContext* ctx;
   char* s;
   size_t len, cap;
   KitWriter* writer;       /* streaming sink; NULL => accumulate into `s` */
   const char* write_path;  /* for the "codegen write failed" diagnostic   */
 } Buf;
 
 /* Bump-allocated arena block. Allocations within a block are a pointer bump;
  * each carries a 16-byte size header so the most-recent allocation can be freed
  * (LIFO rewind) and realloc can grow in place. Non-LIFO frees are no-ops and
  * the space is reclaimed when the whole arena is released. */
 typedef struct ArenaBlock {
   struct ArenaBlock* next;
   size_t cap;  /* usable bytes in the data area */
   size_t used; /* bytes consumed so far          */
 } ArenaBlock;
 
 /* Tracked block for the freeing (list) allocator. Used for the scratch region
  * so a phase with large, simultaneously-live intermediates (the UTF-8 lexer
  * build) frees incrementally and keeps peak memory low — a bump arena can only
  * reclaim at the end, which would balloon mid-build peak. */
 typedef struct MemBlock {
   void* ptr;
   size_t size;
   struct MemBlock* next;
 } MemBlock;
 
 typedef struct InternEntry {
   char* s;
   size_t len;
   uint64_t hash;
   struct InternEntry* next;
 } InternEntry;
 
 /* A region allocator in one of two modes. MEM_BUMP (the permanent arena) bumps
  * a pointer and frees everything at once. MEM_LIST (the scratch region) tracks
  * each allocation so individual frees reclaim immediately, bounding peak. */
 typedef enum { MEM_BUMP, MEM_LIST } MemKind;
 typedef struct {
   KitHeap* heap;
   MemKind kind;
   ArenaBlock* head; /* MEM_BUMP: first block (for release) */
   ArenaBlock* cur;  /* MEM_BUMP: active bump block          */
   MemBlock* blocks; /* MEM_LIST: tracked allocations        */
 } GramgenMem;
 
 struct GramgenContext {
   GramgenMem mem;     /* permanent arena: owned by the compiled object   */
   GramgenMem scratch; /* transient arena: released after a scoped phase  */
   GramgenMem* active; /* arena that x* allocate from (NULL => &mem)       */
   InternEntry** intern_buckets;
   size_t intern_nbuckets, intern_count;
   ScalarSetCacheEntry* scalar_set_cache;
   int* union_tmp; /* reused scratch for intset_union merges (perm)    */
   size_t union_tmp_cap;
   const KitContext* kit; /* caller context: heap + diag sink              */
   GramDiag* diag;        /* -> diag_storage; populated then flushed to kit */
   GramDiag diag_storage;
   jmp_buf jmp;
   int can_jump;
 };
 
 typedef enum { AST_NAME, AST_LITERAL, AST_GROUP, AST_OPT, AST_REP } AstNodeKind;
 /* LEX_ANCHOR (edge anchor), LEX_REPEAT (counted/`+` quantifier), and LEX_NAME
  * (`%def` fragment reference) are front-end-only kinds: the anchor-extraction
  * and desugaring passes strip/expand/inline them on the lex AST before NFA
  * construction, so the byte and scalar pipelines never encounter them. */
 typedef enum {
   LEX_LITERAL,
   LEX_CLASS,
   LEX_PROP,
   LEX_ANY,
   LEX_GROUP,
   LEX_OPT,
   LEX_REP,
   LEX_ANCHOR,
   LEX_REPEAT,
   LEX_NAME
 } AstLexNodeKind;
 typedef enum { PRATT_NAME, PRATT_LITERAL } PrattAtomKind;
 /* LEX_LINE_DEF is a `%def` fragment definition: inlined and dropped during
  * desugaring, it never becomes a recognizer. LEX_LINE_KEYWORDS is a `%keywords`
  * block: its entries are extracted literals, never DFA recognizers. */
 typedef enum {
   LEX_LINE_TOKEN,
   LEX_LINE_SKIP,
   LEX_LINE_DEF,
   LEX_LINE_KEYWORDS
 } LexLineKind;
 
 /* Counted-repetition bound cap. `A{n,m}` bodies are unrolled into NFA states,
  * so both bounds are capped; overflow is a located error. */
 #define KIT_GRAM_MAX_REPEAT 255
 
 #define KIT_GRAM_HASH_OFFSET_BASIS 1469598103934665603ull
 #define KIT_GRAM_HASH_PRIME 1099511628211ull
 #define GROW_CAP(cap, init) ((cap) ? (cap) * 2 : (init))
 
 typedef struct AstNode AstNode;
 typedef struct AstSeq AstSeq;
 typedef struct AstAlt AstAlt;
 typedef struct AstLexNode AstLexNode;
 typedef struct AstLexSeq AstLexSeq;
 typedef struct AstLexAlt AstLexAlt;
 
 struct AstSeq {
   GramgenContext* ctx;
   AstNode** items;
   size_t nitems, cap;
   Loc loc;
 };
 
 struct AstAlt {
   GramgenContext* ctx;
   AstSeq** seqs;
   size_t nseqs, cap;
   Loc loc;
 };
 
 struct AstNode {
   AstNodeKind kind;
   Loc loc;
   Str value;
   AstAlt* alts;
 };
 
 struct AstLexSeq {
   GramgenContext* ctx;
   AstLexNode** items;
   size_t nitems, cap;
   Loc loc;
 };
 
 struct AstLexAlt {
   GramgenContext* ctx;
   AstLexSeq** seqs;
   size_t nseqs, cap;
   Loc loc;
 };
 
 struct AstLexNode {
   AstLexNodeKind kind;
   Loc loc;
   Str value;
   Str scalar_value;
   AstLexAlt* alts;
 };
 
 typedef struct {
   PrattAtomKind kind;
   Loc loc;
   Str value;
 } AstPrattAtom;
 
 typedef struct {
   GramgenContext* ctx;
   char* kind;
   Loc loc;
   AstPrattAtom** atoms;
   size_t natoms, cap;
 } AstPrattLine;
 
 typedef struct {
   GramgenContext* ctx;
   AstPrattLine** lines;
   size_t nlines, cap;
 } AstPrattSpec;
 
 typedef struct {
   char* name;
   Loc loc;
   AstAlt* alts;
   AstPrattSpec* pratt;
 } AstRule;
 
 typedef struct {
   Str literal;
   char* name;
   Loc loc;
   int used;
 } TokenDecl;
 
 /* One %keywords entry. A token name, a literal, or both; the missing side is
  * resolved by the builder (string -> auto-named token; bare name -> its %token
  * literal). */
 typedef struct {
   char* name;  /* token name, or NULL for the string-only form */
   Str literal; /* literal bytes when has_literal                */
   int has_literal;
   Loc loc;
 } AstKwEntry;
 
 typedef struct {
   LexLineKind kind;
   char* name; /* token/skip/def name; the host name for KEYWORDS */
   Loc loc;
   AstLexAlt* alts; /* NULL for a KEYWORDS line                        */
   AstKwEntry* kw_entries;
   size_t nkw_entries;
 } AstLexLine;
 
 typedef struct {
   GramgenContext* ctx;
   char* name;
   Loc loc;
   KitGramLexInputMode mode;
   AstLexLine** lines;
   size_t nlines, cap;
 } AstLexBlock;
 
 typedef struct {
   GramgenContext* ctx;
   AstRule** rules;
   size_t nrules, cap_rules;
   TokenDecl** token_decls;
   size_t ntoken_decls, cap_token_decls;
   AstLexBlock** lex_blocks;
   size_t nlex_blocks, cap_lex_blocks;
 } ParsedGrammar;
 
 typedef struct {
   int* v;
   size_t n, cap;
 } IntSet;
 
 typedef struct {
   const char** keys;
   int* values;
   size_t n, cap;
 } NameIndexMap;
 
 typedef enum { SYM_TERM, SYM_RULE, SYM_REP, SYM_OPT } SymKind;
 
 typedef struct Sym Sym;
 struct Sym {
   SymKind kind;
   Loc loc;
   int value;
   Sym* sub;
   int set_index;
   int sub_array_id;
 };
 
 typedef struct {
   Sym* syms;
   size_t nsyms, cap;
   Loc loc;
   IntSet first;
   int nullable;
 } Prod;
 
 typedef struct {
   char* role;
   int tok;        /* prefix/postfix/infix op; ternary op1; circumfix open */
   int tok2;       /* ternary op2; circumfix close (0 for unary/binary)    */
   int inner_rule; /* circumfix inner rule index (-1 otherwise)            */
   int prod;
   int lbp, rbp;
   Loc loc;
   char* enum_name;
 } PrattOp;
 
 typedef struct {
   int primary_rule;
   int primary_prod;
   PrattOp* ops;
   size_t nops, cap;
 } PrattInfo;
 
 typedef struct {
   char* name;
   Str display;
   Loc loc;
 } TokenDef;
 
 typedef struct {
   char* name;
   Loc loc;
   int public_rule;
   int index;
   Prod* prods;
   size_t nprods, cap_prods;
   PrattInfo* pratt;
   IntSet first, follow;
   int nullable;
   int empty_prod;
 } Rule;
 
 typedef struct {
   char* name;
   Loc loc;
   AstLexAlt* alts;
   int tok;
   int skip;
   int seq;
   int has_literal_key;
   Str literal_key;
   int start_anchor; /* KitGramLexAnchor */
   int end_anchor;   /* KitGramLexAnchor */
 } LexRecognizer;
 
 /* One extracted literal: its bytes and the token kind to rewrite to. */
 typedef struct {
   Str literal;
   int tok;
 } LexKeyword;
 
 /* All literals extracted under one host kind, addressed by a minimal perfect
  * hash. `slots` holds nslots keywords placed at their hash positions and
  * `seeds` holds the CHD bucket displacements. Mirrors the runtime
  * KitGramLexKeywordTable. */
 typedef struct {
   int host;          /* host token kind whose lexemes are looked up */
   LexKeyword* slots; /* nslots entries; keyword i at its perfect slot */
   size_t nslots;
   uint32_t* seeds; /* nseeds CHD displacements */
   size_t nseeds;
   size_t min_len;
   size_t max_len;
 } LexKeywordTable;
 
 /* A declared %keywords entry, resolved to (lexeme, token) plus its host name
  * and owning lexer. Collected before parser rules so a parser literal matching
  * a keyword resolves to its token without creating a DFA recognizer. */
 typedef struct {
   Str literal;
   int tok;
   char* host;  /* host token name; resolved to a kind in prepare_lexer */
   char* group; /* lexer (lex-block) name this keyword belongs to       */
   Loc loc;
 } KeywordBinding;
 
 typedef struct {
   char* name;
   Loc loc;
   LexRecognizer* recognizers;
   size_t nrecognizers;
   KitGramLexInputMode input;
   uint8_t class_of[256];
   uint16_t nclasses;
   uint16_t* trans;
   uint16_t nstates;
   uint16_t* accept;
   /* End-of-match context accepts (zero-width `\z`/`$` edges). NULL when no end
    * anchors. start_text/start_line are alternate start states for the start-of-
    * match context (`\A`/`^`); 0 when no start anchors (state 0 serves all). */
   uint16_t* accept_text;
   uint16_t* accept_line;
   uint16_t start_text;
   uint16_t start_line;
   /* Automatic keyword extraction: literals moved out of the DFA, grouped by
    * host kind (sorted by host, then lexeme). Empty when nothing was extracted.
    */
   LexKeywordTable* keyword_tables;
   size_t nkeyword_tables;
   /* ---- Token-machine (%machine) fields; NULL/0 for byte/scalar lexers. ----
    * A token machine is a regular language over an abstract symbol alphabet,
    * compiled codegen-only (it never materializes an KitGramLexGrammar).
    * class_of[] above is byte-only and unused here; the symbol->class map is
    * sym_class_of (uint16, NSYM-wide, no 256 cap), and `accept`/`trans` carry
    * the atom DFA directly (no UTF-8 lowering). Each recognizer is a named rule;
    * the accept id is the winning rule's source-order index (the kind report).
    */
   int is_machine;
   uint16_t* sym_class_of; /* nsym symbol -> class id                       */
   uint16_t nsym;          /* number of alphabet symbols                    */
   char** sym_names;       /* nsym symbol names, by id (first-use order)    */
   uint8_t* live;          /* per state: an accepting state still reachable */
 } LexDFA;
 
 typedef struct {
   GramgenContext* ctx;
   ParsedGrammar* pg;
   Rule* rules;
   size_t nrules, cap_rules;
   size_t public_count;
   TokenDef* tokens;
   size_t ntokens, cap_tokens;
   NameIndexMap token_index;
   NameIndexMap rule_index;
   /* O(1) lookups keyed by interned-pointer identity (every name/literal is
    * interned to a canonical pointer, so pointer identity == byte identity).
    * literal_* key on the literal's interned bytes; decl_name on a token
    * declaration's name. They replace what were linear scans over these lists.
    */
   NameIndexMap literal_decl_index; /* interned literal -> token_decls index */
   NameIndexMap decl_name_index;    /* token decl name  -> token_decls index */
   NameIndexMap kw_literal_index;   /* interned literal -> kw_bindings index  */
   NameIndexMap literal_rec_index;  /* interned literal -> literal_recs index */
   LexRecognizer* literal_recs;
   size_t nliteral_recs, cap_literal_recs;
   KeywordBinding* kw_bindings; /* declared %keywords entries (all lexers) */
   size_t nkw_bindings, cap_kw_bindings;
   LexDFA* lex_dfa; /* main lexer, when present */
   LexDFA* lex_dfas;
   size_t nlex_dfas, cap_lex_dfas;
   /* Token machines (%machine blocks). Kept separate from lex_dfas so they never
    * enter the byte/scalar KitGramLexGrammar emission or the TOK_* enum. */
   LexDFA* machines;
   size_t nmachines, cap_machines;
   IntSet* sets;
   size_t nsets, cap_sets;
   Sym** wrapper_syms;
   size_t nwrapper_syms, cap_wrapper_syms;
   int rec_seq;
   int multiline;        /* bake newline-aware ^/$ into emitted lexers */
   int lexer_standalone; /* emit a self-contained tokenizer + match API,
                            table-free (re2c-style scanner, no runtime link) */
   int fold_keywords;    /* standalone: fold %keywords into the DFA instead of
                            the default minimal-perfect-hash lookup */
   int position_lazy;    /* standalone: omit per-token line/col (emit 0) */
   int parser_codegen; /* also emit a recursive-descent parser (experimental) */
   int parser_recover; /* emit RD error-recovery machinery (opt-in) */
   /* Interned Pratt role/kind tokens. line->kind and op->role are interned, so
    * classification compares against these by pointer identity rather than
    * strcmp. Interned once in the compile context and carried on the builder,
    * so codegen (a separate context) compares against them too. */
   struct {
     char *primary, *prefix, *postfix, *infixl, *infixr, *infix, *ternary,
         *circumfix;
   } role;
 } Builder;
 
 typedef struct {
   uint32_t lo, hi;
 } ScalarRange;
 
 #define UNICODE_MAX_SCALAR 0x10FFFFu
 #define UNICODE_SURROGATE_FIRST 0xD800u
 #define UNICODE_SURROGATE_LAST 0xDFFFu
 
 typedef struct {
   GramgenContext* ctx;
   ScalarRange* v;
   size_t n, cap;
 } ScalarSet;
 
 struct ScalarSetCacheEntry {
   char* key;
   size_t key_len;
   uint64_t hash;
   ScalarSet set;
   ScalarSetCacheEntry* next;
 };
 
 /* Shared NFA->DFA core (subset construction, minimization, canonical relabel)
  * used by both the byte and scalar lexer pipelines. */
 #include "lex_dfa.h"
 
 /* Alphabet-neutral range-set NFA + atom partition + determinization, shared by
  * the utf8 scalar pipeline and the token-alphabet (%machine) pipeline. */
 #include "lex_range.h"
 
 void diag_vset(GramgenContext* ctx, Loc loc, const char* fmt, va_list ap);
 void diag_set(GramgenContext* ctx, Loc loc, const char* fmt, ...);
 void die_oom(GramgenContext* ctx);
 
 void* xmalloc(GramgenContext* ctx, size_t n);
 void* xrealloc(GramgenContext* ctx, void* p, size_t n);
 void* xcalloc(GramgenContext* ctx, size_t n, size_t size);
 void xfree(GramgenContext* ctx, void* p);
 void mem_release(GramgenMem* m);
 void scratch_enter(GramgenContext* ctx); /* route x* to the scratch arena   */
 void scratch_leave(GramgenContext* ctx); /* route x* back to the perm arena  */
 void scratch_release(GramgenContext* ctx); /* free everything in the scratch */
 char* xstrndup(GramgenContext* ctx, const char* s, size_t n);
 char* xstrdup(GramgenContext* ctx, const char* s);
 char* xasprintf(GramgenContext* ctx, const char* fmt, ...);
 
 void buf_init(GramgenContext* ctx, Buf* b);
 void buf_init_writer(GramgenContext* ctx, Buf* b, KitWriter* writer,
                      const char* path);
 void buf_reserve(Buf* b, size_t add);
 void buf_appendn(Buf* b, const char* s, size_t n);
 void buf_append(Buf* b, const char* s);
 void buf_append_uint(Buf* b, unsigned v);
 char* buf_take(Buf* b);
 
 Str str_from_owned(char* s, size_t len);
 Str str_dup_len(GramgenContext* ctx, const char* s, size_t len);
 Str str_dup_c(GramgenContext* ctx, const char* s);
 int str_eq(Str a, Str b);
 int str_eq_c(Str a, const char* b);
 char* str_to_c(GramgenContext* ctx, Str s);
 char* intern_len(GramgenContext* ctx, const char* s, size_t len);
 char* intern_c(GramgenContext* ctx, const char* s);
 int name_index_find(const NameIndexMap* map, const char* key, int* out);
 void name_index_put(GramgenContext* ctx, NameIndexMap* map, const char* key,
                     int value);
 
 AstSeq* ast_seq_new(GramgenContext* ctx, Loc loc);
 AstAlt* ast_alt_new(GramgenContext* ctx, Loc loc);
 AstNode* ast_node_new(GramgenContext* ctx, AstNodeKind kind, Loc loc);
 AstLexSeq* ast_lex_seq_new(GramgenContext* ctx, Loc loc);
 AstLexAlt* ast_lex_alt_new(GramgenContext* ctx, Loc loc);
 AstLexNode* ast_lex_node_new(GramgenContext* ctx, AstLexNodeKind kind, Loc loc);
 void ast_seq_append(AstSeq* seq, AstNode* node);
 void ast_alt_append(AstAlt* alt, AstSeq* seq);
 void ast_lex_seq_append(AstLexSeq* seq, AstLexNode* node);
 void ast_lex_alt_append(AstLexAlt* alt, AstLexSeq* seq);
 
 void kit_gram_error(GramgenContext* ctx, Loc loc, const char* fmt, ...);
 int hex_val(int c);
 int is_c_ident_str(Str s);
 int is_rule_name(const char* name);
 int is_token_name(const char* name);
 
 int intset_contains(const IntSet* s, int v);
 int intset_add(GramgenContext* ctx, IntSet* s, int v);
 int intset_union(GramgenContext* ctx, IntSet* dst, const IntSet* src);
 IntSet intset_copy(GramgenContext* ctx, const IntSet* src);
 int* intset_sorted(GramgenContext* ctx, const IntSet* s);
 int intset_equal(const IntSet* a, const IntSet* b);
 int intset_intersection_min(const IntSet* a, const IntSet* b);
 
 char* c_string_str(GramgenContext* ctx, Str s);
 char* kit_gram_pratt_prod_enum_name(GramgenContext* ctx, Builder* b,
                                    const char* rule_name, const char* role,
                                    int tok);
 
 Builder* kit_gram_builder_new(GramgenContext* ctx, ParsedGrammar* pg);
 Builder* kit_gram_builder_build(Builder* b);
 void kit_gram_prepare_lexer(Builder* b);
 int kit_gram_token_for_ident(Builder* b, const char* name, Loc loc);
 int kit_gram_next_rec_seq(Builder* b);
 
 /* Token-alphabet (%machine) compile: desugared `recognizers` (each a named
  * rule) over an abstract symbol alphabet -> a codegen-only token-machine
  * LexDFA. Fills is_machine/sym_class_of/nsym/sym_names/live plus the shared
  * atom DFA tables (nclasses/trans/nstates/accept). `sets` are the machine's
  * %def lines (LEX_LINE_DEF): a %def whose body is a set doubles as a named
  * symbol set that a `[ … ]` may reference by lowercase name; resolved and
  * inlined into the bracket text up front. gen/kit_gram_lex_tokens.c. */
 LexDFA* kit_gram_compile_machine(GramgenContext* ctx, const char* name,
                                 LexRecognizer* recognizers, size_t nrecognizers,
                                 AstLexLine** sets, size_t nsets, Loc loc);
 
 /* Parse a %machine `[ A B ]` / `[^ A B ]` set node into interned member names;
  * shared by the NFA leaves and the sampler. *complement set for `[^ … ]`.
  * Members must be UPPERCASE symbol names: lowercase set references are resolved
  * and inlined into the bracket text before this runs (see
  * kit_gram_compile_machine), so reaching one here is a grammar/internal error.
  */
 char* const* kit_gram_machine_class_names(GramgenContext* ctx, Str raw, Loc loc,
                                          size_t* nout, int* complement);
 
 /* In-memory AST-directed sampler for a token machine: generate one valid trace
  * of rule `which` into out[0..cap) as symbol-name pointers (into m->sym_names);
  * *n is the produced length (may exceed cap if truncated). Deterministic for a
  * given *seed (advanced on return). Returns KIT_GRAM_GEN_DONE, or KIT_GRAM_GEN_LIMIT if
  * max_tokens was hit. Matches the emitted gen_* sampler bit-for-bit. */
 KitGramGenStatus kit_gram_machine_sample(GramgenContext* ctx, const LexDFA* m,
                                        size_t which, uint64_t* seed,
                                        double stop_prob, size_t max_repeat,
                                        size_t max_tokens, const char** out,
                                        size_t cap, size_t* n);
 
 ScalarSet kit_gram_scalar_set_universe(GramgenContext* ctx);
 void kit_gram_scalar_set_require_nonempty(GramgenContext* ctx,
                                          const ScalarSet* set, Loc loc);
 ScalarSet kit_gram_unicode_char_class_set(GramgenContext* ctx, Str raw, Loc loc);
 ScalarSet kit_gram_unicode_prop_atom_set(GramgenContext* ctx, Str raw, Loc loc);
 
 #ifndef KIT_GRAM_NO_UNICODE
 /* utf8 lexer pipeline entry point (gen/kit_gram_lex_scalar.c); the byte driver
  * in gen/kit_gram_lex_byte.c calls this for %lex :utf8 grammars. */
 void kit_gram_compile_lexer_utf8(
     GramgenContext* ctx, Loc loc, LexRecognizer* recognizers,
     const size_t* compile_indices, size_t ncompile, uint8_t class_of[256],
     uint16_t* nclasses_out, uint16_t** trans_out, uint16_t* nstates_out,
     uint16_t** accept_out, uint16_t** accept_text_out,
     uint16_t** accept_line_out, uint16_t* start_text_out,
     uint16_t* start_line_out);
 #endif
 
 #endif /* KIT_GRAM_INTERNAL_H */