kit

meta.ebnf (6343B)

---

 // gramgen EBNF source language.
 //
 // A file is a stream of declarations and rules. Whitespace and line comments
 // `// ...` are ignored by the gramgen lexer. Block comments are not supported.
 // Terminals are token kinds. They may be produced by a generated lexer or
 // supplied externally by the embedder. Uppercase NAME values refer to named
 // token kinds, and STRING literals refer to literal token kinds. Rule names
 // must start with a lowercase letter.
 
 // Literal declarations choose emitted token enum names. When a generated lexer
 // is emitted, each literal also contributes an exact-match recognizer.
 %token TOKEN_DECL = "%token";
 %token EMPTY_DECL = "%empty";
 %token PRATT_DECL = "%pratt";
 %token LEX_DECL = "%lex";
 %token SKIP_DECL = "%skip";
 %token DEF_DECL = "%def";
 %token KEYWORDS_DECL = "%keywords";
 %token COLON = ":";
 // `%machine NAME { ... }` declares a regular language over an abstract token
 // alphabet (codegen-only verifier + sampler).
 %token MACHINE_DECL = "%machine";
 
 %lex {
   %skip WS = [ \t\r\n] [ \t\r\n]*;
   %skip LINE_COMMENT = "//" [^\n]*;
 
   NAME = [A-Za-z_] [A-Za-z0-9_]*;
   STRING = "\"" ( "\\" [^\n] | [^"\\\n] )* "\"";
   UNICODE_PROP = "\\" [pP] "{" [^}\n] [^}\n]* "}";
   CHAR_CLASS = "[" "^"? ( "\\" [^\n] | [^\]\\\n] ) ( "\\" [^\n] | [^\]\\\n] )* "]";
   // Tier-1 edge anchors: \A/\z (text) and ^/$ (line). Recorded as
   // per-recognizer flags during lowering; mid-pattern uses are rejected.
   ANCHOR = "\\A" | "\\z" | "^" | "$";
   // ASCII shorthand classes \d \w \s and their complements \D \W \S, usable
   // standalone and inside `[...]`. Expanded to character classes during lowering.
   SHORTHAND = "\\" [dDwWsS];
   // Counted-repetition suffix `{n}` `{n,}` `{n,m}` `{,m}`; the bounds are
   // hand-parsed and validated (cap, ordering) during lowering.
   REPEAT = "{" [0-9,] [0-9,]* "}";
 }
 
 grammar = item*;
 item    = decl | rule | pratt_rule | lex_block | machine_block;
 
 decl    = token_decl;
 token_decl = "%token" NAME "=" STRING ";";
 
 rule    = NAME "=" alt ";";
 
 // Rules are LL(1): alternatives are separated by `|`, and each alternative is
 // a sequence of factors. `%empty` is the explicit empty alternative.
 alt     = seq alt_tail*;
 alt_tail = "|" seq;
 seq     = "%empty"
         | factor*;
 
 // A factor is a primary item plus optional quantifier sugar. `?` is
 // zero-or-one and `*` is zero-or-more.
 factor  = primary quant?;
 primary = NAME
         | STRING
         | "(" alt ")";
 
 quant   = "?" | "*";
 
 // Pratt declarations are optional expression parsers. `%pratt NAME { ... }`
 // declares a named parser rule whose body is an operator table instead of LL(1)
 // alternatives. A `primary RULE;` line names the LL(1) leaf parser, and each
 // operator line names a role followed by one or more token names or literals.
 pratt_rule = "%pratt" NAME "{" pratt_line* "}";
 pratt_line = NAME pratt_atom pratt_atom* ";";
 pratt_atom = NAME | STRING;
 
 // Lexer declarations are optional. `%lex { ... }` groups generated DFA
 // recognizers in one block. `TOKEN = pattern;` binds an uppercase token.
 // Tokens without a rule may still be supplied externally and pushed directly to
 // the parser. `%skip NAME = pattern;` recognizes input that is consumed but not
 // emitted to the parser.
 //
 // `%lex :utf8 { ... }` keeps generated DFA tables byte-oriented, but marks the
 // runtime input model as UTF-8 for scalar-position tracking and encoding
 // diagnostics.
 //
 // Patterns are regex-like but not slash-delimited. They support alternation,
 // concatenation, grouping, string literals, character classes, and `.` as a
 // wildcard. Postfix quantifiers are `?`, `*`, `+`, and counted `{n}` `{n,}`
 // `{n,m}` `{,m}` (capped; lazy forms like `*?` are an error). The ASCII
 // shorthands `\d \w \s` and complements `\D \W \S` work standalone and inside
 // `[...]`, in both byte and utf8 mode. `%def name = pattern;` declares an
 // inlined, acyclic fragment (lowercase name) scoped to its `%lex`/`%machine`
 // block and referenced by name in other patterns in the same block; fragments
 // are never tokens. All of these lower to the primitive pattern forms during a
 // desugaring pass before NFA construction.
 //
 // Character classes are lexical primitives, not parsed by this grammar:
 // `[A-Z_]`, `[^"\n]`, or `[A-Za-z0-9_]`. `^` negates only immediately after
 // `[`. In byte mode ranges are written as `a-z`; a literal `-`, `]`, `^`, or
 // `\` must be escaped inside the class. In UTF-8 mode the generator parses
 // Unicode scalar escapes, `\p{...}`/`\P{...}`, and class algebra inside the
 // bracketed primitive. Empty classes are invalid.
 lex_block = "%lex" lex_name lex_mode "{" lex_line* "}";
 // `%machine NAME { ... }` is a token-alphabet lexer block: the same lex_line
 // productions, compiled over an abstract symbol alphabet (UPPERCASE leaves are
 // symbols, `[ A B ]` is a symbol set). Codegen-only verifier + sampler.
 machine_block = "%machine" NAME "{" lex_line* "}";
 lex_name = NAME | %empty;
 lex_mode = ":" NAME | %empty;
 lex_line = lex_rule | skip_rule | def_rule | keywords_block;
 lex_rule = NAME "=" lex_alt ";";
 skip_rule = "%skip" NAME "=" lex_alt ";";
 // `%def name = pattern;` declares an inlined pattern fragment (lowercase name).
 // Fragments are scoped to their `%lex`/`%machine` block, acyclic, and never
 // become tokens (a reference to another block's %def is an "unknown fragment").
 // In a %machine a %def whose body is a set (`[ A B ]`, a union `a | b`, or a
 // class with set algebra) doubles as a named symbol set referenced inside
 // `[ … ]`.
 def_rule = "%def" NAME "=" lex_alt ";";
 // `%keywords HOST { ... }` lists string-literal tokens that the more general
 // recognizer HOST shadows; they are kept out of the DFA and recovered by a
 // lexeme lookup. Each entry names a token, a literal string, or both.
 keywords_block = "%keywords" NAME "{" kw_entry* "}";
 kw_entry = NAME kw_value? ";" | STRING ";";
 kw_value = "=" STRING;
 lex_alt = lex_seq lex_alt_tail*;
 lex_alt_tail = "|" lex_seq;
 lex_seq = lex_factor lex_factor*;
 // Lexer quantifiers extend the parser's `? *` with `+` and counted `{...}`.
 lex_factor = lex_primary lex_quant?;
 lex_quant = "?" | "*" | "+" | REPEAT;
 lex_primary = STRING
             | CHAR_CLASS
             | UNICODE_PROP
             | SHORTHAND
             | "."
             | NAME
             | "(" lex_alt ")"
             | ANCHOR;