sublexer.ebnf (1771B)
1 // Sub-lexer demonstration for --lexer-standalone (driven by test_sublexer.c). 2 // 3 // One grammar, three tokenizers in one file: a main lexer that yields whole 4 // NUMBER and STRING tokens, plus two sub-lexers that re-scan a token's *bytes* 5 // into their lexical parts. This is the standalone sub-lexing idiom — point a 6 // sub-lexer at a parent token's lexeme/len and drain it. There is no shared 7 // cursor and no gram_lex_stack: that is a table-runtime construct, and the 8 // standalone lexer carries none of the runtime (it links no libgram). 9 // 10 // The main NUMBER/STRING productions are the real (RFC 8259-ish) ones; the 11 // sub-lexers are the post-tokenization step a programming-language frontend 12 // does to turn a NUMBER lexeme into a value and a STRING lexeme into its 13 // unescaped bytes. 14 %lex { 15 // `digit` is private to this lexer; the `number` sub-lexer below defines its 16 // own `digit` independently — %def fragments are scoped to their block. 17 %def digit = [0-9]; 18 %skip WS = [ \t\r\n]+; 19 NUMBER = "-"? ( "0" | [1-9] digit* ) ( "." digit+ )? ( [eE] [+\-]? digit+ )?; 20 STRING = "\"" ( [^"\\] | "\\" ( ["\\/bfnrt] | "u" [0-9A-Fa-f] [0-9A-Fa-f] [0-9A-Fa-f] [0-9A-Fa-f] ) )* "\""; 21 } 22 23 // NUMBER payload -> sign / integer / fraction / exponent parts. 24 %lex number { 25 %def digit = [0-9]; // same fragment name as the main lexer, scoped to here 26 NSIGN = "-"; 27 NINT = digit+; 28 NFRAC = "." digit+; 29 NEXP = [eE] [+\-]? digit+; 30 } 31 32 // STRING payload -> quotes, literal runs, and the escape atoms a frontend 33 // decodes (one SESC token per escape, so the consumer never re-scans bytes). 34 %lex strescape { 35 SQUOTE = "\""; 36 SESC = "\\" ( ["\\/bfnrt] | "u" [0-9A-Fa-f] [0-9A-Fa-f] [0-9A-Fa-f] [0-9A-Fa-f] ); 37 SCHARS = [^"\\]+; 38 } 39 40 start = (NUMBER | STRING)*;