kit

asm_lex.c (19900B)

---

 /* Assembler lexer. Streams tokens out of a borrowed source buffer.
  *
  * It intentionally keeps C-like number/string spelling rules because .S
  * sources arrive after C preprocessing and GNU as accepts those spellings
  * in directives and expressions. It does not own macro expansion or C
  * keyword classification.
  *
  * Comments are consumed as whitespace; physical newlines surface as
  * ASM_TOK_NEWLINE so the asm driver can keep line-oriented directive and
  * instruction parsing. */
 
 #include "asm/asm_lex.h"
 
 #include <string.h>
 
 #include "core/heap.h"
 #include "core/pool.h"
 #include "core/slice.h"
 
 struct AsmLexer {
   Compiler* c;
   Pool* pool;
   Heap* heap;
   const char* src;
   size_t len;
   size_t pos;
   u32 file_id;
   u32 line;
   u32 col;
   u8 at_bol;
   u8 had_space;
 };
 
 /* §5.1.1.2 translation phase 2: splice physical lines joined by
  * backslash-newline. Advance past any splice sequence at l->pos so the
  * cursor never rests on the leading backslash of a splice. */
 static void skip_splices(AsmLexer* l) {
   while (l->pos + 1 < l->len && l->src[l->pos] == '\\' &&
          l->src[l->pos + 1] == '\n') {
     l->pos += 2;
     l->line++;
     l->col = 1;
   }
 }
 
 /* Logical peek: returns the off-th post-splice byte starting at l->pos,
  * or -1 at end of input. Does not mutate l->pos. */
 static int peek(const AsmLexer* l, size_t off) {
   size_t pos = l->pos;
   size_t k = 0;
   while (pos < l->len) {
     if (pos + 1 < l->len && l->src[pos] == '\\' && l->src[pos + 1] == '\n') {
       pos += 2;
       continue;
     }
     if (k == off) return (unsigned char)l->src[pos];
     ++pos;
     ++k;
   }
   return -1;
 }
 
 static int bump(AsmLexer* l) {
   int ch;
   skip_splices(l);
   if (l->pos >= l->len) return -1;
   ch = (unsigned char)l->src[l->pos++];
   if (ch == '\n') {
     l->line++;
     l->col = 1;
   } else {
     l->col++;
   }
   return ch;
 }
 
 static int is_digit(int c) { return c >= '0' && c <= '9'; }
 static int is_hex_digit(int c) {
   return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
          (c >= 'A' && c <= 'F');
 }
 /* Identifier-start byte (§6.4.2.1). Letters and underscore are ASCII; bytes
  * ≥ 0x80 are accepted as the implementation-defined "other characters"
  * permitted in identifiers — in practice UTF-8 lead/continuation bytes for
  * extended source characters. UCNs are matched separately via ucn_len since
  * they span multiple source bytes. */
 static int is_alpha(int c) {
   return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' ||
          c >= 0x80;
 }
 static int is_alnum(int c) { return is_alpha(c) || is_digit(c); }
 
 /* Match a UCN at offset `off` from the current position. Returns the total
  * length (6 for \uXXXX, 10 for \UXXXXXXXX), or 0 if no UCN matches. The
  * range constraints from §6.4.3 (no UCN < 00A0 except $/@/`, and none in
  * D800–DFFF) are not enforced here — the lexical form is matched and any
  * downstream phase that cares can diagnose. */
 static int ucn_len(const AsmLexer* l, size_t off) {
   int n, i;
   if (peek(l, off) != '\\') return 0;
   if (peek(l, off + 1) == 'u')
     n = 4;
   else if (peek(l, off + 1) == 'U')
     n = 8;
   else
     return 0;
   for (i = 0; i < n; ++i) {
     if (!is_hex_digit(peek(l, off + 2 + i))) return 0;
   }
   return 2 + n;
 }
 
 static SrcLoc asm_lex_here(const AsmLexer* l) {
   SrcLoc loc;
   loc.file_id = l->file_id;
   loc.line = l->line;
   loc.col = l->col;
   return loc;
 }
 
 AsmLexer* asm_lex_open_mem(Compiler* c, const char* name, const char* src,
                            size_t len) {
   Heap* h = (Heap*)c->ctx->heap;
   AsmLexer* l = (AsmLexer*)h->alloc(h, sizeof(*l), _Alignof(AsmLexer));
   if (!l) return NULL;
   memset(l, 0, sizeof(*l));
   l->c = c;
   l->pool = c->global;
   l->heap = h;
   l->src = src ? src : "";
   l->len = src ? len : 0;
   l->pos = 0;
   if (source_add_memory(c->sources, slice_from_cstr(name), &l->file_id) !=
       KIT_OK) {
     h->free(h, l, sizeof(*l));
     return NULL;
   }
   l->line = 1;
   l->col = 1;
   l->at_bol = 1;
   l->had_space = 0;
   return l;
 }
 
 void asm_lex_close(AsmLexer* l) {
   if (!l) return;
   l->heap->free(l->heap, l, sizeof(*l));
 }
 
 SrcLoc asm_lex_loc(const AsmLexer* l) { return asm_lex_here(l); }
 u32 asm_lex_file_id(const AsmLexer* l) { return l->file_id; }
 const AsmLitInfo* asm_lex_lit(const AsmLexer* l, AsmLitId id) {
   (void)l;
   (void)id;
   return NULL;
 }
 
 /* Intern bytes [start, end) with line splices (\<newline>) removed, so token
  * spellings reflect post-phase-2 logical text. */
 static Sym intern_spliced(AsmLexer* l, size_t start, size_t end) {
   size_t i;
   int has_splice = 0;
   char* buf;
   size_t k;
   Sym sym;
 
   for (i = start; i + 1 < end; ++i) {
     if (l->src[i] == '\\' && l->src[i + 1] == '\n') {
       has_splice = 1;
       break;
     }
   }
   if (!has_splice)
     return pool_intern_slice(l->pool,
                              (Slice){.s = l->src + start, .len = end - start});
 
   buf = (char*)l->heap->alloc(l->heap, end - start, 1);
   k = 0;
   for (i = start; i < end;) {
     if (i + 1 < end && l->src[i] == '\\' && l->src[i + 1] == '\n') {
       i += 2;
       continue;
     }
     buf[k++] = l->src[i++];
   }
   sym = pool_intern_slice(l->pool, (Slice){.s = buf, .len = k});
   l->heap->free(l->heap, buf, end - start);
   return sym;
 }
 
 /* Skip whitespace and comments. Returns 1 if a newline boundary was crossed
  * via comment consumption (caller still emits the explicit newline token on
  * an in-source '\n'). */
 static void skip_ws_and_comments(AsmLexer* l) {
   for (;;) {
     int ch = peek(l, 0);
     if (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\v' || ch == '\f') {
       bump(l);
       l->had_space = 1;
       continue;
     }
     if (ch == '/' && peek(l, 1) == '/') {
       bump(l);
       bump(l);
       while (peek(l, 0) >= 0 && peek(l, 0) != '\n') bump(l);
       l->had_space = 1;
       continue;
     }
     if (ch == '/' && peek(l, 1) == '*') {
       bump(l);
       bump(l);
       while (peek(l, 0) >= 0) {
         if (peek(l, 0) == '*' && peek(l, 1) == '/') {
           bump(l);
           bump(l);
           break;
         }
         bump(l);
       }
       l->had_space = 1;
       continue;
     }
     break;
   }
 }
 
 /* Consume a pp-number per §6.4.8. The cursor is positioned at the leading
  * digit (or `.` followed by a digit) on entry. */
 static void scan_pp_number(AsmLexer* l) {
   if (peek(l, 0) == '.') bump(l);
   bump(l); /* first digit */
   while (l->pos < l->len) {
     int c = peek(l, 0);
     int n = peek(l, 1);
     if ((c == 'e' || c == 'E' || c == 'p' || c == 'P') &&
         (n == '+' || n == '-')) {
       bump(l);
       bump(l);
     } else if (is_alnum(c) || c == '.') {
       bump(l);
     } else {
       break;
     }
   }
 }
 
 /* 1 if the pp-number text is a floating constant (§6.4.4.2): contains a
  * radix `.`, a hex `p`/`P` exponent, or a decimal `e`/`E` exponent. */
 static int pp_number_is_float(const char* s, size_t n) {
   int is_hex = 0;
   size_t i = 0;
   if (n >= 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) {
     is_hex = 1;
     i = 2;
   }
   for (; i < n; ++i) {
     char c = s[i];
     if (c == '.') return 1;
     if (is_hex && (c == 'p' || c == 'P')) return 1;
     if (!is_hex && (c == 'e' || c == 'E')) {
       if (i + 1 < n) {
         char nx = s[i + 1];
         if (nx == '+' || nx == '-' || (nx >= '0' && nx <= '9')) return 1;
       }
     }
   }
   return 0;
 }
 
 /* Consume a quoted body — string ('"') or character ('\''). The cursor is
  * positioned at the opening quote on entry. Returns 1 on an unterminated or
  * newline-broken literal, 0 on a clean close. */
 static int scan_quoted(AsmLexer* l, int quote) {
   bump(l); /* opening quote */
   for (;;) {
     int ch = peek(l, 0);
     if (ch < 0) return 1;
     if (ch == quote) {
       bump(l);
       return 0;
     }
     if (ch == '\n') return 1;
     if (ch == '\\') {
       bump(l); /* backslash */
       if (peek(l, 0) < 0) return 1;
       bump(l); /* the escaped char */
       continue;
     }
     bump(l);
   }
 }
 
 AsmTok asm_lex_next(AsmLexer* l) {
   AsmTok t;
   SrcLoc tloc;
   size_t start;
   int ch;
 
   memset(&t, 0, sizeof(t));
 
   /* Skip whitespace and comments. A newline token is emitted before any
    * subsequent content tokens for the line that follows. */
   for (;;) {
     skip_ws_and_comments(l);
     skip_splices(l);
     if (l->pos >= l->len) {
       t.kind = ASM_TOK_EOF;
       t.loc = asm_lex_here(l);
       return t;
     }
     if (peek(l, 0) == '\n') {
       tloc = asm_lex_here(l);
       bump(l);
       t.kind = ASM_TOK_NEWLINE;
       t.loc = tloc;
       l->at_bol = 1;
       l->had_space = 0;
       return t;
     }
     break;
   }
 
   tloc = asm_lex_here(l);
   start = l->pos;
   ch = peek(l, 0);
 
   if (l->at_bol) t.flags |= ASM_TF_AT_BOL;
   if (l->had_space) t.flags |= ASM_TF_HAS_SPACE;
   l->at_bol = 0;
   l->had_space = 0;
   t.loc = tloc;
 
   /* String / character literal, with optional encoding prefix. The prefix
    * length and encoding flag are decoded together so the spelling we
    * intern includes the prefix bytes. */
   {
     int sp_len = -1;
     int is_char = 0;
     u32 encf = 0;
 
     if (ch == '"') {
       sp_len = 0;
       is_char = 0;
     } else if (ch == '\'') {
       sp_len = 0;
       is_char = 1;
     } else if (ch == 'L' && peek(l, 1) == '"') {
       sp_len = 1;
       is_char = 0;
       encf = ASM_TF_STR_WIDE;
     } else if (ch == 'L' && peek(l, 1) == '\'') {
       sp_len = 1;
       is_char = 1;
       encf = ASM_TF_STR_WIDE;
     } else if (ch == 'u' && peek(l, 1) == '8' && peek(l, 2) == '"') {
       sp_len = 2;
       is_char = 0;
       encf = ASM_TF_STR_U8;
     } else if (ch == 'u' && peek(l, 1) == '"') {
       sp_len = 1;
       is_char = 0;
       encf = ASM_TF_STR_U16;
     } else if (ch == 'u' && peek(l, 1) == '\'') {
       sp_len = 1;
       is_char = 1;
       encf = ASM_TF_STR_U16;
     } else if (ch == 'U' && peek(l, 1) == '"') {
       sp_len = 1;
       is_char = 0;
       encf = ASM_TF_STR_U32;
     } else if (ch == 'U' && peek(l, 1) == '\'') {
       sp_len = 1;
       is_char = 1;
       encf = ASM_TF_STR_U32;
     }
 
     if (sp_len >= 0) {
       int i;
       for (i = 0; i < sp_len; ++i) bump(l);
       if (scan_quoted(l, is_char ? '\'' : '"')) t.flags |= ASM_TF_LITERAL_BAD;
       t.kind = (u16)(is_char ? ASM_TOK_CHR : ASM_TOK_STR);
       t.flags |= encf;
       t.spelling = intern_spliced(l, start, l->pos);
       t.v.str = t.spelling;
       return t;
     }
   }
 
   /* Local-label identifier: a `.L`-prefixed symbol name (the universal GNU
    * convention for assembler-local labels, e.g. `.Lkit_ro.0`, `.L.str`,
    * `.LBB0_1`). Lexed as a single ASM_TOK_IDENT — including the leading dot
    * and any embedded dots — so it flows through the same operand / label /
    * `.type` paths as an ordinary identifier. This is unambiguous against
    * directives: no assembler directive begins with `.L`, so `.text`,
    * `.section`, `.quad` etc. still tokenize as PUNCT('.') + IDENT and reach
    * the directive dispatcher. Embedded `.` is consumed only when followed by
    * another symbol char, so `.Lfoo, x` and `.Lfoo+4` stop at the delimiter. */
   if (ch == '.' && peek(l, 1) == 'L') {
     bump(l); /* '.' */
     bump(l); /* 'L' */
     for (;;) {
       int c = peek(l, 0);
       if (is_alnum(c) || c == '$') {
         bump(l);
       } else if (c == '.' && (is_alnum(peek(l, 1)) || peek(l, 1) == '$' ||
                               peek(l, 1) == '_')) {
         bump(l);
       } else {
         break;
       }
     }
     t.kind = ASM_TOK_IDENT;
     t.spelling = intern_spliced(l, start, l->pos);
     t.v.ident = t.spelling;
     return t;
   }
 
   /* Identifier (§6.4.2). Encoding-prefix candidates above are matched
    * before this since L/u/U followed by a quote is a literal, not an
    * identifier. The grammar's identifier-nondigit covers letters, _,
    * extended source chars (impl-defined; bytes ≥ 0x80 here), and UCNs
    * (§6.4.3) — the latter span multiple source bytes so they're matched
    * via ucn_len rather than the per-byte is_alpha predicate. */
   {
     int u = ucn_len(l, 0);
     if (is_alpha(ch) || u) {
       if (u) {
         int i;
         for (i = 0; i < u; ++i) bump(l);
       } else
         bump(l);
       for (;;) {
         int c = peek(l, 0);
         if (is_alnum(c)) {
           bump(l);
         } else if (c == '.' && is_digit(peek(l, 1))) {
           /* Discriminator-mangled symbol: `name.N` (static locals, lambda /
            * block-scope renaming, e.g. `acc.1`). A `.` followed by a digit
            * continues the identifier. Restricted to `.`+digit so it never
            * swallows a `.`-led mnemonic suffix (`b.eq`, `fcvt.w.s`) or the
            * `.size foo, .-foo` location-counter dot. */
           bump(l);
         } else if ((u = ucn_len(l, 0))) {
           int i;
           for (i = 0; i < u; ++i) bump(l);
         } else {
           break;
         }
       }
       t.kind = ASM_TOK_IDENT;
       t.spelling = intern_spliced(l, start, l->pos);
       t.v.ident = t.spelling;
       return t;
     }
   }
 
   /* Preprocessor-number shaped token, classified to ASM_TOK_NUM /
    * ASM_TOK_FLT for expression diagnostics and future directive support. */
   if (is_digit(ch) || (ch == '.' && is_digit(peek(l, 1)))) {
     size_t plen;
     char* pbuf;
     size_t i, k;
     scan_pp_number(l);
     /* Classify on the post-splice text (the spelling we'll intern). */
     plen = l->pos - start;
     pbuf = (char*)l->heap->alloc(l->heap, plen ? plen : 1, 1);
     k = 0;
     for (i = start; i < l->pos;) {
       if (i + 1 < l->pos && l->src[i] == '\\' && l->src[i + 1] == '\n') {
         i += 2;
         continue;
       }
       pbuf[k++] = l->src[i++];
     }
     t.kind = (u16)(pp_number_is_float(pbuf, k) ? ASM_TOK_FLT : ASM_TOK_NUM);
     /* Preserve common C-style integer/float suffixes in token flags. The
      * current assembler expression evaluator ignores them, but keeping the
      * spelling metadata makes the lexer useful for future directive work. */
     if (t.kind == ASM_TOK_FLT) {
       size_t j = k;
       while (j > 0) {
         char c = pbuf[j - 1];
         if (c == 'f' || c == 'F') {
           t.flags |= ASM_TF_FLT_F;
           --j;
           continue;
         }
         if (c == 'l' || c == 'L') {
           t.flags |= ASM_TF_FLT_L;
           --j;
           continue;
         }
         break;
       }
     } else {
       size_t j = k;
       while (j > 0) {
         char c = pbuf[j - 1];
         if (c == 'u' || c == 'U') {
           t.flags |= ASM_TF_INT_U;
           --j;
           continue;
         }
         if (c == 'l' || c == 'L') {
           if (j >= 2 && (pbuf[j - 2] == 'l' || pbuf[j - 2] == 'L')) {
             t.flags |= ASM_TF_INT_LL;
             j -= 2;
           } else {
             t.flags |= ASM_TF_INT_L;
             --j;
           }
           continue;
         }
         break;
       }
     }
     t.spelling = pool_intern_slice(l->pool, (Slice){.s = pbuf, .len = k});
     l->heap->free(l->heap, pbuf, plen ? plen : 1);
     return t;
   }
 
   /* Punctuator, longest match. `#` is a distinct token because it is both
    * an asm immediate marker and, at BOL in preprocessed assembler, a line
    * marker introducer. */
   {
     int n0 = peek(l, 0);
     int n1 = peek(l, 1);
     int n2 = peek(l, 2);
     int n3 = peek(l, 3);
     int adv = 1;
     u32 punct = ASM_P_NONE;
     u16 kind = ASM_TOK_PUNCT;
     int i;
 
     switch (n0) {
       case '#':
         if (n1 == '#') {
           adv = 2;
           kind = ASM_TOK_HASH_HASH;
           punct = ASM_P_HASH_HASH;
         } else {
           adv = 1;
           kind = ASM_TOK_HASH;
           punct = '#';
         }
         break;
       case '.':
         if (n1 == '.' && n2 == '.') {
           adv = 3;
           punct = ASM_P_ELLIPSIS;
         } else {
           adv = 1;
           punct = '.';
         }
         break;
       case '-':
         if (n1 == '>') {
           adv = 2;
           punct = ASM_P_ARROW;
         } else if (n1 == '-') {
           adv = 2;
           punct = ASM_P_DEC;
         } else if (n1 == '=') {
           adv = 2;
           punct = ASM_P_SUB_ASSIGN;
         } else {
           adv = 1;
           punct = '-';
         }
         break;
       case '+':
         if (n1 == '+') {
           adv = 2;
           punct = ASM_P_INC;
         } else if (n1 == '=') {
           adv = 2;
           punct = ASM_P_ADD_ASSIGN;
         } else {
           adv = 1;
           punct = '+';
         }
         break;
       case '<':
         if (n1 == '<' && n2 == '=') {
           adv = 3;
           punct = ASM_P_SHL_ASSIGN;
         } else if (n1 == '<') {
           adv = 2;
           punct = ASM_P_SHL;
         } else if (n1 == '=') {
           adv = 2;
           punct = ASM_P_LE;
         } else if (n1 == ':') {
           adv = 2;
           punct = '[';
         } /* digraph */
         else if (n1 == '%') {
           adv = 2;
           punct = '{';
         } /* digraph */
         else {
           adv = 1;
           punct = '<';
         }
         break;
       case '>':
         if (n1 == '>' && n2 == '=') {
           adv = 3;
           punct = ASM_P_SHR_ASSIGN;
         } else if (n1 == '>') {
           adv = 2;
           punct = ASM_P_SHR;
         } else if (n1 == '=') {
           adv = 2;
           punct = ASM_P_GE;
         } else {
           adv = 1;
           punct = '>';
         }
         break;
       case '=':
         if (n1 == '=') {
           adv = 2;
           punct = ASM_P_EQ;
         } else {
           adv = 1;
           punct = '=';
         }
         break;
       case '!':
         if (n1 == '=') {
           adv = 2;
           punct = ASM_P_NE;
         } else {
           adv = 1;
           punct = '!';
         }
         break;
       case '&':
         if (n1 == '&') {
           adv = 2;
           punct = ASM_P_AND;
         } else if (n1 == '=') {
           adv = 2;
           punct = ASM_P_AND_ASSIGN;
         } else {
           adv = 1;
           punct = '&';
         }
         break;
       case '|':
         if (n1 == '|') {
           adv = 2;
           punct = ASM_P_OR;
         } else if (n1 == '=') {
           adv = 2;
           punct = ASM_P_OR_ASSIGN;
         } else {
           adv = 1;
           punct = '|';
         }
         break;
       case '^':
         if (n1 == '=') {
           adv = 2;
           punct = ASM_P_XOR_ASSIGN;
         } else {
           adv = 1;
           punct = '^';
         }
         break;
       case '*':
         if (n1 == '=') {
           adv = 2;
           punct = ASM_P_MUL_ASSIGN;
         } else {
           adv = 1;
           punct = '*';
         }
         break;
       case '/':
         if (n1 == '=') {
           adv = 2;
           punct = ASM_P_DIV_ASSIGN;
         } else {
           adv = 1;
           punct = '/';
         }
         break;
       case '%':
         if (n1 == ':' && n2 == '%' && n3 == ':') {
           adv = 4;
           kind = ASM_TOK_HASH_HASH;
           punct = ASM_P_HASH_HASH;
         } else if (n1 == ':') {
           adv = 2;
           kind = ASM_TOK_HASH;
           punct = '#';
         } else if (n1 == '=') {
           adv = 2;
           punct = ASM_P_MOD_ASSIGN;
         } else if (n1 == '>') {
           adv = 2;
           punct = '}';
         } /* digraph */
         else {
           adv = 1;
           punct = '%';
         }
         break;
       case ':':
         if (n1 == '>') {
           adv = 2;
           punct = ']';
         } /* digraph */
         else {
           adv = 1;
           punct = ':';
         }
         break;
       case '(':
       case ')':
       case '{':
       case '}':
       case '[':
       case ']':
       case ',':
       case ';':
       case '?':
       case '~':
         adv = 1;
         punct = (u32)n0;
         break;
       default:
         /* Unknown byte. Surface as a single-char punct so the token
          * stream still progresses; PP/parse may diagnose. */
         adv = 1;
         punct = (u32)n0;
         break;
     }
 
     for (i = 0; i < adv; ++i) bump(l);
     t.kind = kind;
     t.v.punct = punct;
     t.spelling = intern_spliced(l, start, l->pos);
     return t;
   }
 }