commit 44c6219f7bef26da23a7e87681417a7b51ffca1c
parent c4eed6ef3afdbc721e74fd31d2448f0a4c875262
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 23 May 2026 13:37:02 -0700
cc: drop --dump-tokens and the test/lex corpus
The --dump-tokens flag and its libcfree backing (cfree_c_dump_tokens
plus its dump_* helpers in lang/c/c.c) were a developer-only feature
whose only consumer was test/lex/. Lexer coverage continues via the
parse, pp, and compile suites.
Diffstat:
38 files changed, 19 insertions(+), 1408 deletions(-)
diff --git a/Makefile b/Makefile
@@ -157,7 +157,7 @@ bench-opt: bin
@bash scripts/opt_bench.sh
format:
- find src include driver lang test rt \( -path test/lex -o -path test/pp \) -prune -o \( -name '*.c' -o -name '*.h' \) -print | xargs clang-format -i --style=google
+ find src include driver lang test rt -path test/pp -prune -o \( -name '*.c' -o -name '*.h' \) -print | xargs clang-format -i --style=google
clean:
rm -rf build
diff --git a/driver/cc.c b/driver/cc.c
@@ -16,7 +16,7 @@
* without -c compiles all C sources, links any .o/.a inputs alongside, and
* emits an executable. The flag surface is a GCC subset:
*
- * -c -E -o -O0/1/2 -g --dump-tokens
+ * -c -E -o -O0/1/2 -g
* -I -isystem -D -U
* -M -MM -MD -MMD -MF -MT -MQ -MP
* -target TRIPLE
@@ -111,7 +111,6 @@ typedef struct CcOptions {
int compile_only; /* -c */
int preprocess_only; /* -E */
- int dump_tokens; /* --dump-tokens */
int emit_c_source; /* --emit=c */
int opt_level; /* -O0/-O1/-O2 */
int debug_info; /* -g */
@@ -210,7 +209,7 @@ typedef struct CcOptions {
static void cc_usage(void) {
driver_errf(CC_TOOL, "%s",
- "usage: cfree cc [-c|-E|--dump-tokens|-shared] [-o out] "
+ "usage: cfree cc [-c|-E|-shared] [-o out] "
"[options...] inputs...\n"
" cfree cc --help for full option reference");
}
@@ -226,7 +225,6 @@ void driver_help_cc(void) {
" cfree cc -c [options] input.c compile one source to "
".o\n"
" cfree cc -E [options] input.c preprocess to -o\n"
- " cfree cc --dump-tokens [options] input.c token dump to -o\n"
" cfree cc -shared [options] inputs... link a shared library\n"
" cfree cc -M|-MM [options] input.c print header deps; no "
"compile\n"
@@ -878,8 +876,7 @@ static int cc_apply_hosted_profile(CcOptions* o) {
uint32_t i;
uint32_t insert_pos = 0;
int link_action = !o->compile_only && !o->preprocess_only &&
- !o->dump_tokens && o->dep_mode != CC_DEP_M &&
- o->dep_mode != CC_DEP_MM;
+ o->dep_mode != CC_DEP_M && o->dep_mode != CC_DEP_MM;
if (!o->wants_hosted_libc || o->shared) return 0;
if (o->no_stdlib || o->no_defaultlibs) {
driver_errf(CC_TOOL,
@@ -987,7 +984,7 @@ static int cc_append_windows_lib_dirs(CcOptions* o) {
}
static int cc_has_link_action(const CcOptions* o) {
- return !o->compile_only && !o->preprocess_only && !o->dump_tokens &&
+ return !o->compile_only && !o->preprocess_only &&
o->dep_mode != CC_DEP_M && o->dep_mode != CC_DEP_MM;
}
@@ -1033,10 +1030,6 @@ static int cc_parse(int argc, char** argv, CcOptions* o) {
o->preprocess_only = 1;
continue;
}
- if (driver_streq(a, "--dump-tokens")) {
- o->dump_tokens = 1;
- continue;
- }
if (driver_streq(a, "--emit=c")) {
/* C-source output instead of object bytes. Forces -c-style single-input
* compile (no link). See doc/CBACKEND.md. */
@@ -1500,13 +1493,8 @@ static int cc_parse(int argc, char** argv, CcOptions* o) {
driver_errf(CC_TOOL, "-c and -E are mutually exclusive");
return 1;
}
- if (o->dump_tokens && (o->compile_only || o->preprocess_only)) {
- driver_errf(CC_TOOL, "--dump-tokens is incompatible with -c/-E");
- return 1;
- }
- if (o->shared &&
- (o->compile_only || o->preprocess_only || o->dump_tokens)) {
- driver_errf(CC_TOOL, "-shared is incompatible with -c/-E/--dump-tokens");
+ if (o->shared && (o->compile_only || o->preprocess_only)) {
+ driver_errf(CC_TOOL, "-shared is incompatible with -c/-E");
return 1;
}
if (!o->shared && o->soname) {
@@ -1534,21 +1522,12 @@ static int cc_parse(int argc, char** argv, CcOptions* o) {
return 1;
}
}
- if (o->dump_tokens) {
- if (total_sources != 1 || total_link != 0) {
- driver_errf(
- CC_TOOL,
- "--dump-tokens requires exactly one C source and no .o/.a inputs");
- return 1;
- }
- }
{
int dep_only = (o->dep_mode == CC_DEP_M || o->dep_mode == CC_DEP_MM);
int dep_with_compile =
(o->dep_mode == CC_DEP_MD || o->dep_mode == CC_DEP_MMD);
- if (o->dep_mode != CC_DEP_NONE &&
- (o->preprocess_only || o->dump_tokens)) {
- driver_errf(CC_TOOL, "-M* is incompatible with -E/--dump-tokens");
+ if (o->dep_mode != CC_DEP_NONE && o->preprocess_only) {
+ driver_errf(CC_TOOL, "-M* is incompatible with -E");
return 1;
}
if (dep_only && total_sources != 1) {
@@ -1651,37 +1630,6 @@ out:
return rc;
}
-static int cc_dump_tokens(DriverEnv* env, const CcOptions* o) {
- CfreeContext ctx = driver_env_to_context(env);
- CfreeCompiler* compiler = NULL;
- CfreeWriter* writer = NULL;
- CfreeFileData fd = {0};
- CfreeBytes input = {0};
- int rc = 1;
- int loaded = 0;
-
- if (cc_load_single_source(&ctx, o, &input, &fd, &loaded) != 0) goto out;
-
- if (ctx.file_io->open_writer(ctx.file_io->user, o->output_path, &writer) !=
- CFREE_OK) {
- driver_errf(CC_TOOL, "failed to open output: %s", o->output_path);
- goto out;
- }
-
- if (driver_compiler_new(o->target, &ctx, &compiler) != CFREE_OK) {
- driver_errf(CC_TOOL, "failed to initialize compiler");
- goto out;
- }
-
- rc = cfree_c_dump_tokens(compiler, &input, writer) == CFREE_OK ? 0 : 1;
-
-out:
- if (compiler) driver_compiler_free(compiler);
- if (writer) cfree_writer_close(writer);
- if (loaded) ctx.file_io->release(ctx.file_io->user, &fd);
- return rc;
-}
-
/* ---- header-dependency output (-M family) ---- */
typedef struct CcDepList {
@@ -2494,7 +2442,7 @@ int driver_cc(int argc, char** argv) {
return rc;
}
- link_action = !co.compile_only && !co.preprocess_only && !co.dump_tokens &&
+ link_action = !co.compile_only && !co.preprocess_only &&
co.dep_mode != CC_DEP_M && co.dep_mode != CC_DEP_MM;
if (driver_runtime_resolve(&env, co.support_dir, co.driver_path, &runtime) ==
0) {
@@ -2549,8 +2497,6 @@ int driver_cc(int argc, char** argv) {
if (co.preprocess_only) {
rc = cc_preprocess(&env, &co, &pp);
- } else if (co.dump_tokens) {
- rc = cc_dump_tokens(&env, &co);
} else if (co.dep_mode == CC_DEP_M || co.dep_mode == CC_DEP_MM) {
rc = cc_run_deps_only(&env, &co, &pp);
} else if (co.compile_only) {
diff --git a/lang/c/c.c b/lang/c/c.c
@@ -85,100 +85,6 @@ CfreeStatus cfree_c_preprocess(CfreeCompiler* c,
return cfree_frontend_run(c, c_preprocess_body, &r);
}
-static void dump_write_str(CfreeWriter* w, const char* s) {
- size_t n = 0;
- while (s[n]) ++n;
- (void)cfree_writer_write(w, s, n);
-}
-
-static void dump_write_sym(CfreeWriter* w, Compiler* c, Sym sym) {
- size_t len = 0;
- const char* s = sym ? compiler_sym_str(c, sym, &len) : NULL;
- if (s && len) (void)cfree_writer_write(w, s, len);
-}
-
-static void dump_emit(CfreeWriter* w, Compiler* c, const Tok* t) {
- switch (t->kind) {
- case TOK_EOF:
- dump_write_str(w, "(eof)\n");
- return;
- case TOK_NEWLINE:
- dump_write_str(w, "(newline)\n");
- return;
- case TOK_PP_HASH:
- dump_write_str(w, "(pp-hash)\n");
- return;
- case TOK_PP_PASTE:
- dump_write_str(w, "(pp-paste)\n");
- return;
- case TOK_HEADER:
- dump_write_str(w, "(header ");
- break;
- case TOK_IDENT:
- dump_write_str(w, "(ident ");
- break;
- case TOK_NUM:
- dump_write_str(w, "(num ");
- break;
- case TOK_FLT:
- dump_write_str(w, "(flt ");
- break;
- case TOK_STR:
- dump_write_str(w, "(str ");
- break;
- case TOK_CHR:
- dump_write_str(w, "(chr ");
- break;
- case TOK_PUNCT:
- dump_write_str(w, "(punct ");
- break;
- default:
- dump_write_str(w, "(unknown ");
- break;
- }
- dump_write_sym(w, c, t->spelling);
- dump_write_str(w, ")\n");
-}
-
-typedef struct CDumpTokensRun {
- const CfreeBytes* input;
- CfreeWriter* out;
-} CDumpTokensRun;
-
-static CfreeStatus c_dump_tokens_body(CfreeCompiler* c, void* user) {
- CDumpTokensRun* r = (CDumpTokensRun*)user;
- const CfreeBytes* input = r->input;
- CfreeWriter* out = r->out;
- Lexer* lex;
- Tok t;
-
- if (!input || !out) {
- c_bad_options(c, "dump_tokens args missing");
- }
- if (!input->name) c_bad_options(c, "input name is NULL");
- if (!input->data && input->len != 0) {
- c_bad_options(c, "input data is NULL but len > 0");
- }
-
- lex = lex_open_mem(c, input->name, (const char*)input->data, input->len);
- if (!lex) compiler_panic(c, c_no_loc(), "C lexer out of memory");
- for (;;) {
- t = lex_next(lex);
- dump_emit(out, c, &t);
- if (t.kind == TOK_EOF) break;
- }
- lex_close(lex);
- return CFREE_OK;
-}
-
-CfreeStatus cfree_c_dump_tokens(CfreeCompiler* c, const CfreeBytes* input,
- CfreeWriter* out) {
- CDumpTokensRun r;
- r.input = input;
- r.out = out;
- return cfree_frontend_run(c, c_dump_tokens_body, &r);
-}
-
static CfreeFrontendState* c_frontend_new(CfreeCompiler* c) {
CfreeHeap* h;
CFrontend* fe;
diff --git a/lang/c/c.h b/lang/c/c.h
@@ -13,9 +13,9 @@
* frontend casts language_options back to CfreeCCompileOptions* to recover
* the preprocessor and diagnostic configuration.
*
- * cfree_c_preprocess and cfree_c_dump_tokens are standalone helpers driven
- * by the driver's preprocess-only and -E modes; they run under the same
- * frontend panic boundary as cfree_c_compile via cfree_frontend_run. */
+ * cfree_c_preprocess is a standalone helper driven by the driver's -E
+ * mode and by -S/.s preprocessing; it runs under the same frontend panic
+ * boundary as cfree_c_compile via cfree_frontend_run. */
#include <cfree/compile.h>
#include <cfree/core.h>
@@ -24,8 +24,6 @@
CfreeStatus cfree_c_preprocess(CfreeCompiler*, const CfreePreprocessOptions*,
const CfreeBytes*, CfreeWriter*);
-CfreeStatus cfree_c_dump_tokens(CfreeCompiler*, const CfreeBytes*,
- CfreeWriter*);
extern const CfreeFrontendVTable cfree_c_frontend_vtable;
diff --git a/test/lex/cases/basic_punct.c b/test/lex/cases/basic_punct.c
@@ -1,3 +0,0 @@
-int main(void) {
- return x->y;
-}
diff --git a/test/lex/cases/basic_punct.expected b/test/lex/cases/basic_punct.expected
@@ -1,16 +0,0 @@
-(ident int)
-(ident main)
-(punct ()
-(ident void)
-(punct ))
-(punct {)
-(newline)
-(ident return)
-(ident x)
-(punct ->)
-(ident y)
-(punct ;)
-(newline)
-(punct })
-(newline)
-(eof)
diff --git a/test/lex/cases/comment_edges.c b/test/lex/cases/comment_edges.c
@@ -1,12 +0,0 @@
-/* /* */ x */
-/* / */ y
-/* * */ z
-/* foo **/ w
-/* a *//* b */ c
-/* /// not line comment */ k
-/* "looks like string" */ m
-/*****/
-/**//**/
-/* x
- y
- z */ n
diff --git a/test/lex/cases/comment_edges.expected b/test/lex/cases/comment_edges.expected
@@ -1,21 +0,0 @@
-(ident x)
-(punct *)
-(punct /)
-(newline)
-(ident y)
-(newline)
-(ident z)
-(newline)
-(ident w)
-(newline)
-(ident c)
-(newline)
-(ident k)
-(newline)
-(ident m)
-(newline)
-(newline)
-(newline)
-(ident n)
-(newline)
-(eof)
diff --git a/test/lex/cases/comments.c b/test/lex/cases/comments.c
@@ -1,15 +0,0 @@
-// just a line comment
-int x; /* block */ int y;
-/* multi
- line
- line */
-int z;
-a /**/ b
-c // tail comment
-d /* with * and / inside */ e
-// "string-like" text and /* nested-looking */ stays comment
-f /* contains // line-comment chars */ g
-h/**/i
-j/* one *//* two */k
-/*/ slash-star-slash content */ m
-// no trailing newline at EOF
-\ No newline at end of file
diff --git a/test/lex/cases/comments.expected b/test/lex/cases/comments.expected
@@ -1,34 +0,0 @@
-(newline)
-(ident int)
-(ident x)
-(punct ;)
-(ident int)
-(ident y)
-(punct ;)
-(newline)
-(newline)
-(ident int)
-(ident z)
-(punct ;)
-(newline)
-(ident a)
-(ident b)
-(newline)
-(ident c)
-(newline)
-(ident d)
-(ident e)
-(newline)
-(newline)
-(ident f)
-(ident g)
-(newline)
-(ident h)
-(ident i)
-(newline)
-(ident j)
-(ident k)
-(newline)
-(ident m)
-(newline)
-(eof)
diff --git a/test/lex/cases/empty.c b/test/lex/cases/empty.c
diff --git a/test/lex/cases/empty.expected b/test/lex/cases/empty.expected
@@ -1 +0,0 @@
-(eof)
diff --git a/test/lex/cases/float_constants.c b/test/lex/cases/float_constants.c
@@ -1,34 +0,0 @@
-1.0
-1.
-.1
-0.0
-3.14
-1e0
-1E10
-1e+1
-1e-1
-1.5e+2
-1.5e-2
-.5e10
-1.e10
-1.E-5
-.0
-.0L
-1.0f
-1.0F
-1.0l
-1.0L
-2.5f
-6.022e23L
-1.0Lf
-0x1p0
-0x1P0
-0X1P0
-0x1p+0
-0x1P-2
-0xFp-1
-0x1.8p+1
-0x.8p1
-0xA.Bp+3
-0x1.8
-0x1.
diff --git a/test/lex/cases/float_constants.expected b/test/lex/cases/float_constants.expected
@@ -1,69 +0,0 @@
-(flt 1.0)
-(newline)
-(flt 1.)
-(newline)
-(flt .1)
-(newline)
-(flt 0.0)
-(newline)
-(flt 3.14)
-(newline)
-(flt 1e0)
-(newline)
-(flt 1E10)
-(newline)
-(flt 1e+1)
-(newline)
-(flt 1e-1)
-(newline)
-(flt 1.5e+2)
-(newline)
-(flt 1.5e-2)
-(newline)
-(flt .5e10)
-(newline)
-(flt 1.e10)
-(newline)
-(flt 1.E-5)
-(newline)
-(flt .0)
-(newline)
-(flt .0L)
-(newline)
-(flt 1.0f)
-(newline)
-(flt 1.0F)
-(newline)
-(flt 1.0l)
-(newline)
-(flt 1.0L)
-(newline)
-(flt 2.5f)
-(newline)
-(flt 6.022e23L)
-(newline)
-(flt 1.0Lf)
-(newline)
-(flt 0x1p0)
-(newline)
-(flt 0x1P0)
-(newline)
-(flt 0X1P0)
-(newline)
-(flt 0x1p+0)
-(newline)
-(flt 0x1P-2)
-(newline)
-(flt 0xFp-1)
-(newline)
-(flt 0x1.8p+1)
-(newline)
-(flt 0x.8p1)
-(newline)
-(flt 0xA.Bp+3)
-(newline)
-(flt 0x1.8)
-(newline)
-(flt 0x1.)
-(newline)
-(eof)
diff --git a/test/lex/cases/identifiers.c b/test/lex/cases/identifiers.c
@@ -1,31 +0,0 @@
-foo
-_bar
-__baz
-foo123
-A1B2C3
-_
-__
-___
-a
-Z
-_0
-_9
-__func__
-__LINE__
-__FILE__
-camelCase
-PascalCase
-SHOUTY_SNAKE
-_private99
-mixed_Case_42
-_1_2_3
-abc_def_ghi
-x0
-x1y2z3_
-naïve
-λ_func
-café
-éstart
-caf\u00e9
-\u00e9start
-with\U0001F600paste
diff --git a/test/lex/cases/identifiers.expected b/test/lex/cases/identifiers.expected
@@ -1,63 +0,0 @@
-(ident foo)
-(newline)
-(ident _bar)
-(newline)
-(ident __baz)
-(newline)
-(ident foo123)
-(newline)
-(ident A1B2C3)
-(newline)
-(ident _)
-(newline)
-(ident __)
-(newline)
-(ident ___)
-(newline)
-(ident a)
-(newline)
-(ident Z)
-(newline)
-(ident _0)
-(newline)
-(ident _9)
-(newline)
-(ident __func__)
-(newline)
-(ident __LINE__)
-(newline)
-(ident __FILE__)
-(newline)
-(ident camelCase)
-(newline)
-(ident PascalCase)
-(newline)
-(ident SHOUTY_SNAKE)
-(newline)
-(ident _private99)
-(newline)
-(ident mixed_Case_42)
-(newline)
-(ident _1_2_3)
-(newline)
-(ident abc_def_ghi)
-(newline)
-(ident x0)
-(newline)
-(ident x1y2z3_)
-(newline)
-(ident naïve)
-(newline)
-(ident λ_func)
-(newline)
-(ident café)
-(newline)
-(ident éstart)
-(newline)
-(ident caf\u00e9)
-(newline)
-(ident \u00e9start)
-(newline)
-(ident with\U0001F600paste)
-(newline)
-(eof)
diff --git a/test/lex/cases/int_constants.c b/test/lex/cases/int_constants.c
@@ -1,49 +0,0 @@
-0
-1
-123
-2147483648
-00
-0755
-01234567
-0x0
-0X0
-0xFF
-0X1AbC
-0xDEADBEEF
-1u
-1U
-1l
-1L
-1ll
-1LL
-1ul
-1uL
-1Ul
-1UL
-1lu
-1Lu
-1lU
-1LU
-1ull
-1uLL
-1Ull
-1ULL
-1llu
-1LLu
-1llU
-1LLU
-0xFFu
-0755L
-0xFFFFFFFFULL
-1lL
-1Ll
-1lll
-1LLL
-1ufL
-0b1010ULL
-123abc
-0xGHI
-0x
-0X
-077u8
-0Lu8
diff --git a/test/lex/cases/int_constants.expected b/test/lex/cases/int_constants.expected
@@ -1,99 +0,0 @@
-(num 0)
-(newline)
-(num 1)
-(newline)
-(num 123)
-(newline)
-(num 2147483648)
-(newline)
-(num 00)
-(newline)
-(num 0755)
-(newline)
-(num 01234567)
-(newline)
-(num 0x0)
-(newline)
-(num 0X0)
-(newline)
-(num 0xFF)
-(newline)
-(num 0X1AbC)
-(newline)
-(num 0xDEADBEEF)
-(newline)
-(num 1u)
-(newline)
-(num 1U)
-(newline)
-(num 1l)
-(newline)
-(num 1L)
-(newline)
-(num 1ll)
-(newline)
-(num 1LL)
-(newline)
-(num 1ul)
-(newline)
-(num 1uL)
-(newline)
-(num 1Ul)
-(newline)
-(num 1UL)
-(newline)
-(num 1lu)
-(newline)
-(num 1Lu)
-(newline)
-(num 1lU)
-(newline)
-(num 1LU)
-(newline)
-(num 1ull)
-(newline)
-(num 1uLL)
-(newline)
-(num 1Ull)
-(newline)
-(num 1ULL)
-(newline)
-(num 1llu)
-(newline)
-(num 1LLu)
-(newline)
-(num 1llU)
-(newline)
-(num 1LLU)
-(newline)
-(num 0xFFu)
-(newline)
-(num 0755L)
-(newline)
-(num 0xFFFFFFFFULL)
-(newline)
-(num 1lL)
-(newline)
-(num 1Ll)
-(newline)
-(num 1lll)
-(newline)
-(num 1LLL)
-(newline)
-(num 1ufL)
-(newline)
-(num 0b1010ULL)
-(newline)
-(num 123abc)
-(newline)
-(num 0xGHI)
-(newline)
-(num 0x)
-(newline)
-(num 0X)
-(newline)
-(num 077u8)
-(newline)
-(num 0Lu8)
-(newline)
-(eof)
diff --git a/test/lex/cases/keywords.c b/test/lex/cases/keywords.c
@@ -1,8 +0,0 @@
-auto break case char const continue default do double
-else enum extern float for goto if inline int long
-register restrict return short signed sizeof static struct
-switch typedef union unsigned void volatile while
-_Alignas _Alignof _Atomic _Bool _Complex _Generic
-_Imaginary _Noreturn _Static_assert _Thread_local
-INT Int iNT _alignas _alignof _atomic _bool
-ints intt return0 returns ifx whilez forX gotoo
diff --git a/test/lex/cases/keywords.expected b/test/lex/cases/keywords.expected
@@ -1,68 +0,0 @@
-(ident auto)
-(ident break)
-(ident case)
-(ident char)
-(ident const)
-(ident continue)
-(ident default)
-(ident do)
-(ident double)
-(newline)
-(ident else)
-(ident enum)
-(ident extern)
-(ident float)
-(ident for)
-(ident goto)
-(ident if)
-(ident inline)
-(ident int)
-(ident long)
-(newline)
-(ident register)
-(ident restrict)
-(ident return)
-(ident short)
-(ident signed)
-(ident sizeof)
-(ident static)
-(ident struct)
-(newline)
-(ident switch)
-(ident typedef)
-(ident union)
-(ident unsigned)
-(ident void)
-(ident volatile)
-(ident while)
-(newline)
-(ident _Alignas)
-(ident _Alignof)
-(ident _Atomic)
-(ident _Bool)
-(ident _Complex)
-(ident _Generic)
-(newline)
-(ident _Imaginary)
-(ident _Noreturn)
-(ident _Static_assert)
-(ident _Thread_local)
-(newline)
-(ident INT)
-(ident Int)
-(ident iNT)
-(ident _alignas)
-(ident _alignof)
-(ident _atomic)
-(ident _bool)
-(newline)
-(ident ints)
-(ident intt)
-(ident return0)
-(ident returns)
-(ident ifx)
-(ident whilez)
-(ident forX)
-(ident gotoo)
-(newline)
-(eof)
diff --git a/test/lex/cases/line_splice.c b/test/lex/cases/line_splice.c
@@ -1,25 +0,0 @@
-he\
-llo
-"foo\
-bar"
-12\
-345
-0x1\
-2.3\
-p+\
-4
-in\
-t ma\
-in
-// foo\
-still
-end
-/* block\
-comment */ y
-a<\
-<b
-str\
-1
-\
-\
-trailing
diff --git a/test/lex/cases/line_splice.expected b/test/lex/cases/line_splice.expected
@@ -1,25 +0,0 @@
-(ident hello)
-(newline)
-(str "foobar")
-(newline)
-(num 12345)
-(newline)
-(flt 0x12.3p+4)
-(newline)
-(ident int)
-(ident main)
-(newline)
-(newline)
-(ident end)
-(newline)
-(ident y)
-(newline)
-(ident a)
-(punct <<)
-(ident b)
-(newline)
-(ident str1)
-(newline)
-(ident trailing)
-(newline)
-(eof)
diff --git a/test/lex/cases/maximal_munch.c b/test/lex/cases/maximal_munch.c
@@ -1,21 +0,0 @@
-+++a
-a---b
-x<<=y
-x>>=y
-a<<b
-a>>b
-p->q
-i==j
-i!=j
-i<=j
-i>=j
-&&||
-...x
-..x
-.x
-x+++++y
-<::>
-<:::
-<%%>
-%:%:
-%:%
diff --git a/test/lex/cases/maximal_munch.expected b/test/lex/cases/maximal_munch.expected
@@ -1,80 +0,0 @@
-(punct ++)
-(punct +)
-(ident a)
-(newline)
-(ident a)
-(punct --)
-(punct -)
-(ident b)
-(newline)
-(ident x)
-(punct <<=)
-(ident y)
-(newline)
-(ident x)
-(punct >>=)
-(ident y)
-(newline)
-(ident a)
-(punct <<)
-(ident b)
-(newline)
-(ident a)
-(punct >>)
-(ident b)
-(newline)
-(ident p)
-(punct ->)
-(ident q)
-(newline)
-(ident i)
-(punct ==)
-(ident j)
-(newline)
-(ident i)
-(punct !=)
-(ident j)
-(newline)
-(ident i)
-(punct <=)
-(ident j)
-(newline)
-(ident i)
-(punct >=)
-(ident j)
-(newline)
-(punct &&)
-(punct ||)
-(newline)
-(punct ...)
-(ident x)
-(newline)
-(punct .)
-(punct .)
-(ident x)
-(newline)
-(punct .)
-(ident x)
-(newline)
-(ident x)
-(punct ++)
-(punct ++)
-(punct +)
-(ident y)
-(newline)
-(punct <:)
-(punct :>)
-(newline)
-(punct <:)
-(punct :)
-(punct :)
-(newline)
-(punct <%)
-(punct %>)
-(newline)
-(pp-paste)
-(newline)
-(pp-hash)
-(punct %)
-(newline)
-(eof)
diff --git a/test/lex/cases/pp_directives.c b/test/lex/cases/pp_directives.c
@@ -1,21 +0,0 @@
-#include "bar.h"
-#include <foo.h>
-#define MAX 100
-#define ID(x) x
-#define STR(x) #x
-#define CAT(a, b) a ## b
-#undef MAX
-#ifdef X
-#endif
-#ifndef Y
-#else
-#endif
-#if 1 + 2
-#elif 3
-#endif
-#error msg
-#pragma once
-#line 42 "f.c"
-#
-%:include "x.h"
-a %:%: b
diff --git a/test/lex/cases/pp_directives.expected b/test/lex/cases/pp_directives.expected
@@ -1,100 +0,0 @@
-(pp-hash)
-(ident include)
-(header "bar.h")
-(newline)
-(pp-hash)
-(ident include)
-(header <foo.h>)
-(newline)
-(pp-hash)
-(ident define)
-(ident MAX)
-(num 100)
-(newline)
-(pp-hash)
-(ident define)
-(ident ID)
-(punct ()
-(ident x)
-(punct ))
-(ident x)
-(newline)
-(pp-hash)
-(ident define)
-(ident STR)
-(punct ()
-(ident x)
-(punct ))
-(pp-hash)
-(ident x)
-(newline)
-(pp-hash)
-(ident define)
-(ident CAT)
-(punct ()
-(ident a)
-(punct ,)
-(ident b)
-(punct ))
-(ident a)
-(pp-paste)
-(ident b)
-(newline)
-(pp-hash)
-(ident undef)
-(ident MAX)
-(newline)
-(pp-hash)
-(ident ifdef)
-(ident X)
-(newline)
-(pp-hash)
-(ident endif)
-(newline)
-(pp-hash)
-(ident ifndef)
-(ident Y)
-(newline)
-(pp-hash)
-(ident else)
-(newline)
-(pp-hash)
-(ident endif)
-(newline)
-(pp-hash)
-(ident if)
-(num 1)
-(punct +)
-(num 2)
-(newline)
-(pp-hash)
-(ident elif)
-(num 3)
-(newline)
-(pp-hash)
-(ident endif)
-(newline)
-(pp-hash)
-(ident error)
-(ident msg)
-(newline)
-(pp-hash)
-(ident pragma)
-(ident once)
-(newline)
-(pp-hash)
-(ident line)
-(num 42)
-(str "f.c")
-(newline)
-(pp-hash)
-(newline)
-(pp-hash)
-(ident include)
-(header "x.h")
-(newline)
-(ident a)
-(pp-paste)
-(ident b)
-(newline)
-(eof)
diff --git a/test/lex/cases/pp_numbers.c b/test/lex/cases/pp_numbers.c
@@ -1,29 +0,0 @@
-0..1
-1...3
-.5.
-.5..6
-1e+1e+1
-123abc
-0xGHI
-1ea
-1e+x
-1e+
-1.e
-.5e
-0xAp
-0xAp+
-0xAp+x
-.
-.x
-. 5
-.5
-1...
-1.2.3.4
-0xA.Bp+3
-0xFFp+2.5
-99e
-99e+
-99e9
-99E-9
-1_underscore
-3.14_pi
diff --git a/test/lex/cases/pp_numbers.expected b/test/lex/cases/pp_numbers.expected
@@ -1,61 +0,0 @@
-(flt 0..1)
-(newline)
-(flt 1...3)
-(newline)
-(flt .5.)
-(newline)
-(flt .5..6)
-(newline)
-(flt 1e+1e+1)
-(newline)
-(num 123abc)
-(newline)
-(num 0xGHI)
-(newline)
-(num 1ea)
-(newline)
-(flt 1e+x)
-(newline)
-(flt 1e+)
-(newline)
-(flt 1.e)
-(newline)
-(flt .5e)
-(newline)
-(flt 0xAp)
-(newline)
-(flt 0xAp+)
-(newline)
-(flt 0xAp+x)
-(newline)
-(punct .)
-(newline)
-(punct .)
-(ident x)
-(newline)
-(punct .)
-(num 5)
-(newline)
-(flt .5)
-(newline)
-(flt 1...)
-(newline)
-(flt 1.2.3.4)
-(newline)
-(flt 0xA.Bp+3)
-(newline)
-(flt 0xFFp+2.5)
-(newline)
-(num 99e)
-(newline)
-(flt 99e+)
-(newline)
-(flt 99e9)
-(newline)
-(flt 99E-9)
-(newline)
-(num 1_underscore)
-(newline)
-(flt 3.14_pi)
-(newline)
-(eof)
diff --git a/test/lex/cases/pp_passthrough.c b/test/lex/cases/pp_passthrough.c
@@ -1,2 +0,0 @@
-#define X 1
-#include "foo.h"
diff --git a/test/lex/cases/pp_passthrough.expected b/test/lex/cases/pp_passthrough.expected
@@ -1,10 +0,0 @@
-(pp-hash)
-(ident define)
-(ident X)
-(num 1)
-(newline)
-(pp-hash)
-(ident include)
-(header "foo.h")
-(newline)
-(eof)
diff --git a/test/lex/cases/punctuators.c b/test/lex/cases/punctuators.c
@@ -1,10 +0,0 @@
-[ ] ( ) { } . ->
-++ -- & * + - ~ !
-/ % << >> < > <= >= == != ^ | && ||
-? : ; ...
-= *= /= %= += -= <<= >>= &= ^= |=
-,
-# ##
-<: :> <% %> %: %:%:
-<:a:>
-<%b%>
diff --git a/test/lex/cases/punctuators.expected b/test/lex/cases/punctuators.expected
@@ -1,71 +0,0 @@
-(punct [)
-(punct ])
-(punct ()
-(punct ))
-(punct {)
-(punct })
-(punct .)
-(punct ->)
-(newline)
-(punct ++)
-(punct --)
-(punct &)
-(punct *)
-(punct +)
-(punct -)
-(punct ~)
-(punct !)
-(newline)
-(punct /)
-(punct %)
-(punct <<)
-(punct >>)
-(punct <)
-(punct >)
-(punct <=)
-(punct >=)
-(punct ==)
-(punct !=)
-(punct ^)
-(punct |)
-(punct &&)
-(punct ||)
-(newline)
-(punct ?)
-(punct :)
-(punct ;)
-(punct ...)
-(newline)
-(punct =)
-(punct *=)
-(punct /=)
-(punct %=)
-(punct +=)
-(punct -=)
-(punct <<=)
-(punct >>=)
-(punct &=)
-(punct ^=)
-(punct |=)
-(newline)
-(punct ,)
-(newline)
-(pp-hash)
-(pp-paste)
-(newline)
-(punct <:)
-(punct :>)
-(punct <%)
-(punct %>)
-(pp-hash)
-(pp-paste)
-(newline)
-(punct <:)
-(ident a)
-(punct :>)
-(newline)
-(punct <%)
-(ident b)
-(punct %>)
-(newline)
-(eof)
diff --git a/test/lex/cases/string_escapes.c b/test/lex/cases/string_escapes.c
@@ -1,38 +0,0 @@
-'\''
-'\"'
-'\?'
-'\\'
-'\a'
-'\b'
-'\f'
-'\n'
-'\r'
-'\t'
-'\v'
-'\0'
-'\7'
-'\077'
-'\377'
-'\x0'
-'\x41'
-'\xff'
-'\xfff'
-'\U0001F600'
-"\a\b\f\n\r\t\v\"\'\\\?"
-"\0\7\077\377"
-"\x0\x41\xff\xfff"
-"\\"
-"\""
-"a\nb"
-"tab\there"
-L"\n"
-u8"\xff"
-u"é"
-U"\U0001F600"
-'\18'
-'\779'
-"\1234"
-"\xffG"
-"\xabc\x12"
-'é'
-"é "
diff --git a/test/lex/cases/string_escapes.expected b/test/lex/cases/string_escapes.expected
@@ -1,77 +0,0 @@
-(chr '\'')
-(newline)
-(chr '\"')
-(newline)
-(chr '\?')
-(newline)
-(chr '\\')
-(newline)
-(chr '\a')
-(newline)
-(chr '\b')
-(newline)
-(chr '\f')
-(newline)
-(chr '\n')
-(newline)
-(chr '\r')
-(newline)
-(chr '\t')
-(newline)
-(chr '\v')
-(newline)
-(chr '\0')
-(newline)
-(chr '\7')
-(newline)
-(chr '\077')
-(newline)
-(chr '\377')
-(newline)
-(chr '\x0')
-(newline)
-(chr '\x41')
-(newline)
-(chr '\xff')
-(newline)
-(chr '\xfff')
-(newline)
-(chr '\U0001F600')
-(newline)
-(str "\a\b\f\n\r\t\v\"\'\\\?")
-(newline)
-(str "\0\7\077\377")
-(newline)
-(str "\x0\x41\xff\xfff")
-(newline)
-(str "\\")
-(newline)
-(str "\"")
-(newline)
-(str "a\nb")
-(newline)
-(str "tab\there")
-(newline)
-(str L"\n")
-(newline)
-(str u8"\xff")
-(newline)
-(str u"é")
-(newline)
-(str U"\U0001F600")
-(newline)
-(chr '\18')
-(newline)
-(chr '\779')
-(newline)
-(str "\1234")
-(newline)
-(str "\xffG")
-(newline)
-(str "\xabc\x12")
-(newline)
-(chr 'é')
-(newline)
-(str "é ")
-(newline)
-(eof)
diff --git a/test/lex/cases/strings_chars.c b/test/lex/cases/strings_chars.c
@@ -1,23 +0,0 @@
-""
-"hello"
-'a'
-'0'
-' '
-'ab'
-"a" "b" "c"
-"L"
-"u8"
-"u"
-"U"
-L"wide"
-u8"utf8"
-u"u16"
-U"u32"
-L'w'
-u'A'
-U'B'
-u8'a'
-L "x"
-L"x"
-'À'
-"À\U0001F600"
diff --git a/test/lex/cases/strings_chars.expected b/test/lex/cases/strings_chars.expected
@@ -1,51 +0,0 @@
-(str "")
-(newline)
-(str "hello")
-(newline)
-(chr 'a')
-(newline)
-(chr '0')
-(newline)
-(chr ' ')
-(newline)
-(chr 'ab')
-(newline)
-(str "a")
-(str "b")
-(str "c")
-(newline)
-(str "L")
-(newline)
-(str "u8")
-(newline)
-(str "u")
-(newline)
-(str "U")
-(newline)
-(str L"wide")
-(newline)
-(str u8"utf8")
-(newline)
-(str u"u16")
-(newline)
-(str U"u32")
-(newline)
-(chr L'w')
-(newline)
-(chr u'A')
-(newline)
-(chr U'B')
-(newline)
-(ident u8)
-(chr 'a')
-(newline)
-(ident L)
-(str "x")
-(newline)
-(str L"x")
-(newline)
-(chr 'À')
-(newline)
-(str "À\U0001F600")
-(newline)
-(eof)
diff --git a/test/lex/run.sh b/test/lex/run.sh
@@ -1,67 +0,0 @@
-#!/bin/sh
-# Data-driven lexer test runner.
-#
-# For each test/lex/cases/*.c, runs `cfree cc --dump-tokens` and diffs the
-# output against the matching .expected file. Leaves .actual files behind on
-# failure so they can be reviewed and copied over the expected baseline once
-# intentional output changes are validated.
-#
-# Honors $CFREE for the binary path; defaults to build/cfree relative to the
-# repo root inferred from this script's location.
-
-set -u
-
-script_dir=$(cd "$(dirname "$0")" && pwd)
-repo_root=$(cd "$script_dir/../.." && pwd)
-cases_dir="$script_dir/cases"
-
-CFREE="${CFREE:-$repo_root/build/cfree}"
-
-if [ ! -x "$CFREE" ]; then
- echo "lex: cfree binary not found at $CFREE" >&2
- exit 2
-fi
-
-pass=0
-fail=0
-failures=
-
-for src in "$cases_dir"/*.c; do
- [ -e "$src" ] || continue
- expected="${src%.c}.expected"
- actual="${src%.c}.actual"
- name=$(basename "${src%.c}")
-
- if [ ! -e "$expected" ]; then
- printf 'FAIL %s (missing %s)\n' "$name" "$(basename "$expected")"
- fail=$((fail + 1))
- failures="$failures $name"
- continue
- fi
-
- if ! "$CFREE" cc --dump-tokens "$src" -o "$actual" >/dev/null 2>&1; then
- printf 'FAIL %s (cfree exit nonzero; see %s)\n' "$name" "$(basename "$actual")"
- fail=$((fail + 1))
- failures="$failures $name"
- continue
- fi
-
- if diff -u "$expected" "$actual" >/dev/null 2>&1; then
- printf 'PASS %s\n' "$name"
- rm -f "$actual"
- pass=$((pass + 1))
- else
- printf 'FAIL %s\n' "$name"
- diff -u "$expected" "$actual" || true
- fail=$((fail + 1))
- failures="$failures $name"
- fi
-done
-
-total=$((pass + fail))
-if [ "$fail" -gt 0 ]; then
- printf '\nlex: failures:%s\n' "$failures"
- printf 'lex: %d/%d passed\n' "$pass" "$total"
- exit 1
-fi
-printf '\nlex: %d/%d passed\n' "$pass" "$total"
diff --git a/test/test.mk b/test/test.mk
@@ -2,8 +2,7 @@
#
# - test-driver: narrow CLI behavior checks that do not belong to a specific
# frontend/linker corpus. Depends on the cfree driver binary.
-# - test-lex / test-pp: C frontend runners; depend on the cfree driver
-# binary, which today fails to link (most of libcfree is header-only).
+# - test-pp: C preprocessor runner; depends on the cfree driver binary.
# - test-elf: ELF roundtrip harness in test/elf/; depends only on
# libcfree.a and compiles its own test binaries against it. Skipped
# layers are reported (set CFREE_TEST_ALLOW_SKIP=1 to allow skips).
@@ -27,9 +26,9 @@
# asm_parse / cfree_disasm_iter_* are still stubs; the harness builds
# and runs end-to-end so the wiring stays exercised. See doc/ASM.md.
-.PHONY: test test-driver test-lex test-pp test-pp-err test-elf test-coff test-coff-mingw-import test-coff-windows-ucrt test-ar test-ar-driver test-strip-driver test-objcopy-driver test-objdump-driver test-link test-cg-api test-toy test-opt test-dwarf test-debug test-parse test-parse-err test-asm test-wasm-front test-isa test-aa64-inline test-rv64-inline test-rv64-jit test-emu test-x64-inline test-x64-dbg test-rt-headers test-rt-runtime test-musl test-musl-rv64 test-glibc test-glibc-rv64 test-lib-deps test-smoke-x64 test-smoke-rv64 test-cbackend rv64-doctor
+.PHONY: test test-driver test-pp test-pp-err test-elf test-coff test-coff-mingw-import test-coff-windows-ucrt test-ar test-ar-driver test-strip-driver test-objcopy-driver test-objdump-driver test-link test-cg-api test-toy test-opt test-dwarf test-debug test-parse test-parse-err test-asm test-wasm-front test-isa test-aa64-inline test-rv64-inline test-rv64-jit test-emu test-x64-inline test-x64-dbg test-rt-headers test-rt-runtime test-musl test-musl-rv64 test-glibc test-glibc-rv64 test-lib-deps test-smoke-x64 test-smoke-rv64 test-cbackend rv64-doctor
-test: test-driver test-lex test-pp test-pp-err test-elf test-coff test-ar test-ar-driver test-strip-driver test-objcopy-driver test-objdump-driver test-link test-toy test-dwarf test-debug test-parse test-parse-err test-asm test-isa test-aa64-inline test-rv64-inline test-rv64-jit test-emu test-x64-inline test-x64-dbg test-rt-headers test-lib-deps
+test: test-driver test-pp test-pp-err test-elf test-coff test-ar test-ar-driver test-strip-driver test-objcopy-driver test-objdump-driver test-link test-toy test-dwarf test-debug test-parse test-parse-err test-asm test-isa test-aa64-inline test-rv64-inline test-rv64-jit test-emu test-x64-inline test-x64-dbg test-rt-headers test-lib-deps
# `test-cbackend` is intentionally not in the default `test` target: the
# Phase 1 C backend skips most fixtures pending later phases, which would
# add noise to the default summary. Run it explicitly to gate progress.
@@ -48,9 +47,6 @@ test-cbackend: bin
@CFREE_TEST_PATHS=C CFREE=$(abspath $(BIN)) sh test/toy/run.sh
@CFREE_TEST_PATHS=C CFREE=$(abspath $(BIN)) bash test/wasm/run.sh
-test-lex: bin
- @CFREE=$(abspath $(BIN)) test/lex/run.sh
-
test-pp: bin
@CFREE=$(abspath $(BIN)) test/pp/run.sh
@@ -380,9 +376,9 @@ test-parse-err: lib $(PARSE_RUNNER)
sh test/parse/run_errors.sh
# test-asm: builds asm-runner inside the run.sh driver (the harness owns
-# its compile rule, mirroring test/lex/test/pp). Phase 1: every smoke
-# case carries a .skip sidecar; the harness wiring runs on every CI run
-# so regressions in the public asm/disasm surface are caught early.
+# its compile rule, mirroring test/pp). Phase 1: every smoke case
+# carries a .skip sidecar; the harness wiring runs on every CI run so
+# regressions in the public asm/disasm surface are caught early.
test-asm: lib
bash test/asm/run.sh
