commit 484bda40cbd5795ea697cb19404e6da34e33888d
parent 8cd4bd32bec1a5404143f121282da56f43474145
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Thu, 18 Jun 2026 02:34:02 -0700
build: test coordinator shell recipes
Diffstat:
6 files changed, 302 insertions(+), 34 deletions(-)
diff --git a/driver/cmd/build_coord.c b/driver/cmd/build_coord.c
@@ -36,10 +36,10 @@ void driver_help_build(void) {
"USAGE\n"
" kit build [--store DIR] [--root DIR] [--def FILE]\n"
" [--config K=V]... [--arg VALUE]... [--verify] TARGET\n"
- " kit build --client config-get KEY\n"
- " kit build --client source PATH\n"
- " kit build --client glob PATTERN\n"
- " kit build --client need [--config K=V]... [--arg VALUE]... TARGET\n"
+ " kit build config-get KEY # inside a build recipe\n"
+ " kit build source PATH # inside a build recipe\n"
+ " kit build glob PATTERN # inside a build recipe\n"
+ " kit build need [--config K=V]... TARGET # inside a build recipe\n"
"\n"
"OPTIONS\n"
" --store DIR Build store root (default: $KIT cache/build)\n"
@@ -99,10 +99,16 @@ static int build_client_glob_print(void* user, KitSlice path) {
return 0;
}
-static int build_client_need_parse(BuildCli* cli, int argc, char** argv) {
+static int build_client_verb(const char* s) {
+ return driver_streq(s, "config-get") || driver_streq(s, "source") ||
+ driver_streq(s, "glob") || driver_streq(s, "need");
+}
+
+static int build_client_need_parse(BuildCli* cli, int argc, char** argv,
+ int first_arg) {
int i;
memset(cli, 0, sizeof *cli);
- for (i = 3; i < argc; ++i) {
+ for (i = first_arg; i < argc; ++i) {
const char* a = argv[i];
if (driver_streq(a, "--config") && i + 1 < argc) {
if (build_parse_config(cli, argv[++i]) != 0) {
@@ -132,7 +138,7 @@ static int build_client_need_parse(BuildCli* cli, int argc, char** argv) {
return 0;
}
-static int build_client_mode(int argc, char** argv) {
+static int build_client_mode(int argc, char** argv, int verb_index) {
DriverEnv env;
DriverBuildHost bh;
KitContext ctx;
@@ -140,11 +146,12 @@ static int build_client_mode(int argc, char** argv) {
KitStatus st;
int rc = 1;
const char* verb;
- if (argc < 3) {
+ int first_arg = verb_index + 1;
+ if (argc <= verb_index) {
driver_errf(BUILD_TOOL, "missing client command");
return 2;
}
- verb = argv[2];
+ verb = argv[verb_index];
driver_env_init(&env);
ctx = driver_env_to_context(&env);
if (driver_build_host_init(&bh, &env) != 0) {
@@ -160,13 +167,13 @@ static int build_client_mode(int argc, char** argv) {
if (driver_streq(verb, "config-get")) {
KitSlice value;
int present = 0;
- if (argc != 4) {
- driver_errf(BUILD_TOOL, "usage: kit build --client config-get KEY");
+ if (argc != first_arg + 1) {
+ driver_errf(BUILD_TOOL, "usage: kit build config-get KEY");
rc = 2;
goto out_client;
}
- st = kit_build_client_config_get(client, kit_slice_cstr(argv[3]), &value,
- &present);
+ st = kit_build_client_config_get(client, kit_slice_cstr(argv[first_arg]),
+ &value, &present);
if (st != KIT_OK) {
driver_errf(BUILD_TOOL, "config-get failed: %s", build_status_name(st));
goto out_client;
@@ -180,12 +187,13 @@ static int build_client_mode(int argc, char** argv) {
} else if (driver_streq(verb, "source")) {
uint8_t blob[KIT_BUILD_HASH_LEN];
KitSlice path;
- if (argc != 4) {
- driver_errf(BUILD_TOOL, "usage: kit build --client source PATH");
+ if (argc != first_arg + 1) {
+ driver_errf(BUILD_TOOL, "usage: kit build source PATH");
rc = 2;
goto out_client;
}
- st = kit_build_client_source(client, kit_slice_cstr(argv[3]), blob, &path);
+ st = kit_build_client_source(client, kit_slice_cstr(argv[first_arg]), blob,
+ &path);
if (st != KIT_OK) {
driver_errf(BUILD_TOOL, "source failed: %s", build_status_name(st));
goto out_client;
@@ -193,12 +201,12 @@ static int build_client_mode(int argc, char** argv) {
driver_printf("%.*s\n", KIT_SLICE_ARG(path));
rc = 0;
} else if (driver_streq(verb, "glob")) {
- if (argc != 4) {
- driver_errf(BUILD_TOOL, "usage: kit build --client glob PATTERN");
+ if (argc != first_arg + 1) {
+ driver_errf(BUILD_TOOL, "usage: kit build glob PATTERN");
rc = 2;
goto out_client;
}
- st = kit_build_client_glob(client, kit_slice_cstr(argv[3]),
+ st = kit_build_client_glob(client, kit_slice_cstr(argv[first_arg]),
build_client_glob_print, NULL);
if (st != KIT_OK) {
driver_errf(BUILD_TOOL, "glob failed: %s", build_status_name(st));
@@ -209,7 +217,7 @@ static int build_client_mode(int argc, char** argv) {
BuildCli cli;
KitBuildRequest req;
KitBuildResult result;
- rc = build_client_need_parse(&cli, argc, argv);
+ rc = build_client_need_parse(&cli, argc, argv, first_arg);
if (rc != 0) goto out_client;
memset(&req, 0, sizeof req);
req.target = kit_slice_cstr(cli.target);
@@ -328,7 +336,10 @@ int driver_build(int argc, char** argv) {
return 0;
}
if (argc >= 2 && driver_streq(argv[1], "--client"))
- return build_client_mode(argc, argv);
+ return build_client_mode(argc, argv, 2);
+ if (argc >= 2 && driver_getenv(KIT_BUILD_ENV_SOCK) &&
+ build_client_verb(argv[1]))
+ return build_client_mode(argc, argv, 1);
rc = build_parse_args(&cli, argc, argv);
if (rc != 0) return rc;
diff --git a/driver/env/build_host_posix.c b/driver/env/build_host_posix.c
@@ -9,6 +9,7 @@
#include <stdint.h>
#include <stdio.h>
#include <string.h>
+#include <sys/select.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/un.h>
@@ -242,6 +243,28 @@ static int db_exec_wait(void* user, KitBuildProc* proc, int* exit_code) {
return 0;
}
+static int db_exec_poll(void* user, KitBuildProc* proc, int* done,
+ int* exit_code) {
+ DriverBuildHost* bh = (DriverBuildHost*)user;
+ int st = 0;
+ pid_t r;
+ if (!bh || !proc || !done || !exit_code) return 1;
+ *done = 0;
+ do {
+ r = waitpid(proc->pid, &st, WNOHANG);
+ } while (r < 0 && errno == EINTR);
+ if (r < 0) return 1;
+ if (r == 0) return 0;
+ *done = 1;
+ if (WIFEXITED(st)) {
+ *exit_code = WEXITSTATUS(st);
+ } else {
+ *exit_code = 128;
+ }
+ driver_free(bh->env, proc, sizeof *proc);
+ return 0;
+}
+
static void db_exec_kill(void* user, KitBuildProc* proc) {
(void)user;
if (proc) kill(proc->pid, SIGTERM);
@@ -334,6 +357,41 @@ static int db_transport_accept(void* user, KitBuildListener* lst,
return 0;
}
+static int db_transport_try_accept(void* user, KitBuildListener* lst,
+ KitBuildConn** out, int* ready) {
+ DriverBuildHost* bh = (DriverBuildHost*)user;
+ KitBuildConn* c;
+ fd_set fds;
+ struct timeval tv;
+ int r;
+ int fd;
+ if (!bh || !lst || !out || !ready) return 1;
+ *out = NULL;
+ *ready = 0;
+ FD_ZERO(&fds);
+ FD_SET(lst->fd, &fds);
+ tv.tv_sec = 0;
+ tv.tv_usec = 10000;
+ do {
+ r = select(lst->fd + 1, &fds, NULL, NULL, &tv);
+ } while (r < 0 && errno == EINTR);
+ if (r < 0) return 1;
+ if (r == 0) return 0;
+ do {
+ fd = accept(lst->fd, NULL, NULL);
+ } while (fd < 0 && errno == EINTR);
+ if (fd < 0) return 1;
+ c = (KitBuildConn*)driver_alloc_zeroed(bh->env, sizeof *c);
+ if (!c) {
+ close(fd);
+ return 1;
+ }
+ c->fd = fd;
+ *out = c;
+ *ready = 1;
+ return 0;
+}
+
static void db_transport_close_listener(void* user, KitBuildListener* lst) {
DriverBuildHost* bh = (DriverBuildHost*)user;
if (!bh || !lst) return;
@@ -418,10 +476,12 @@ int driver_build_host_init(DriverBuildHost* out, DriverEnv* env) {
out->store_io.user = out;
out->exec.spawn = db_exec_spawn;
out->exec.wait = db_exec_wait;
+ out->exec.poll = db_exec_poll;
out->exec.kill = db_exec_kill;
out->exec.user = out;
out->transport.listen = db_transport_listen;
out->transport.accept = db_transport_accept;
+ out->transport.try_accept = db_transport_try_accept;
out->transport.close_listener = db_transport_close_listener;
out->transport.dial = db_transport_dial;
out->transport.read_frame = db_transport_read;
diff --git a/include/kit/build_coord.h b/include/kit/build_coord.h
@@ -81,9 +81,13 @@ typedef struct KitBuildListener KitBuildListener;
*
* Server lifecycle, per recipe: `listen` creates a fresh endpoint and reports
* its name (the coordinator puts that name in the child's $KIT_BUILD_SOCK);
- * `accept` waits for the recipe to dial in; the connection is serviced; then
- * `close` drops the connection and `close_listener` tears down the endpoint.
- * Per-recipe endpoints are what let many recipes run concurrently. */
+ * `accept` waits for a recipe client to dial in; the connection is serviced;
+ * then `close` drops that connection and `close_listener` tears down the
+ * endpoint. Shell recipes commonly invoke a helper process once per request,
+ * creating several sequential connections. Hosts that can support that should
+ * also provide `try_accept`, which waits briefly/polls for one connection and
+ * reports ready=0 when none is available yet. Per-recipe endpoints are what let
+ * many recipes run concurrently. */
typedef struct KitBuildTransport {
/* Server: create a uniquely-named endpoint for one recipe. The name to hand
* the child (via $KIT_BUILD_SOCK) is written NUL-terminated into name_out
@@ -91,6 +95,11 @@ typedef struct KitBuildTransport {
int (*listen)(void* user, char* name_out, size_t cap, KitBuildListener** out);
/* Server: accept the recipe's one connection on `lst`. */
int (*accept)(void* user, KitBuildListener* lst, KitBuildConn** out);
+ /* Optional server poll: ready=1 and *out set when a connection was accepted;
+ * ready=0 when no client is waiting yet. A small internal timeout is allowed
+ * to avoid busy-spinning. */
+ int (*try_accept)(void* user, KitBuildListener* lst, KitBuildConn** out,
+ int* ready);
/* Server: tear down a listener once its connection is closed. */
void (*close_listener)(void* user, KitBuildListener* lst);
/* Client: connect to the endpoint named in $KIT_BUILD_SOCK. */
@@ -120,6 +129,10 @@ typedef struct KitBuildExec {
KitBuildProc** out);
/* Block until `proc` exits; *exit_code receives its status; frees `proc`. */
int (*wait)(void* user, KitBuildProc* proc, int* exit_code);
+ /* Optional nonblocking status check for recipe processes. On exit, sets
+ * *done=1, stores the exit status, and frees `proc` exactly as wait does.
+ * Otherwise sets *done=0 and leaves `proc` live. */
+ int (*poll)(void* user, KitBuildProc* proc, int* done, int* exit_code);
/* Best-effort terminate (cancel on cycle/abort); `wait` still reaps. */
void (*kill)(void* user, KitBuildProc* proc);
void* user;
diff --git a/mk/test.mk b/mk/test.mk
@@ -72,6 +72,7 @@ TEST_TARGETS = \
test-driver-cas \
test-driver-cc \
test-driver-build \
+ test-driver-build-coord \
test-driver-lsan \
test-driver-objcopy \
test-driver-objdump \
@@ -326,7 +327,7 @@ test-images:
test-cf-corpus-selftest:
@bash test/lib/kit_corpus_selftest.sh
-test-driver: test-driver-cc test-driver-build test-driver-ar test-driver-cpio test-driver-cas test-driver-strip test-driver-objcopy test-driver-objdump test-driver-pkg test-driver-strings test-driver-tools test-driver-wasm
+test-driver: test-driver-cc test-driver-build test-driver-build-coord test-driver-ar test-driver-cpio test-driver-cas test-driver-strip test-driver-objcopy test-driver-objdump test-driver-pkg test-driver-strings test-driver-tools test-driver-wasm
test-driver-cc: bin
@KIT=$(abspath $(BIN)) sh test/driver/run.sh
@@ -334,6 +335,9 @@ test-driver-cc: bin
test-driver-build: bin
@KIT=$(abspath $(BIN)) sh test/buildcmds/run.sh
+test-driver-build-coord: bin
+ @KIT=$(abspath $(BIN)) sh test/buildcoord/run.sh
+
BUILD_PURE_TEST_BIN = build/test/build_pure_test
BUILD_PUBLIC_LINK_TEST_BIN = build/test/build_public_link_test
diff --git a/src/build/runner.c b/src/build/runner.c
@@ -183,6 +183,52 @@ out_close:
return ok;
}
+static int service_recipe_connections(KitBuildCoordinator* c,
+ KitBuildListener* listener,
+ KitBuildProc** proc, KitSlice target,
+ const BuildConfig* cfg,
+ const BuildChainFrame* chain,
+ BuildDepLog* log, int* exit_code) {
+ int done = 0;
+ KitBuildConn* conn = NULL;
+ if (!c || !listener || !proc || !*proc || !cfg || !log || !exit_code)
+ return BUILD_ERR;
+ if (!c->host.transport->try_accept || !c->host.exec->poll) {
+ if (c->host.transport->accept(c->host.transport->user, listener, &conn) !=
+ 0 ||
+ !conn)
+ return BUILD_ERR;
+ if (build_runner_service(c, conn, target, cfg, chain, log) != BUILD_OK) {
+ c->host.transport->close(c->host.transport->user, conn);
+ return BUILD_ERR;
+ }
+ c->host.transport->close(c->host.transport->user, conn);
+ if (c->host.exec->wait(c->host.exec->user, *proc, exit_code) != 0)
+ return BUILD_ERR;
+ *proc = NULL;
+ return BUILD_OK;
+ }
+ while (!done) {
+ int ready = 0;
+ conn = NULL;
+ if (c->host.exec->poll(c->host.exec->user, *proc, &done, exit_code) != 0)
+ return BUILD_ERR;
+ if (done) break;
+ if (c->host.transport->try_accept(c->host.transport->user, listener, &conn,
+ &ready) != 0)
+ return BUILD_ERR;
+ if (!ready) continue;
+ if (!conn) return BUILD_ERR;
+ if (build_runner_service(c, conn, target, cfg, chain, log) != BUILD_OK) {
+ c->host.transport->close(c->host.transport->user, conn);
+ return BUILD_ERR;
+ }
+ c->host.transport->close(c->host.transport->user, conn);
+ }
+ *proc = NULL;
+ return BUILD_OK;
+}
+
int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
const BuildConfig* cfg, const BuildArgv* argv,
const BuildChainFrame* chain, BuildResolved* out) {
@@ -254,9 +300,6 @@ int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
!proc)
goto out_cleanup;
build_coord_stat_bump(c, BUILD_STAT_RECIPE_RUN);
- if (c->host.transport->accept(c->host.transport->user, listener, &conn) != 0 ||
- !conn)
- goto out_kill;
memset(&log, 0, sizeof log);
log.config_keys = config_keys;
@@ -272,13 +315,9 @@ int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
log.pending = pending;
log.cap_pending = sizeof pending / sizeof pending[0];
log.next_token = 1;
- if (build_runner_service(c, conn, target, cfg, chain, &log) != BUILD_OK)
+ if (service_recipe_connections(c, listener, &proc, target, cfg, chain, &log,
+ &exit_code) != BUILD_OK)
goto out_kill;
- c->host.transport->close(c->host.transport->user, conn);
- conn = NULL;
- if (c->host.exec->wait(c->host.exec->user, proc, &exit_code) != 0)
- goto out_cleanup;
- proc = NULL;
if (exit_code != 0) goto out_cleanup;
if (build_store_ingest_output(&c->store, out_dir, out->output_tree, out->path,
sizeof out->path) != BUILD_OK)
diff --git a/test/buildcoord/run.sh b/test/buildcoord/run.sh
@@ -0,0 +1,141 @@
+#!/bin/sh
+# Driver-level checks for the content-addressed build coordinator.
+
+set -u
+
+script_dir=$(cd "$(dirname "$0")" && pwd)
+repo_root=$(cd "$script_dir/../.." && pwd)
+
+KIT="${KIT:-$repo_root/build/kit}"
+
+if [ ! -x "$KIT" ]; then
+ echo "buildcoord: kit binary not found at $KIT" >&2
+ exit 2
+fi
+
+work=$(mktemp -d "${TMPDIR:-/tmp}/kit-buildcoord-test.XXXXXX")
+trap 'rm -rf "$work"' EXIT
+
+KIT_KIT_DIR="$repo_root/test/lib"
+. "$repo_root/test/lib/kit_sh_kit.sh"
+kit_report_init
+
+ws="$work/ws"
+store="$work/store"
+mkdir -p "$ws/recipes" "$ws/src" "$store"
+
+cat > "$ws/BUILD.kit" <<'EOF'
+kit-build 1
+[target //app:bundle]
+recipe recipes/app.sh
+[target //lib:data]
+recipe recipes/lib.sh
+EOF
+
+cat > "$ws/src/a.txt" <<'EOF'
+alpha
+EOF
+cat > "$ws/src/b.txt" <<'EOF'
+beta
+EOF
+
+cat > "$ws/recipes/lib.sh" <<'EOF'
+#!/bin/sh
+set -eu
+mode=$("$KIT" build config-get mode || true)
+mkdir -p "$KIT_BUILD_OUT"
+printf 'lib:%s\n' "$mode" > "$KIT_BUILD_OUT/lib.txt"
+EOF
+
+cat > "$ws/recipes/app.sh" <<'EOF'
+#!/bin/sh
+set -eu
+mkdir -p "$KIT_BUILD_OUT"
+: "${KIT_BUILD_TARGET:?missing target}"
+lib_dir=$("$KIT" build need --config mode=debug //lib:data)
+for p in $("$KIT" build glob 'src/*.txt'); do
+ cat "$("$KIT" build source "$p")"
+done > "$KIT_BUILD_OUT/sources.txt"
+cat "$lib_dir/lib.txt" > "$KIT_BUILD_OUT/lib.txt"
+printf '%s\n' "$KIT_BUILD_TARGET" > "$KIT_BUILD_OUT/target.txt"
+EOF
+chmod +x "$ws/recipes/app.sh" "$ws/recipes/lib.sh"
+
+run_build() {
+ name=$1
+ shift
+ "$KIT" build --store "$store" --root "$ws" --def BUILD.kit \
+ --config "env.KIT=$KIT" "$@" \
+ > "$work/$name.out" 2> "$work/$name.err"
+}
+
+tree_path_from() {
+ awk 'NF >= 2 {print $2; exit}' "$1"
+}
+
+run_build cold //app:bundle
+if [ $? -eq 0 ]; then
+ ok "buildcoord-cold-build"
+else
+ not_ok "buildcoord-cold-build" "$work/cold.err"
+fi
+cold_path=$(tree_path_from "$work/cold.out")
+if [ -n "$cold_path" ] && [ -d "$cold_path" ]; then
+ ok "buildcoord-cold-output-path"
+else
+ echo "missing output path: $cold_path" > "$work/cold-path.diag"
+ not_ok "buildcoord-cold-output-path" "$work/cold-path.diag"
+fi
+contains "buildcoord-source-a" "$cold_path/sources.txt" "alpha"
+contains "buildcoord-source-b" "$cold_path/sources.txt" "beta"
+contains "buildcoord-need-output" "$cold_path/lib.txt" "lib:debug"
+contains "buildcoord-target-env" "$cold_path/target.txt" "//app:bundle"
+
+run_build cached //app:bundle
+if [ $? -eq 0 ]; then
+ ok "buildcoord-cache-build"
+else
+ not_ok "buildcoord-cache-build" "$work/cached.err"
+fi
+cached_path=$(tree_path_from "$work/cached.out")
+if [ "$cached_path" = "$cold_path" ]; then
+ ok "buildcoord-cache-reuses-tree"
+else
+ printf 'cold=%s\ncached=%s\n' "$cold_path" "$cached_path" \
+ > "$work/cache-tree.diag"
+ not_ok "buildcoord-cache-reuses-tree" "$work/cache-tree.diag"
+fi
+
+cat > "$ws/src/b.txt" <<'EOF'
+beta2
+EOF
+run_build changed //app:bundle
+if [ $? -eq 0 ]; then
+ ok "buildcoord-glob-change-build"
+else
+ not_ok "buildcoord-glob-change-build" "$work/changed.err"
+fi
+changed_path=$(tree_path_from "$work/changed.out")
+contains "buildcoord-glob-change-visible" "$changed_path/sources.txt" "beta2"
+if [ "$changed_path" != "$cold_path" ]; then
+ ok "buildcoord-glob-change-new-tree"
+else
+ printf 'unchanged tree path=%s\n' "$changed_path" > "$work/change-tree.diag"
+ not_ok "buildcoord-glob-change-new-tree" "$work/change-tree.diag"
+fi
+
+run_build verify --verify //app:bundle
+if [ $? -eq 0 ]; then
+ ok "buildcoord-verify"
+else
+ not_ok "buildcoord-verify" "$work/verify.err"
+fi
+
+run_fail "buildcoord-missing-target-fails" \
+ "$KIT" build --store "$store" --root "$ws" --def BUILD.kit \
+ --config "env.KIT=$KIT" //missing:target
+contains "buildcoord-missing-target-diag" "$work/buildcoord-missing-target-fails.err" \
+ "build failed"
+
+kit_summary "buildcoord"
+kit_exit
