commit c24b9e0300ad0eb206cd9dec61b78ad83e49800f
parent acf989f25bf6be87ce81b0603667600523746dee
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Thu, 18 Jun 2026 03:32:37 -0700
build: satisfy coordinator spec checks
Diffstat:
7 files changed, 185 insertions(+), 26 deletions(-)
diff --git a/driver/cmd/build_coord.c b/driver/cmd/build_coord.c
@@ -42,6 +42,9 @@ void driver_help_build(void) {
" kit build glob PATTERN # inside a build recipe\n"
" kit build need [--config K=V]... TARGET [-- ARG...]\n"
" # inside a build recipe\n"
+ " kit build need-submit [--config K=V]... TARGET [-- ARG...]\n"
+ " # inside a build recipe\n"
+ " kit build need-await TOKEN # inside a build recipe\n"
"\n"
"OPTIONS\n"
" --store DIR Build store root (default: $KIT cache/build)\n"
@@ -103,7 +106,22 @@ static int build_client_glob_print(void* user, KitSlice path) {
static int build_client_verb(const char* s) {
return driver_streq(s, "config-get") || driver_streq(s, "source") ||
- driver_streq(s, "glob") || driver_streq(s, "need");
+ driver_streq(s, "glob") || driver_streq(s, "need") ||
+ driver_streq(s, "need-submit") || driver_streq(s, "need-await");
+}
+
+static int build_parse_u64(const char* s, uint64_t* out) {
+ uint64_t v = 0;
+ size_t i;
+ if (!s || !s[0] || !out) return 1;
+ for (i = 0; s[i]; ++i) {
+ unsigned c = (unsigned char)s[i];
+ if (c < '0' || c > '9') return 1;
+ if (v > (UINT64_MAX - (uint64_t)(c - '0')) / 10u) return 1;
+ v = v * 10u + (uint64_t)(c - '0');
+ }
+ *out = v;
+ return 0;
}
static int build_client_need_parse(BuildCli* cli, int argc, char** argv,
@@ -248,6 +266,44 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
}
driver_printf("%s\n", result.path);
rc = 0;
+ } else if (driver_streq(verb, "need-submit")) {
+ BuildCli cli;
+ KitBuildRequest req;
+ KitBuildNeedToken token;
+ rc = build_client_need_parse(&cli, argc, argv, first_arg);
+ if (rc != 0) goto out_client;
+ memset(&req, 0, sizeof req);
+ req.target = kit_slice_cstr(cli.target);
+ req.config = cli.config;
+ req.nconfig = cli.nconfig;
+ req.argv = cli.args;
+ req.argc = cli.nargs;
+ st = kit_build_client_need_submit(client, &req, &token);
+ if (st != KIT_OK) {
+ driver_errf(BUILD_TOOL, "need-submit failed: %s",
+ build_status_name(st));
+ rc = 1;
+ goto out_client;
+ }
+ driver_printf("%llu\n", (unsigned long long)token.id);
+ rc = 0;
+ } else if (driver_streq(verb, "need-await")) {
+ KitBuildNeedToken token;
+ KitBuildResult result;
+ if (argc != first_arg + 1 ||
+ build_parse_u64(argv[first_arg], &token.id) != 0) {
+ driver_errf(BUILD_TOOL, "usage: kit build need-await TOKEN");
+ rc = 2;
+ goto out_client;
+ }
+ st = kit_build_client_need_await(client, token, &result);
+ if (st != KIT_OK) {
+ driver_errf(BUILD_TOOL, "need-await failed: %s", build_status_name(st));
+ rc = 1;
+ goto out_client;
+ }
+ driver_printf("%s\n", result.path);
+ rc = 0;
} else {
driver_errf(BUILD_TOOL, "unknown client command: %s", verb);
rc = 2;
diff --git a/src/build/bundle.c b/src/build/bundle.c
@@ -1245,7 +1245,6 @@ int build_trace_remote_pull(KitBuildCoordinator* c, KitSlice target) {
iopts.trusted_keys_len = r->trusted_keys.len;
iopts.tofu = r->tofu;
if (build_bundle_import(c, &iopts, &ires) == BUILD_OK && ires.n_traces) {
- build_coord_stat_bump(c, BUILD_STAT_TRACE_PULL);
release_file(c, &fd);
ok = BUILD_OK;
break;
diff --git a/src/build/resolve.c b/src/build/resolve.c
@@ -210,6 +210,29 @@ int build_materialize(KitBuildCoordinator* c,
return BUILD_ERR;
}
+static int verify_cached_hit(KitBuildCoordinator* c, KitSlice target,
+ const BuildConfig* cfg, const BuildArgv* argv,
+ const BuildChainFrame* chain,
+ const uint8_t expected[BUILD_HASH_LEN],
+ BuildResolved* out) {
+ BuildResolved fresh;
+ char want[BUILD_HEX_LEN], got[BUILD_HEX_LEN];
+ if (!c || !expected || !out) return BUILD_ERR;
+ if (!c->opts.verify) return BUILD_OK;
+ memset(&fresh, 0, sizeof fresh);
+ if (build_run_recipe_probe(c, target, cfg, argv, chain, &fresh) != BUILD_OK)
+ return BUILD_ERR;
+ if (!build_id_eq(fresh.output_tree, expected)) {
+ kit_hex_encode(want, expected, BUILD_HASH_LEN);
+ kit_hex_encode(got, fresh.output_tree, BUILD_HASH_LEN);
+ build_diagf(c->ctx,
+ "build: verify mismatch for %.*s: cached %s fresh %s",
+ KIT_SLICE_ARG(target), want, got);
+ return BUILD_ERR;
+ }
+ return BUILD_OK;
+}
+
int build_resolve(KitBuildCoordinator* c, KitSlice target, const BuildConfig* cfg,
const BuildArgv* argv, const BuildChainFrame* chain,
BuildResolved* out) {
@@ -220,6 +243,7 @@ int build_resolve(KitBuildCoordinator* c, KitSlice target, const BuildConfig* cf
BuildTargetRecord rec;
size_t i;
char err[128];
+ int pulled_remote = 0;
if (!c || !cfg || !argv || !out) return BUILD_ERR;
if (build_config_id(c->ctx->heap, cfg, config_id) != BUILD_OK ||
build_argv_id(c->ctx->heap, argv, argv_id) != BUILD_OK)
@@ -230,11 +254,20 @@ int build_resolve(KitBuildCoordinator* c, KitSlice target, const BuildConfig* cf
return BUILD_ERR;
}
if (build_target_key(target, target_key) == BUILD_OK) {
+scan_record:
memset(&rec, 0, sizeof rec);
rec.rows = rows;
rec.cap_rows = sizeof rows / sizeof rows[0];
if (build_store_record_load(&c->store, target_key, target, &rec) ==
BUILD_OK) {
+ if (rec.n_rows == 0u && !pulled_remote) {
+ int pulled_now = 0;
+ pulled_remote = 1;
+ if (build_coord_trace_remote_pull_once(c, target, &pulled_now) ==
+ BUILD_OK &&
+ pulled_now)
+ goto scan_record;
+ }
for (i = 0; i < rec.n_rows; ++i) {
KitFileData fd;
BuildDeepTrace dt;
@@ -259,6 +292,11 @@ int build_resolve(KitBuildCoordinator* c, KitSlice target, const BuildConfig* cf
BUILD_OK) {
memcpy(out->output_tree, dt.output, BUILD_HASH_LEN);
out->leafset = leaf;
+ if (verify_cached_hit(c, target, cfg, argv, child, dt.output, out) !=
+ BUILD_OK) {
+ build_store_release(&c->store, &fd);
+ return BUILD_ERR;
+ }
build_store_release(&c->store, &fd);
build_coord_stat_bump(c, BUILD_STAT_DEEP_HIT);
return BUILD_OK;
@@ -296,6 +334,11 @@ int build_resolve(KitBuildCoordinator* c, KitSlice target, const BuildConfig* cf
memcmp(st.target, target.s, target.len) == 0 &&
try_shallow_trace(c, target, &st, cfg, argv, argv_id, child, out) ==
BUILD_OK) {
+ if (verify_cached_hit(c, target, cfg, argv, child, st.output, out) !=
+ BUILD_OK) {
+ build_store_release(&c->store, &fd);
+ return BUILD_ERR;
+ }
build_store_release(&c->store, &fd);
build_coord_stat_bump(c, BUILD_STAT_SHALLOW_HIT);
return BUILD_OK;
diff --git a/src/build/resolve.h b/src/build/resolve.h
@@ -101,6 +101,14 @@ int build_run_recipe(KitBuildCoordinator*, KitSlice target,
const BuildConfig* cfg, const BuildArgv* argv,
const BuildChainFrame* chain, BuildResolved* out);
+/* Verification-only sibling of build_run_recipe: executes the recipe and
+ * ingests/materializes the fresh output tree, but writes NO shallow/deep trace
+ * and does not update the target record. Used by verify mode to audit a cache
+ * hit without replacing the cache's claim before the comparison succeeds. */
+int build_run_recipe_probe(KitBuildCoordinator*, KitSlice target,
+ const BuildConfig* cfg, const BuildArgv* argv,
+ const BuildChainFrame* chain, BuildResolved* out);
+
/* The materialization ladder shared by every cache hit:
* tree cache/ -> local CAS -> remote object fetch.
* A PURE locator: writes the materialized directory path and returns BUILD_OK,
diff --git a/src/build/runner.c b/src/build/runner.c
@@ -30,8 +30,14 @@ static int target_copy(KitSlice target, char out[BUILD_TARGET_MAX]) {
static int append_config_key(BuildDepLog* log, KitSlice key) {
BuildConfigKey* row;
+ size_t i;
if (!log || !key.s || key.len == 0u || key.len >= BUILD_KEY_MAX)
return BUILD_ERR;
+ for (i = 0; i < log->n_config_keys; ++i) {
+ if (strlen(log->config_keys[i].name) == key.len &&
+ memcmp(log->config_keys[i].name, key.s, key.len) == 0)
+ return BUILD_OK;
+ }
if (log->n_config_keys >= log->cap_config_keys) return BUILD_ERR;
row = &log->config_keys[log->n_config_keys++];
memcpy(row->name, key.s, key.len);
@@ -76,9 +82,20 @@ static int append_dep(BuildDepLog* log, KitSlice target,
const uint8_t argv_id[BUILD_HASH_LEN],
const BuildResolved* r) {
BuildDepEdge* row;
+ size_t i;
if (!log || !target.s || target.len == 0u || target.len >= BUILD_TARGET_MAX ||
!overlay_id || !config_id || !argv_id || !r || !r->leafset)
return BUILD_ERR;
+ for (i = 0; i < log->n_deps; ++i) {
+ row = &log->deps[i];
+ if (strlen(row->name) == target.len &&
+ memcmp(row->name, target.s, target.len) == 0 &&
+ build_id_eq(row->overlay_id, overlay_id) &&
+ build_id_eq(row->config_id, config_id) &&
+ build_id_eq(row->argv_id, argv_id) &&
+ build_id_eq(row->output_tree, r->output_tree))
+ return BUILD_OK;
+ }
if (log->n_deps >= log->cap_deps ||
log->n_children >= log->cap_children)
return BUILD_ERR;
@@ -231,9 +248,10 @@ static int service_recipe_connections(KitBuildCoordinator* c,
return BUILD_OK;
}
-int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
- const BuildConfig* cfg, const BuildArgv* argv,
- const BuildChainFrame* chain, BuildResolved* out) {
+static int build_run_recipe_impl(KitBuildCoordinator* c, KitSlice target,
+ const BuildConfig* cfg, const BuildArgv* argv,
+ const BuildChainFrame* chain,
+ BuildResolved* out, int record_traces) {
const BuildTargetDefn* defn;
KitBuildListener* listener = NULL;
KitBuildConn* conn = NULL;
@@ -258,6 +276,10 @@ int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
if (!c || !cfg || !argv || !out || !c->host.exec || !c->host.transport)
return BUILD_ERR;
+ endpoint[0] = '\0';
+ sandbox[0] = '\0';
+ out_dir[0] = '\0';
+ recipe_path[0] = '\0';
defn = build_defn_find(&c->defn, target);
if (!defn) {
build_diagf(c->ctx, "build: unknown target %.*s", KIT_SLICE_ARG(target));
@@ -274,6 +296,21 @@ int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
sizeof out_dir) != BUILD_OK)
goto out_cleanup;
+ memset(&log, 0, sizeof log);
+ log.config_keys = config_keys;
+ log.cap_config_keys = sizeof config_keys / sizeof config_keys[0];
+ log.sources = sources;
+ log.cap_sources = sizeof sources / sizeof sources[0];
+ log.globs = globs;
+ log.cap_globs = sizeof globs / sizeof globs[0];
+ log.deps = deps;
+ log.cap_deps = sizeof deps / sizeof deps[0];
+ log.child_leafsets = child_leafsets;
+ log.cap_children = sizeof child_leafsets / sizeof child_leafsets[0];
+ log.pending = pending;
+ log.cap_pending = sizeof pending / sizeof pending[0];
+ log.next_token = 1;
+
proc_argv[argc++] = kit_slice_cstr(recipe_path);
for (i = 0; i < argv->n && argc < sizeof proc_argv / sizeof proc_argv[0];
++i)
@@ -290,6 +327,8 @@ int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
const char* key = cfg->entries[i].key;
size_t prefix_len = sizeof(KIT_BUILD_ENV_PREFIX) - 1u;
if (strncmp(key, KIT_BUILD_ENV_PREFIX, prefix_len) != 0) continue;
+ if (append_config_key(&log, kit_slice_cstr(key)) != BUILD_OK)
+ goto out_cleanup;
snprintf(env_keys[nenv - 3u], sizeof env_keys[nenv - 3u], "%s",
key + prefix_len);
env[nenv].key = kit_slice_cstr(env_keys[nenv - 3u]);
@@ -302,21 +341,6 @@ int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
!proc)
goto out_cleanup;
build_coord_stat_bump(c, BUILD_STAT_RECIPE_RUN);
-
- memset(&log, 0, sizeof log);
- log.config_keys = config_keys;
- log.cap_config_keys = sizeof config_keys / sizeof config_keys[0];
- log.sources = sources;
- log.cap_sources = sizeof sources / sizeof sources[0];
- log.globs = globs;
- log.cap_globs = sizeof globs / sizeof globs[0];
- log.deps = deps;
- log.cap_deps = sizeof deps / sizeof deps[0];
- log.child_leafsets = child_leafsets;
- log.cap_children = sizeof child_leafsets / sizeof child_leafsets[0];
- log.pending = pending;
- log.cap_pending = sizeof pending / sizeof pending[0];
- log.next_token = 1;
if (service_recipe_connections(c, listener, &proc, target, cfg, chain, &log,
&exit_code) != BUILD_OK)
goto out_kill;
@@ -324,9 +348,13 @@ int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
if (build_store_ingest_output(&c->store, out_dir, out->output_tree, out->path,
sizeof out->path) != BUILD_OK)
goto out_cleanup;
- if (build_runner_record_traces(c, target, cfg, argv, &log, out->output_tree,
- &out->leafset) != BUILD_OK)
- goto out_cleanup;
+ if (record_traces) {
+ if (build_runner_record_traces(c, target, cfg, argv, &log,
+ out->output_tree, &out->leafset) != BUILD_OK)
+ goto out_cleanup;
+ } else {
+ out->leafset = NULL;
+ }
ok = BUILD_OK;
goto out_cleanup;
@@ -343,6 +371,18 @@ out_cleanup:
return ok;
}
+int build_run_recipe(KitBuildCoordinator* c, KitSlice target,
+ const BuildConfig* cfg, const BuildArgv* argv,
+ const BuildChainFrame* chain, BuildResolved* out) {
+ return build_run_recipe_impl(c, target, cfg, argv, chain, out, 1);
+}
+
+int build_run_recipe_probe(KitBuildCoordinator* c, KitSlice target,
+ const BuildConfig* cfg, const BuildArgv* argv,
+ const BuildChainFrame* chain, BuildResolved* out) {
+ return build_run_recipe_impl(c, target, cfg, argv, chain, out, 0);
+}
+
int build_runner_service(KitBuildCoordinator* c, KitBuildConn* conn,
KitSlice target, const BuildConfig* cfg,
const BuildChainFrame* chain, BuildDepLog* log) {
@@ -490,11 +530,18 @@ int build_runner_service(KitBuildCoordinator* c, KitBuildConn* conn,
} else if (req.cmd == BUILD_CMD_NEED_AWAIT) {
BuildPendingNeed* p = find_pending(log, req.token);
BuildResolved r;
- if (!p || build_coord_target_await(c, p->future, &r) != BUILD_OK ||
- append_dep(log, kit_slice_cstr(p->dep), p->overlay_id, p->config_id,
- p->argv_id, &r) != BUILD_OK) {
+ if (!p) {
+ resp_error(&resp, KIT_ERR, "need-await failed");
+ } else if (p->awaited) {
+ memcpy(resp.id, p->result.output_tree, BUILD_HASH_LEN);
+ resp.text = kit_slice_cstr(p->result.path);
+ } else if (build_coord_target_await(c, p->future, &r) != BUILD_OK ||
+ append_dep(log, kit_slice_cstr(p->dep), p->overlay_id,
+ p->config_id, p->argv_id, &r) != BUILD_OK) {
resp_error(&resp, KIT_ERR, "need-await failed");
} else {
+ p->awaited = 1;
+ p->result = r;
memcpy(resp.id, r.output_tree, BUILD_HASH_LEN);
resp.text = kit_slice_cstr(r.path);
}
diff --git a/src/build/runner.h b/src/build/runner.h
@@ -35,6 +35,8 @@ typedef struct BuildPendingNeed {
uint8_t config_id[BUILD_HASH_LEN];
uint8_t argv_id[BUILD_HASH_LEN];
BuildTargetFuture* future; /* from build_coord_dispatch */
+ int awaited;
+ BuildResolved result;
} BuildPendingNeed;
/* The dependency log accumulated while servicing one recipe: the direct base
diff --git a/test/buildcoord/run.sh b/test/buildcoord/run.sh
@@ -313,6 +313,8 @@ mkdir -p "$KIT_BUILD_OUT"
tok=$("$KIT" build need-submit //argv:echo -- via-submit)
dep_dir=$("$KIT" build need-await "$tok")
cat "$dep_dir/args.txt" > "$KIT_BUILD_OUT/submit.txt"
+dep_dir_again=$("$KIT" build need-await "$tok")
+cat "$dep_dir_again/args.txt" > "$KIT_BUILD_OUT/submit-again.txt"
EOF
chmod +x "$ws"/recipes/*.sh
@@ -757,6 +759,8 @@ submit_path=$(tree_path_from "$work/buildcoord-spec-need-submit-await.out")
if [ -n "$submit_path" ] && [ -f "$submit_path/submit.txt" ]; then
contains "buildcoord-spec-need-submit-output" "$submit_path/submit.txt" \
"via-submit"
+ contains "buildcoord-spec-need-submit-await-idempotent" \
+ "$submit_path/submit-again.txt" "via-submit"
fi
kit_summary "buildcoord"
