commit 30edea9e33151f5d7fc5ba2c19f3b7111b6ab797
parent 191086c86039bef04b03180077d01a52acf86360
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Thu, 18 Jun 2026 11:55:08 -0700
build: improve recipe client helper CLI
Diffstat:
3 files changed, 304 insertions(+), 33 deletions(-)
diff --git a/doc/BUILD_COORDINATOR.md b/doc/BUILD_COORDINATOR.md
@@ -687,7 +687,7 @@ command.
| `config-get <key>` | the propagated value (or *unset*) | config dep: key recorded as consumed; its value is the one in the effective `config-id` map (absent = unset) |
| `source <path>` | `blob-id` (or *absent*) + a path to read | source dep: `(path, blob-id)`, or `(path, absent)` |
| `glob <pattern>` | sorted list of matching paths (streamed) | glob dep: `(pattern, glob-result-hash)` |
-| `fetch <blob-id> <url>...` | local CAS path to the verified blob | fetch dep: `blob-id` |
+| `fetch <blob-id> <url>...` | the verified blob-id + a local CAS path | fetch dep: `blob-id` |
| `need <target> [k=v…] [env…] [argv…]` | the dep's output `tree-id` + a path (blocks) | target dep edge: `(dep, dep-config-id, dep-argv-id, output-tree-id)` |
| `need-submit <target> [k=v…] [env…] [argv…]` | a **token** (does not block) | *(nothing yet — logged on await)* |
| `need-await <token>` | the submitted dep's output `tree-id` + a path | target dep edge: `(dep, dep-config-id, dep-argv-id, output-tree-id)` |
@@ -747,6 +747,11 @@ command.
- **Outputs** need no command: the recipe writes under `$KIT_BUILD_OUT` and the
coordinator snapshots that directory on exit.
+The shell helper also provides `depfile [--lines] FILE`, which is not a separate
+wire command. It parses Make/GCC-style depfiles (or one path per line with
+`--lines`) and submits every prerequisite through `source`, so each prerequisite is
+logged as an ordinary source dependency.
+
## CLI
Build execution:
@@ -770,6 +775,29 @@ errors also exit nonzero and print diagnostics on stderr. `--stats` prints
cumulative coordinator counters, including `test_runs`, `test_cache_hits`, and
`test_failures`, to stderr.
+Recipe-side shell helper commands are available only inside a running recipe
+(`$KIT_BUILD_SOCK` set), or explicitly as `kit build --client ...`:
+
+```
+kit build config-get [--default VALUE] KEY
+kit build source [--optional] [--format path|id|id-path] PATH
+kit build fetch [--format path|id|id-path] BLOB URL...
+kit build depfile [--lines] FILE
+kit build glob PATTERN
+kit build need [--format path|id|id-path]
+ [--config K=V|env.NAME]... [--env NAME[=V]]...
+ TARGET [-- ARG...]
+kit build need-submit [--config K=V|env.NAME]... [--env NAME[=V]]...
+ TARGET [-- ARG...]
+kit build need-await [--format path|id|id-path] TOKEN
+```
+
+The default helper output is the readable path where a path exists. `--format id`
+prints the blob/tree id, and `--format id-path` prints both as `<hex> <path>`.
+`config-get --default VALUE` exits 0 and prints `VALUE` when a key is unset.
+`source --optional` exits 0 with no output when the source is absent, while still
+recording the absent-source dependency.
+
## Remote CAS and shared traces
Two independent network capabilities, split along the DISTRIBUTE.md line —
diff --git a/driver/cmd/build_coord.c b/driver/cmd/build_coord.c
@@ -18,11 +18,18 @@
#define BUILD_MAX_CONFIG_KEY 64u
#define BUILD_MAX_ARGS 128u
+typedef enum BuildClientFormat {
+ BUILD_CLIENT_FORMAT_PATH = 0,
+ BUILD_CLIENT_FORMAT_ID = 1,
+ BUILD_CLIENT_FORMAT_ID_PATH = 2,
+} BuildClientFormat;
+
typedef struct BuildCli {
const char* store;
const char* root;
const char* def;
const char* target;
+ const char* config_default;
KitBuildKV config[BUILD_MAX_CONFIG];
char config_keys[BUILD_MAX_CONFIG][BUILD_MAX_CONFIG_KEY];
size_t nconfig;
@@ -31,6 +38,10 @@ typedef struct BuildCli {
int verify;
int stats;
int test_mode;
+ int has_config_default;
+ int source_optional;
+ int has_format;
+ BuildClientFormat format;
} BuildCli;
void driver_help_build(void) {
@@ -44,18 +55,22 @@ void driver_help_build(void) {
" kit build test [--store DIR] [--root DIR] [--def FILE]\n"
" [--config K=V|env.NAME]... [--env NAME[=V]]...\n"
" [--verify] [--stats] TARGET [-- ARG...]\n"
- " kit build config-get KEY # inside a build recipe\n"
- " kit build source PATH # inside a build recipe\n"
- " kit build fetch BLOB URL... # inside a build recipe\n"
+ " kit build config-get [--default VALUE] KEY # inside a build recipe\n"
+ " kit build source [--optional] [--format path|id|id-path] PATH\n"
+ " # inside a build recipe\n"
+ " kit build fetch [--format path|id|id-path] BLOB URL...\n"
+ " # inside a build recipe\n"
" kit build depfile [--lines] FILE # inside a build recipe\n"
" kit build glob PATTERN # inside a build recipe\n"
- " kit build need [--config K=V|env.NAME]... [--env NAME[=V]]...\n"
+ " kit build need [--format path|id|id-path]\n"
+ " [--config K=V|env.NAME]... [--env NAME[=V]]...\n"
" TARGET [-- ARG...]\n"
" # inside a build recipe\n"
" kit build need-submit [--config K=V|env.NAME]... [--env NAME[=V]]...\n"
" TARGET [-- ARG...]\n"
" # inside a build recipe\n"
- " kit build need-await TOKEN # inside a build recipe\n"
+ " kit build need-await [--format path|id|id-path] TOKEN\n"
+ " # inside a build recipe\n"
"\n"
"OPTIONS\n"
" --store DIR Build store root (default: $KIT cache/build)\n"
@@ -169,6 +184,39 @@ static int build_client_glob_print(void* user, KitSlice path) {
return 0;
}
+static int build_client_parse_format(const char* s, BuildClientFormat* out) {
+ if (!s || !out) return 1;
+ if (driver_streq(s, "path")) {
+ *out = BUILD_CLIENT_FORMAT_PATH;
+ return 0;
+ }
+ if (driver_streq(s, "id")) {
+ *out = BUILD_CLIENT_FORMAT_ID;
+ return 0;
+ }
+ if (driver_streq(s, "id-path")) {
+ *out = BUILD_CLIENT_FORMAT_ID_PATH;
+ return 0;
+ }
+ return 1;
+}
+
+static void build_client_print_value(BuildClientFormat fmt,
+ const uint8_t id[KIT_BUILD_HASH_LEN],
+ KitSlice path) {
+ char hex[2u * KIT_BUILD_HASH_LEN + 1u];
+ if (fmt == BUILD_CLIENT_FORMAT_PATH) {
+ driver_printf("%.*s\n", KIT_SLICE_ARG(path));
+ return;
+ }
+ kit_hex_encode(hex, id, KIT_BUILD_HASH_LEN);
+ if (fmt == BUILD_CLIENT_FORMAT_ID) {
+ driver_printf("%s\n", hex);
+ return;
+ }
+ driver_printf("%s %.*s\n", hex, KIT_SLICE_ARG(path));
+}
+
static int build_client_verb(const char* s) {
return driver_streq(s, "config-get") || driver_streq(s, "source") ||
driver_streq(s, "fetch") || driver_streq(s, "depfile") ||
@@ -207,6 +255,12 @@ static int build_client_need_parse(BuildCli* cli, int argc, char** argv,
driver_errf(BUILD_TOOL, "bad --env, expected NAME or NAME=VALUE");
return 2;
}
+ } else if (driver_streq(a, "--format") && i + 1 < argc) {
+ if (build_client_parse_format(argv[++i], &cli->format) != 0) {
+ driver_errf(BUILD_TOOL, "bad --format, expected path, id, or id-path");
+ return 2;
+ }
+ cli->has_format = 1;
} else if (driver_streq(a, "--")) {
if (!cli->target) {
driver_errf(BUILD_TOOL, "missing client target before --");
@@ -272,18 +326,42 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
if (driver_streq(verb, "config-get")) {
KitSlice value;
int present = 0;
- if (argc != first_arg + 1) {
- driver_errf(BUILD_TOOL, "usage: kit build config-get KEY");
+ BuildCli cli;
+ int i;
+ memset(&cli, 0, sizeof cli);
+ for (i = first_arg; i < argc; ++i) {
+ if (driver_streq(argv[i], "--default") && i + 1 < argc) {
+ cli.has_config_default = 1;
+ cli.config_default = argv[++i];
+ } else if (argv[i][0] == '-') {
+ driver_errf(BUILD_TOOL, "unexpected config-get option: %s", argv[i]);
+ rc = 2;
+ goto out_client;
+ } else if (!cli.target) {
+ cli.target = argv[i];
+ } else {
+ driver_errf(BUILD_TOOL, "unexpected config-get argument: %s", argv[i]);
+ rc = 2;
+ goto out_client;
+ }
+ }
+ if (!cli.target) {
+ driver_errf(BUILD_TOOL, "usage: kit build config-get [--default VALUE] KEY");
rc = 2;
goto out_client;
}
- st = kit_build_client_config_get(client, kit_slice_cstr(argv[first_arg]),
- &value, &present);
+ st = kit_build_client_config_get(client, kit_slice_cstr(cli.target), &value,
+ &present);
if (st != KIT_OK) {
driver_errf(BUILD_TOOL, "config-get failed: %s", build_status_name(st));
goto out_client;
}
if (!present) {
+ if (cli.has_config_default) {
+ driver_printf("%s\n", cli.config_default);
+ rc = 0;
+ goto out_client;
+ }
rc = 1;
goto out_client;
}
@@ -292,18 +370,47 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
} else if (driver_streq(verb, "source")) {
uint8_t blob[KIT_BUILD_HASH_LEN];
KitSlice path;
- if (argc != first_arg + 1) {
- driver_errf(BUILD_TOOL, "usage: kit build source PATH");
+ BuildCli cli;
+ int i;
+ memset(&cli, 0, sizeof cli);
+ for (i = first_arg; i < argc; ++i) {
+ if (driver_streq(argv[i], "--optional")) {
+ cli.source_optional = 1;
+ } else if (driver_streq(argv[i], "--format") && i + 1 < argc) {
+ if (build_client_parse_format(argv[++i], &cli.format) != 0) {
+ driver_errf(BUILD_TOOL, "bad --format, expected path, id, or id-path");
+ rc = 2;
+ goto out_client;
+ }
+ } else if (argv[i][0] == '-') {
+ driver_errf(BUILD_TOOL, "unexpected source option: %s", argv[i]);
+ rc = 2;
+ goto out_client;
+ } else if (!cli.target) {
+ cli.target = argv[i];
+ } else {
+ driver_errf(BUILD_TOOL, "unexpected source argument: %s", argv[i]);
+ rc = 2;
+ goto out_client;
+ }
+ }
+ if (!cli.target) {
+ driver_errf(BUILD_TOOL,
+ "usage: kit build source [--optional] "
+ "[--format path|id|id-path] PATH");
rc = 2;
goto out_client;
}
- st = kit_build_client_source(client, kit_slice_cstr(argv[first_arg]), blob,
- &path);
+ st = kit_build_client_source(client, kit_slice_cstr(cli.target), blob, &path);
+ if (st == KIT_NOT_FOUND && cli.source_optional) {
+ rc = 0;
+ goto out_client;
+ }
if (st != KIT_OK) {
driver_errf(BUILD_TOOL, "source failed: %s", build_status_name(st));
goto out_client;
}
- driver_printf("%.*s\n", KIT_SLICE_ARG(path));
+ build_client_print_value(cli.format, blob, path);
rc = 0;
} else if (driver_streq(verb, "fetch")) {
uint8_t blob[KIT_BUILD_HASH_LEN];
@@ -311,27 +418,41 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
KitSlice path;
size_t nurls = 0;
int i;
- if (argc < first_arg + 2 ||
- driver_strlen(argv[first_arg]) != 2u * KIT_BUILD_HASH_LEN ||
- kit_hex_decode(blob, argv[first_arg], KIT_BUILD_HASH_LEN) != KIT_OK) {
- driver_errf(BUILD_TOOL, "usage: kit build fetch BLOB URL...");
+ BuildCli cli;
+ int blob_arg = 0;
+ memset(&cli, 0, sizeof cli);
+ for (i = first_arg; i < argc; ++i) {
+ if (driver_streq(argv[i], "--format") && i + 1 < argc) {
+ if (build_client_parse_format(argv[++i], &cli.format) != 0) {
+ driver_errf(BUILD_TOOL, "bad --format, expected path, id, or id-path");
+ rc = 2;
+ goto out_client;
+ }
+ } else if (!blob_arg) {
+ blob_arg = i;
+ } else {
+ if (nurls >= BUILD_MAX_ARGS) {
+ driver_errf(BUILD_TOOL, "too many fetch URLs");
+ rc = 2;
+ goto out_client;
+ }
+ urls[nurls++] = kit_slice_cstr(argv[i]);
+ }
+ }
+ if (!blob_arg || nurls == 0u ||
+ driver_strlen(argv[blob_arg]) != 2u * KIT_BUILD_HASH_LEN ||
+ kit_hex_decode(blob, argv[blob_arg], KIT_BUILD_HASH_LEN) != KIT_OK) {
+ driver_errf(BUILD_TOOL,
+ "usage: kit build fetch [--format path|id|id-path] BLOB URL...");
rc = 2;
goto out_client;
}
- for (i = first_arg + 1; i < argc; ++i) {
- if (nurls >= BUILD_MAX_ARGS) {
- driver_errf(BUILD_TOOL, "too many fetch URLs");
- rc = 2;
- goto out_client;
- }
- urls[nurls++] = kit_slice_cstr(argv[i]);
- }
st = kit_build_client_fetch(client, blob, urls, nurls, &path);
if (st != KIT_OK) {
driver_errf(BUILD_TOOL, "fetch failed: %s", build_status_name(st));
goto out_client;
}
- driver_printf("%.*s\n", KIT_SLICE_ARG(path));
+ build_client_print_value(cli.format, blob, path);
rc = 0;
} else if (driver_streq(verb, "depfile")) {
DriverLoad load;
@@ -389,7 +510,8 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
rc = 1;
goto out_client;
}
- driver_printf("%s\n", result.path);
+ build_client_print_value(cli.format, result.output_tree,
+ kit_slice_cstr(result.path));
rc = 0;
} else if (driver_streq(verb, "need-submit")) {
BuildCli cli;
@@ -397,6 +519,11 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
KitBuildNeedToken token;
rc = build_client_need_parse(&cli, argc, argv, first_arg);
if (rc != 0) goto out_client;
+ if (cli.has_format) {
+ driver_errf(BUILD_TOOL, "need-submit does not support --format");
+ rc = 2;
+ goto out_client;
+ }
memset(&req, 0, sizeof req);
req.target = kit_slice_cstr(cli.target);
req.config = cli.config;
@@ -415,9 +542,32 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
} else if (driver_streq(verb, "need-await")) {
KitBuildNeedToken token;
KitBuildResult result;
- if (argc != first_arg + 1 ||
- build_parse_u64(argv[first_arg], &token.id) != 0) {
- driver_errf(BUILD_TOOL, "usage: kit build need-await TOKEN");
+ BuildCli cli;
+ const char* token_arg = NULL;
+ int i;
+ memset(&cli, 0, sizeof cli);
+ for (i = first_arg; i < argc; ++i) {
+ if (driver_streq(argv[i], "--format") && i + 1 < argc) {
+ if (build_client_parse_format(argv[++i], &cli.format) != 0) {
+ driver_errf(BUILD_TOOL, "bad --format, expected path, id, or id-path");
+ rc = 2;
+ goto out_client;
+ }
+ } else if (argv[i][0] == '-') {
+ driver_errf(BUILD_TOOL, "unexpected need-await option: %s", argv[i]);
+ rc = 2;
+ goto out_client;
+ } else if (!token_arg) {
+ token_arg = argv[i];
+ } else {
+ driver_errf(BUILD_TOOL, "unexpected need-await argument: %s", argv[i]);
+ rc = 2;
+ goto out_client;
+ }
+ }
+ if (!token_arg || build_parse_u64(token_arg, &token.id) != 0) {
+ driver_errf(BUILD_TOOL,
+ "usage: kit build need-await [--format path|id|id-path] TOKEN");
rc = 2;
goto out_client;
}
@@ -427,7 +577,8 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
rc = 1;
goto out_client;
}
- driver_printf("%s\n", result.path);
+ build_client_print_value(cli.format, result.output_tree,
+ kit_slice_cstr(result.path));
rc = 0;
} else {
driver_errf(BUILD_TOOL, "unknown client command: %s", verb);
diff --git a/test/buildcoord/run.sh b/test/buildcoord/run.sh
@@ -35,6 +35,8 @@ recipe recipes/app.sh
recipe recipes/argv.sh
[target //cfg:probe]
recipe recipes/cfg.sh
+[target //client:formats]
+recipe recipes/client_formats.sh
[target //cycle:a]
recipe recipes/cycle_a.sh
[target //cycle:b]
@@ -55,6 +57,8 @@ recipe recipes/env_probe.sh
recipe recipes/fail.sh
[target //fetch:blob]
recipe recipes/fetch.sh
+[target //fetch:format]
+recipe recipes/fetch_format.sh
[target //globset:probe]
recipe recipes/globset.sh
[target //globstar:probe]
@@ -175,6 +179,18 @@ mode=$("$KIT" build config-get mode || true)
printf 'mode:%s\n' "$mode" > "$KIT_BUILD_OUT/mode.txt"
EOF
+cat > "$ws/recipes/client_formats.sh" <<'EOF'
+#!/bin/sh
+set -eu
+"$KIT" build config-get --default fallback missing-key > "$KIT_BUILD_OUT/default.txt"
+"$KIT" build source --optional src/missing-optional.txt > "$KIT_BUILD_OUT/optional.txt"
+"$KIT" build source --format id-path src/a.txt > "$KIT_BUILD_OUT/source.txt"
+"$KIT" build need --format id-path //argv:echo -- format-arg \
+ > "$KIT_BUILD_OUT/need.txt"
+tok=$("$KIT" build need-submit //argv:echo -- format-arg)
+"$KIT" build need-await --format id-path "$tok" > "$KIT_BUILD_OUT/await.txt"
+EOF
+
cat > "$ws/recipes/cycle_a.sh" <<'EOF'
#!/bin/sh
set -eu
@@ -288,6 +304,18 @@ IFS= read -r line < "$p"
printf '%s\n' "$line" > "$KIT_BUILD_OUT/fetched.txt"
EOF
+cat > "$ws/recipes/fetch_format.sh" <<'EOF'
+#!/bin/sh
+set -eu
+blob=$1
+shift
+info=$("$KIT" build fetch --format id-path "$blob" "$@")
+printf '%s\n' "$info" > "$KIT_BUILD_OUT/fetch-info.txt"
+path=${info#* }
+IFS= read -r line < "$path"
+printf '%s\n' "$line" > "$KIT_BUILD_OUT/fetched.txt"
+EOF
+
cat > "$ws/recipes/globset.sh" <<'EOF'
#!/bin/sh
set -eu
@@ -540,6 +568,14 @@ test_tree_path_from() {
awk 'NF >= 3 {print $3; exit}' "$1"
}
+first_field_from() {
+ awk 'NF >= 1 {print $1; exit}' "$1"
+}
+
+second_field_from() {
+ awk 'NF >= 2 {print $2; exit}' "$1"
+}
+
tree_id_from() {
awk 'NF >= 1 {print $1; exit}' "$1"
}
@@ -887,6 +923,48 @@ cfg_unset_again_path=$(tree_path_from \
contains "buildcoord-config-unset-restores-trace-output" \
"$cfg_unset_again_path/mode.txt" "mode:"
+build_assert_ok buildcoord-client-formats //client:formats
+client_formats_path=$(tree_path_from "$work/buildcoord-client-formats.out")
+contains "buildcoord-config-get-default" "$client_formats_path/default.txt" \
+ "fallback"
+if [ ! -s "$client_formats_path/optional.txt" ]; then
+ ok "buildcoord-source-optional-absent"
+else
+ printf 'optional output was not empty\n' > "$work/source-optional.diag"
+ not_ok "buildcoord-source-optional-absent" "$work/source-optional.diag"
+fi
+source_fmt_id=$(first_field_from "$client_formats_path/source.txt")
+source_fmt_path=$(second_field_from "$client_formats_path/source.txt")
+if [ ${#source_fmt_id} -eq 64 ] && [ -f "$source_fmt_path" ]; then
+ ok "buildcoord-source-format-id-path"
+else
+ printf 'id=%s\npath=%s\n' "$source_fmt_id" "$source_fmt_path" \
+ > "$work/source-format.diag"
+ not_ok "buildcoord-source-format-id-path" "$work/source-format.diag"
+fi
+need_fmt_id=$(first_field_from "$client_formats_path/need.txt")
+need_fmt_path=$(second_field_from "$client_formats_path/need.txt")
+if [ ${#need_fmt_id} -eq 64 ] && [ -d "$need_fmt_path" ]; then
+ ok "buildcoord-need-format-id-path"
+else
+ printf 'id=%s\npath=%s\n' "$need_fmt_id" "$need_fmt_path" \
+ > "$work/need-format.diag"
+ not_ok "buildcoord-need-format-id-path" "$work/need-format.diag"
+fi
+contains "buildcoord-need-format-output" "$need_fmt_path/args.txt" \
+ "format-arg"
+await_fmt_id=$(first_field_from "$client_formats_path/await.txt")
+await_fmt_path=$(second_field_from "$client_formats_path/await.txt")
+if [ ${#await_fmt_id} -eq 64 ] && [ -d "$await_fmt_path" ]; then
+ ok "buildcoord-need-await-format-id-path"
+else
+ printf 'id=%s\npath=%s\n' "$await_fmt_id" "$await_fmt_path" \
+ > "$work/await-format.diag"
+ not_ok "buildcoord-need-await-format-id-path" "$work/await-format.diag"
+fi
+contains "buildcoord-need-await-format-output" "$await_fmt_path/args.txt" \
+ "format-arg"
+
build_assert_ok buildcoord-defn-v1 --stats //defn:probe
contains "buildcoord-defn-v1-run" "$work/buildcoord-defn-v1.err" \
"recipes_run=1"
@@ -1098,6 +1176,20 @@ contains "buildcoord-fetch-cas-hit-rerun" \
fetch_hit_path=$(tree_path_from "$work/buildcoord-fetch-cas-hit-bad-url.out")
contains "buildcoord-fetch-cas-hit-output" "$fetch_hit_path/fetched.txt" \
"remote:v1"
+build_assert_ok buildcoord-fetch-format --stats --config env.PATH=/no/such \
+ //fetch:format -- "$fetch_id" "file://$ws/remote/fetch.txt"
+fetch_format_path=$(tree_path_from "$work/buildcoord-fetch-format.out")
+fetch_format_id=$(first_field_from "$fetch_format_path/fetch-info.txt")
+fetch_format_file=$(second_field_from "$fetch_format_path/fetch-info.txt")
+if [ "$fetch_format_id" = "$fetch_id" ] && [ -f "$fetch_format_file" ]; then
+ ok "buildcoord-fetch-format-id-path"
+else
+ printf 'want=%s\ngot=%s\npath=%s\n' "$fetch_id" "$fetch_format_id" \
+ "$fetch_format_file" > "$work/fetch-format.diag"
+ not_ok "buildcoord-fetch-format-id-path" "$work/fetch-format.diag"
+fi
+contains "buildcoord-fetch-format-output" "$fetch_format_path/fetched.txt" \
+ "remote:v1"
run_fail "buildcoord-cycle-self-fails" \
"$KIT" build --store "$store" --root "$ws" --def BUILD.kit \
