commit d02cb3a24fa7df0304e537501ec3bb3b90602d24
parent cd24673c722c7c2e23062175043c8fd3eb9b6df4
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Thu, 18 Jun 2026 09:11:14 -0700
build: add hash-pinned client fetch
Diffstat:
18 files changed, 547 insertions(+), 11 deletions(-)
diff --git a/driver/cmd/build_coord.c b/driver/cmd/build_coord.c
@@ -42,6 +42,7 @@ void driver_help_build(void) {
" [--config K=V]... [--verify] [--stats] TARGET [-- ARG...]\n"
" kit build config-get KEY # inside a build recipe\n"
" kit build source PATH # inside a build recipe\n"
+ " kit build fetch BLOB URL... # inside a build recipe\n"
" kit build depfile [--lines] FILE # inside a build recipe\n"
" kit build glob PATTERN # inside a build recipe\n"
" kit build need [--config K=V]... TARGET [-- ARG...]\n"
@@ -110,7 +111,8 @@ static int build_client_glob_print(void* user, KitSlice path) {
static int build_client_verb(const char* s) {
return driver_streq(s, "config-get") || driver_streq(s, "source") ||
- driver_streq(s, "depfile") || driver_streq(s, "glob") ||
+ driver_streq(s, "fetch") || driver_streq(s, "depfile") ||
+ driver_streq(s, "glob") ||
driver_streq(s, "need") ||
driver_streq(s, "need-submit") || driver_streq(s, "need-await");
}
@@ -238,6 +240,34 @@ static int build_client_mode(int argc, char** argv, int verb_index) {
}
driver_printf("%.*s\n", KIT_SLICE_ARG(path));
rc = 0;
+ } else if (driver_streq(verb, "fetch")) {
+ uint8_t blob[KIT_BUILD_HASH_LEN];
+ KitSlice urls[BUILD_MAX_ARGS];
+ KitSlice path;
+ size_t nurls = 0;
+ int i;
+ if (argc < first_arg + 2 ||
+ driver_strlen(argv[first_arg]) != 2u * KIT_BUILD_HASH_LEN ||
+ kit_hex_decode(blob, argv[first_arg], KIT_BUILD_HASH_LEN) != KIT_OK) {
+ driver_errf(BUILD_TOOL, "usage: kit build fetch BLOB URL...");
+ rc = 2;
+ goto out_client;
+ }
+ for (i = first_arg + 1; i < argc; ++i) {
+ if (nurls >= BUILD_MAX_ARGS) {
+ driver_errf(BUILD_TOOL, "too many fetch URLs");
+ rc = 2;
+ goto out_client;
+ }
+ urls[nurls++] = kit_slice_cstr(argv[i]);
+ }
+ st = kit_build_client_fetch(client, blob, urls, nurls, &path);
+ if (st != KIT_OK) {
+ driver_errf(BUILD_TOOL, "fetch failed: %s", build_status_name(st));
+ goto out_client;
+ }
+ driver_printf("%.*s\n", KIT_SLICE_ARG(path));
+ rc = 0;
} else if (driver_streq(verb, "depfile")) {
DriverLoad load;
KitSlice depfile;
diff --git a/driver/env.h b/driver/env.h
@@ -270,12 +270,10 @@ int driver_remove_tree(const char* path);
int driver_kit_home(char* buf, size_t cap);
/* Fetch `url` to the local file `dest`, overwriting it. The transport is
- * UNTRUSTED — the caller verifies a signature/content-id over the bytes — so
- * this is a thin, narrow download primitive, not a general subprocess API:
- * POSIX execs `curl -fsSL -o dest -- url` (falling back to `wget -O dest url`)
- * via fork/exec with the URL passed as a distinct argv element (no shell); the
- * Windows port uses curl.exe likewise. Returns 0 on success, nonzero on any
- * failure (no fetcher found, network/HTTP error, write error). */
+ * UNTRUSTED — the caller verifies a signature/content-id over the bytes. file://
+ * URLs are copied internally; other schemes use curl/wget (POSIX) or curl.exe
+ * (Windows), with the URL passed as a distinct argv element (no shell). Returns
+ * 0 on success, nonzero on any failure. */
int driver_fetch_url(const char* url, const char* dest);
/* Set a linked binary output's final mode according to the active umask.
diff --git a/driver/env/build_host_posix.c b/driver/env/build_host_posix.c
@@ -500,6 +500,25 @@ static void db_transport_close(void* user, KitBuildConn* conn) {
driver_free(bh->env, conn, sizeof *conn);
}
+static int db_fetch_url(void* user, KitSlice url, KitSlice dest) {
+ DriverBuildHost* bh = (DriverBuildHost*)user;
+ char* url_s;
+ char* dest_s;
+ int rc;
+ if (!bh || !bh->env) return 1;
+ url_s = db_slice_dup(bh->env, url);
+ dest_s = db_slice_dup(bh->env, dest);
+ if (!url_s || !dest_s) {
+ if (url_s) driver_free(bh->env, url_s, strlen(url_s) + 1u);
+ if (dest_s) driver_free(bh->env, dest_s, strlen(dest_s) + 1u);
+ return 1;
+ }
+ rc = driver_fetch_url(url_s, dest_s);
+ driver_free(bh->env, url_s, strlen(url_s) + 1u);
+ driver_free(bh->env, dest_s, strlen(dest_s) + 1u);
+ return rc;
+}
+
int driver_build_host_init(DriverBuildHost* out, DriverEnv* env) {
if (!out || !env) return 1;
memset(out, 0, sizeof *out);
@@ -526,10 +545,13 @@ int driver_build_host_init(DriverBuildHost* out, DriverEnv* env) {
out->transport.write_frame = db_transport_write;
out->transport.close = db_transport_close;
out->transport.user = out;
+ out->fetch.fetch_url = db_fetch_url;
+ out->fetch.user = out;
out->host.cas_host = &out->cas_host;
out->host.store_io = &out->store_io;
out->host.exec = &out->exec;
out->host.transport = &out->transport;
+ out->host.fetch = &out->fetch;
out->next_endpoint = 1;
return 0;
}
diff --git a/driver/env/build_host_windows.c b/driver/env/build_host_windows.c
@@ -789,6 +789,25 @@ static void db_transport_close(void* user, KitBuildConn* conn) {
driver_free(bh->env, conn, sizeof *conn);
}
+static int db_fetch_url(void* user, KitSlice url, KitSlice dest) {
+ DriverBuildHost* bh = (DriverBuildHost*)user;
+ char* url_s;
+ char* dest_s;
+ int rc;
+ if (!bh || !bh->env) return 1;
+ url_s = db_slice_dup(bh->env, url);
+ dest_s = db_slice_dup(bh->env, dest);
+ if (!url_s || !dest_s) {
+ if (url_s) driver_free(bh->env, url_s, strlen(url_s) + 1u);
+ if (dest_s) driver_free(bh->env, dest_s, strlen(dest_s) + 1u);
+ return 1;
+ }
+ rc = driver_fetch_url(url_s, dest_s);
+ driver_free(bh->env, url_s, strlen(url_s) + 1u);
+ driver_free(bh->env, dest_s, strlen(dest_s) + 1u);
+ return rc;
+}
+
int driver_build_host_init(DriverBuildHost* out, DriverEnv* env) {
if (!out || !env) return 1;
memset(out, 0, sizeof *out);
@@ -815,10 +834,13 @@ int driver_build_host_init(DriverBuildHost* out, DriverEnv* env) {
out->transport.write_frame = db_transport_write;
out->transport.close = db_transport_close;
out->transport.user = out;
+ out->fetch.fetch_url = db_fetch_url;
+ out->fetch.user = out;
out->host.cas_host = &out->cas_host;
out->host.store_io = &out->store_io;
out->host.exec = &out->exec;
out->host.transport = &out->transport;
+ out->host.fetch = &out->fetch;
out->next_endpoint = 1;
return 0;
}
diff --git a/driver/env/posix.c b/driver/env/posix.c
@@ -811,10 +811,118 @@ int driver_kit_home(char* buf, size_t cap) {
return 0;
}
+static int fetch_hex_val(char c, unsigned* out) {
+ if (c >= '0' && c <= '9') {
+ *out = (unsigned)(c - '0');
+ return 0;
+ }
+ if (c >= 'a' && c <= 'f') {
+ *out = (unsigned)(c - 'a') + 10u;
+ return 0;
+ }
+ if (c >= 'A' && c <= 'F') {
+ *out = (unsigned)(c - 'A') + 10u;
+ return 0;
+ }
+ return 1;
+}
+
+static char* fetch_decode_url_path(const char* s) {
+ char* out;
+ char* w;
+ size_t n;
+ if (!s) return NULL;
+ n = strlen(s);
+ out = (char*)malloc(n + 1u);
+ if (!out) return NULL;
+ w = out;
+ while (*s) {
+ if (*s == '%') {
+ unsigned hi, lo;
+ if (!s[1] || !s[2] || fetch_hex_val(s[1], &hi) ||
+ fetch_hex_val(s[2], &lo)) {
+ free(out);
+ return NULL;
+ }
+ if (((hi << 4) | lo) == 0u) {
+ free(out);
+ return NULL;
+ }
+ *w++ = (char)((hi << 4) | lo);
+ s += 3;
+ } else {
+ *w++ = *s++;
+ }
+ }
+ *w = '\0';
+ return out;
+}
+
+static char* fetch_file_url_path(const char* url) {
+ const char* p;
+ if (!url || strncmp(url, "file://", 7) != 0) return NULL;
+ p = url + 7;
+ if (strncmp(p, "localhost/", 10) == 0) {
+ p += 9; /* keep the slash before the absolute path */
+ } else if (p[0] != '/') {
+ return NULL;
+ }
+ return fetch_decode_url_path(p);
+}
+
+static int fetch_copy_file(const char* src, const char* dest) {
+ uint8_t buf[32768];
+ int in_fd, out_fd;
+ int rc = 1;
+ if (!src || !dest) return 1;
+ in_fd = open(src, O_RDONLY);
+ if (in_fd < 0) return 1;
+ out_fd = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0666);
+ if (out_fd < 0) {
+ close(in_fd);
+ return 1;
+ }
+ for (;;) {
+ ssize_t r = read(in_fd, buf, sizeof buf);
+ size_t off = 0;
+ if (r == 0) {
+ rc = 0;
+ break;
+ }
+ if (r < 0) {
+ if (errno == EINTR) continue;
+ break;
+ }
+ while (off < (size_t)r) {
+ ssize_t w = write(out_fd, buf + off, (size_t)r - off);
+ if (w > 0) {
+ off += (size_t)w;
+ } else if (w < 0 && errno == EINTR) {
+ continue;
+ } else {
+ goto out;
+ }
+ }
+ }
+out:
+ if (close(out_fd) != 0) rc = 1;
+ close(in_fd);
+ return rc;
+}
+
int driver_fetch_url(const char* url, const char* dest) {
pid_t pid;
int status;
+ char* file_path;
if (!url || !dest) return 1;
+ if (strncmp(url, "file://", 7) == 0) {
+ int rc;
+ file_path = fetch_file_url_path(url);
+ if (!file_path) return 1;
+ rc = fetch_copy_file(file_path, dest);
+ free(file_path);
+ return rc;
+ }
pid = fork();
if (pid < 0) return 1;
if (pid == 0) {
diff --git a/driver/env/windows.c b/driver/env/windows.c
@@ -1155,10 +1155,147 @@ int driver_kit_home(char* buf, size_t cap) {
return 0;
}
+static int fetch_hex_val(char c, unsigned* out) {
+ if (c >= '0' && c <= '9') {
+ *out = (unsigned)(c - '0');
+ return 0;
+ }
+ if (c >= 'a' && c <= 'f') {
+ *out = (unsigned)(c - 'a') + 10u;
+ return 0;
+ }
+ if (c >= 'A' && c <= 'F') {
+ *out = (unsigned)(c - 'A') + 10u;
+ return 0;
+ }
+ return 1;
+}
+
+static char* fetch_decode_url_path(const char* s) {
+ char* out;
+ char* w;
+ size_t n;
+ if (!s) return NULL;
+ n = strlen(s);
+ out = (char*)malloc(n + 1u);
+ if (!out) return NULL;
+ w = out;
+ while (*s) {
+ if (*s == '%') {
+ unsigned hi, lo;
+ if (!s[1] || !s[2] || fetch_hex_val(s[1], &hi) ||
+ fetch_hex_val(s[2], &lo)) {
+ free(out);
+ return NULL;
+ }
+ if (((hi << 4) | lo) == 0u) {
+ free(out);
+ return NULL;
+ }
+ *w++ = (char)((hi << 4) | lo);
+ s += 3;
+ } else {
+ *w++ = *s++;
+ }
+ }
+ *w = '\0';
+ return out;
+}
+
+static char* fetch_file_url_path(const char* url) {
+ const char* p;
+ char* decoded;
+ if (!url || strncmp(url, "file://", 7) != 0) return NULL;
+ p = url + 7;
+ if (strncmp(p, "localhost/", 10) == 0) {
+ p += 10;
+ } else if (p[0] == '/') {
+ ++p;
+ } else {
+ const char* slash = strchr(p, '/');
+ char* unc;
+ size_t host_len, rest_len;
+ if (!slash || slash == p) return NULL;
+ host_len = (size_t)(slash - p);
+ rest_len = strlen(slash + 1u);
+ unc = (char*)malloc(2u + host_len + 1u + rest_len + 1u);
+ if (!unc) return NULL;
+ unc[0] = '\\';
+ unc[1] = '\\';
+ memcpy(unc + 2u, p, host_len);
+ unc[2u + host_len] = '\\';
+ memcpy(unc + 2u + host_len + 1u, slash + 1u, rest_len + 1u);
+ decoded = fetch_decode_url_path(unc);
+ free(unc);
+ return decoded;
+ }
+ decoded = fetch_decode_url_path(p);
+ return decoded;
+}
+
+static int fetch_copy_file(const char* src, const char* dest) {
+ uint8_t buf[32768];
+ wchar_t *wsrc, *wdest;
+ HANDLE in_h, out_h;
+ int rc = 1;
+ wsrc = widen(src);
+ wdest = widen(dest);
+ if (!wsrc || !wdest) {
+ free(wsrc);
+ free(wdest);
+ return 1;
+ }
+ in_h = CreateFileW(wsrc, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING,
+ FILE_ATTRIBUTE_NORMAL, NULL);
+ free(wsrc);
+ if (in_h == INVALID_HANDLE_VALUE) {
+ free(wdest);
+ return 1;
+ }
+ out_h = CreateFileW(wdest, GENERIC_WRITE, FILE_SHARE_READ, NULL,
+ CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+ free(wdest);
+ if (out_h == INVALID_HANDLE_VALUE) {
+ CloseHandle(in_h);
+ return 1;
+ }
+ for (;;) {
+ DWORD got = 0;
+ if (!ReadFile(in_h, buf, sizeof buf, &got, NULL)) break;
+ if (got == 0u) {
+ rc = 0;
+ break;
+ }
+ {
+ DWORD off = 0;
+ while (off < got) {
+ DWORD wrote = 0;
+ if (!WriteFile(out_h, buf + off, got - off, &wrote, NULL) ||
+ wrote == 0u)
+ goto out;
+ off += wrote;
+ }
+ }
+ }
+out:
+ if (!CloseHandle(out_h)) rc = 1;
+ CloseHandle(in_h);
+ return rc;
+}
+
int driver_fetch_url(const char* url, const char* dest) {
wchar_t *wurl, *wdest;
intptr_t rc;
+ char* file_path;
if (!url || !dest) return 1;
+ if (strncmp(url, "file://", 7) == 0) {
+ int copy_rc;
+ file_path = fetch_file_url_path(url);
+ if (!file_path) return 1;
+ copy_rc = fetch_copy_file(file_path, dest);
+ free(file_path);
+ return copy_rc;
+ }
wurl = widen(url);
wdest = widen(dest);
if (!wurl || !wdest) {
diff --git a/driver/env_build.h b/driver/env_build.h
@@ -11,6 +11,7 @@ typedef struct DriverBuildHost {
KitBuildStoreIo store_io;
KitBuildExec exec;
KitBuildTransport transport;
+ KitBuildFetch fetch;
KitBuildHost host;
uint64_t next_endpoint;
} DriverBuildHost;
diff --git a/include/kit/build_coord.h b/include/kit/build_coord.h
@@ -172,6 +172,15 @@ typedef struct KitBuildStoreIo {
void* user;
} KitBuildStoreIo;
+/* Narrow hosted network fetch primitive for hash-pinned client fetches. The
+ * coordinator supplies an untrusted URL hint and a local destination path, then
+ * verifies the fetched bytes against the requested content id before installing
+ * them in the CAS. */
+typedef struct KitBuildFetch {
+ int (*fetch_url)(void* user, KitSlice url, KitSlice dest);
+ void* user;
+} KitBuildFetch;
+
/* Opaque concurrency primitives. */
typedef struct KitBuildThread KitBuildThread;
typedef struct KitBuildMutex KitBuildMutex;
@@ -206,6 +215,7 @@ typedef struct KitBuildHost {
const KitBuildStoreIo* store_io;
const KitBuildExec* exec;
const KitBuildTransport* transport;
+ const KitBuildFetch* fetch;
const KitBuildSched* sched;
void* user;
} KitBuildHost;
@@ -421,6 +431,14 @@ KIT_API KitStatus kit_build_client_source(KitBuildClient*, KitSlice path,
uint8_t blob[KIT_BUILD_HASH_LEN],
KitSlice* realpath);
+/* Fetch a hash-pinned blob into the coordinator CAS, trying URL hints in order.
+ * The URLs are transport hints only and are not cache identity; the expected
+ * blob id must come from tracked recipe/config/source data. On success *path is
+ * the local CAS blob path, borrowed until the next client call. */
+KIT_API KitStatus kit_build_client_fetch(
+ KitBuildClient*, const uint8_t blob[KIT_BUILD_HASH_LEN],
+ const KitSlice* urls, size_t nurls, KitSlice* path);
+
typedef enum KitBuildDepfileFlags {
KIT_BUILD_DEPFILE_DEFAULT = 0,
KIT_BUILD_DEPFILE_LINES = 1u << 0, /* one source path per non-empty line */
diff --git a/include/kit/cas.h b/include/kit/cas.h
@@ -82,6 +82,13 @@ KIT_API KitStatus kit_cas_add_blob(KitCas* cas, const uint8_t* data, size_t len,
KIT_API KitStatus kit_cas_has_blob(KitCas* cas,
const uint8_t id[KIT_CAS_HASH_LEN]);
+/* Return the canonical local path for a stored blob id under this CAS root.
+ * This does not verify presence; call kit_cas_has_blob or kit_cas_get_blob when
+ * the caller needs a checked read. */
+KIT_API KitStatus kit_cas_blob_path(KitCas* cas,
+ const uint8_t id[KIT_CAS_HASH_LEN],
+ char* out, size_t cap);
+
/* Read a stored blob's bytes by id. On success *out borrows the bytes; release
* them with kit_cas_release. Returns KIT_NOT_FOUND on a miss. This is the
* by-id read path that lets a recorded config-id / argv-id resolve back to the
diff --git a/src/api/build_coord.c b/src/api/build_coord.c
@@ -249,6 +249,30 @@ KitStatus kit_build_client_source(KitBuildClient* c, KitSlice path,
return KIT_OK;
}
+KitStatus kit_build_client_fetch(KitBuildClient* c,
+ const uint8_t blob[KIT_BUILD_HASH_LEN],
+ const KitSlice* urls, size_t nurls,
+ KitSlice* path) {
+ char hex[2u * KIT_BUILD_HASH_LEN + 1u];
+ BuildReq req;
+ BuildResp resp;
+ KitStatus st;
+ if (path) *path = KIT_SLICE_NULL;
+ if (!c || !blob || !urls || nurls == 0u || !path) return KIT_INVALID;
+ kit_hex_encode(hex, blob, KIT_BUILD_HASH_LEN);
+ memset(&req, 0, sizeof req);
+ req.cmd = BUILD_CMD_FETCH;
+ req.arg = kit_slice_cstr(hex);
+ req.argv = urls;
+ req.argc = nurls;
+ st = build_client_rpc(c, &req, &resp, NULL, NULL);
+ if (st != KIT_OK) return st;
+ if (resp.status != BUILD_RESP_OK || !build_id_eq(resp.id, blob))
+ return KIT_MALFORMED;
+ *path = resp.text;
+ return KIT_OK;
+}
+
static KitStatus build_client_depfile_submit(KitBuildClient* c,
const char* path, size_t len) {
uint8_t blob[KIT_BUILD_HASH_LEN];
diff --git a/src/api/cas.c b/src/api/cas.c
@@ -105,6 +105,13 @@ KitStatus kit_cas_has_blob(KitCas* cas,
return st;
}
+KitStatus kit_cas_blob_path(KitCas* cas, const uint8_t id[KIT_CAS_HASH_LEN],
+ char* out, size_t cap) {
+ if (!cas || !id || !out || cap == 0u) return KIT_INVALID;
+ return dist_cas_blob_path(out, cap, cas->dist.root, id) == DIST_OK ? KIT_OK
+ : KIT_ERR;
+}
+
KitStatus kit_cas_get_blob(KitCas* cas, const uint8_t id[KIT_CAS_HASH_LEN],
KitFileData* out) {
if (!cas || !id || !out) return KIT_INVALID;
diff --git a/src/build/build.h b/src/build/build.h
@@ -71,6 +71,7 @@
#define BUILD_KEY_MAX 64u /* config key name */
#define BUILD_VAL_MAX 512u /* config value */
#define BUILD_PATH_MAX 1024u /* workspace-relative or absolute fs path */
+#define BUILD_URL_MAX 1024u /* one fetch URL hint */
#define BUILD_PATTERN_MAX 256u /* glob pattern */
/* The two kinds of trace a build records. Written together on every real
diff --git a/src/build/coord.c b/src/build/coord.c
@@ -511,6 +511,64 @@ err:
return BUILD_ERR;
}
+int build_coord_fetch_blob(KitBuildCoordinator* c,
+ const uint8_t expected_blob[BUILD_HASH_LEN],
+ const KitSlice* urls, size_t nurls, char* path_out,
+ size_t path_cap) {
+ char tmp_parent[BUILD_PATH_MAX];
+ char tmp_dir[BUILD_PATH_MAX];
+ char dest[BUILD_PATH_MAX];
+ size_t i;
+ int installed = 0;
+ if (!c || !expected_blob || !urls || nurls == 0u || !path_out ||
+ path_cap == 0u)
+ return BUILD_ERR;
+ path_out[0] = '\0';
+ if (kit_cas_has_blob(c->cas, expected_blob) == KIT_OK)
+ return kit_cas_blob_path(c->cas, expected_blob, path_out, path_cap) ==
+ KIT_OK
+ ? BUILD_OK
+ : BUILD_ERR;
+ if (!c->host.fetch || !c->host.fetch->fetch_url || !c->host.store_io ||
+ !c->host.store_io->make_temp_dir || !c->host.store_io->remove)
+ return BUILD_ERR;
+ if (path_join2(tmp_parent, sizeof tmp_parent, c->store.root, "tmp") !=
+ BUILD_OK ||
+ c->host.store_io->make_temp_dir(c->host.store_io->user,
+ kit_slice_cstr(tmp_parent), tmp_dir,
+ sizeof tmp_dir) != 0 ||
+ path_join2(dest, sizeof dest, tmp_dir, "fetch") != BUILD_OK)
+ return BUILD_ERR;
+
+ for (i = 0; i < nurls; ++i) {
+ KitFileData fd;
+ KitBlobInfo info;
+ fd.data = NULL;
+ fd.size = 0;
+ fd.token = NULL;
+ if (c->host.fetch->fetch_url(c->host.fetch->user, urls[i],
+ kit_slice_cstr(dest)) != 0)
+ continue;
+ if (c->host.cas_host->file_io->read_all(c->host.cas_host->file_io->user,
+ dest, &fd) != KIT_OK)
+ continue;
+ kit_blob_info(&info, fd.data, fd.size);
+ if (build_id_eq(info.id, expected_blob) &&
+ kit_cas_add_blob(c->cas, fd.data, fd.size, &info) == KIT_OK)
+ installed = 1;
+ if (c->host.cas_host->file_io->release)
+ c->host.cas_host->file_io->release(c->host.cas_host->file_io->user, &fd);
+ if (installed) break;
+ }
+
+ (void)c->host.store_io->remove(c->host.store_io->user,
+ kit_slice_cstr(tmp_dir), 1);
+ if (!installed) return BUILD_ERR;
+ return kit_cas_blob_path(c->cas, expected_blob, path_out, path_cap) == KIT_OK
+ ? BUILD_OK
+ : BUILD_ERR;
+}
+
int build_coord_glob(KitBuildCoordinator* c, KitSlice pattern,
uint8_t out_result_hash[BUILD_HASH_LEN],
BuildCoordGlobFn cb, void* cb_user) {
diff --git a/src/build/coord.h b/src/build/coord.h
@@ -144,6 +144,15 @@ void build_coord_close(KitBuildCoordinator*);
int build_coord_source_hash(KitBuildCoordinator*, KitSlice path,
uint8_t out_blob[BUILD_HASH_LEN], int* present);
+/* Fetch a pinned blob into the CAS if absent, trying URL hints in order. Returns
+ * the verified local CAS blob path in path_out. The URL list is untrusted and
+ * not part of cache identity; only expected_blob is trusted after verification.
+ */
+int build_coord_fetch_blob(KitBuildCoordinator*,
+ const uint8_t expected_blob[BUILD_HASH_LEN],
+ const KitSlice* urls, size_t nurls, char* path_out,
+ size_t path_cap);
+
/* Expand a glob and return its glob-result-hash; optionally enumerate the
* sorted matches via cb. Memoized per process. */
typedef int (*BuildCoordGlobFn)(void* user, const char* path,
diff --git a/src/build/protocol.c b/src/build/protocol.c
@@ -108,19 +108,33 @@ static KitSlice get_slice(Cursor* c) {
static int known_cmd(uint8_t cmd) {
return cmd == BUILD_CMD_CONFIG_GET || cmd == BUILD_CMD_SOURCE ||
cmd == BUILD_CMD_GLOB || cmd == BUILD_CMD_NEED ||
- cmd == BUILD_CMD_NEED_SUBMIT || cmd == BUILD_CMD_NEED_AWAIT;
+ cmd == BUILD_CMD_NEED_SUBMIT || cmd == BUILD_CMD_NEED_AWAIT ||
+ cmd == BUILD_CMD_FETCH;
}
static int token_valid(KitSlice s, size_t cap) {
return s.len > 0 && text_valid(s, cap, 0);
}
+static int hex_id_valid(KitSlice s) {
+ size_t i;
+ if (s.len != 2u * BUILD_HASH_LEN || !s.s) return 0;
+ for (i = 0; i < s.len; ++i) {
+ unsigned char c = (unsigned char)s.s[i];
+ if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
+ (c >= 'A' && c <= 'F')))
+ return 0;
+ }
+ return 1;
+}
+
static int valid_arg_for_cmd(uint8_t cmd, KitSlice s) {
if (cmd == BUILD_CMD_CONFIG_GET) return token_valid(s, BUILD_KEY_MAX);
if (cmd == BUILD_CMD_SOURCE) return token_valid(s, BUILD_PATH_MAX);
if (cmd == BUILD_CMD_GLOB) return token_valid(s, BUILD_PATTERN_MAX);
if (cmd == BUILD_CMD_NEED || cmd == BUILD_CMD_NEED_SUBMIT)
return token_valid(s, BUILD_TARGET_MAX);
+ if (cmd == BUILD_CMD_FETCH) return hex_id_valid(s);
return 1;
}
@@ -140,6 +154,16 @@ int build_proto_encode_req(const BuildReq* req, uint8_t* buf, size_t cap,
req->cmd == BUILD_CMD_GLOB) {
if (!valid_arg_for_cmd(req->cmd, req->arg)) return BUILD_ERR;
put_slice(&c, req->arg);
+ } else if (req->cmd == BUILD_CMD_FETCH) {
+ if (!valid_arg_for_cmd(req->cmd, req->arg)) return BUILD_ERR;
+ if (req->argc == 0u || req->argc > 0xffffu || !req->argv)
+ return BUILD_ERR;
+ put_slice(&c, req->arg);
+ put_u16(&c, (uint16_t)req->argc);
+ for (i = 0; i < req->argc; ++i) {
+ if (!token_valid(req->argv[i], BUILD_URL_MAX)) return BUILD_ERR;
+ put_slice(&c, req->argv[i]);
+ }
} else if (req->cmd == BUILD_CMD_NEED ||
req->cmd == BUILD_CMD_NEED_SUBMIT) {
if (!valid_arg_for_cmd(req->cmd, req->arg)) return BUILD_ERR;
@@ -186,6 +210,19 @@ int build_proto_decode_req(const uint8_t* buf, size_t len, BuildReq* out,
out->cmd == BUILD_CMD_GLOB) {
out->arg = get_slice(&c);
if (!valid_arg_for_cmd(out->cmd, out->arg)) return BUILD_ERR;
+ } else if (out->cmd == BUILD_CMD_FETCH) {
+ uint16_t argc;
+ out->arg = get_slice(&c);
+ if (!valid_arg_for_cmd(out->cmd, out->arg)) return BUILD_ERR;
+ argc = get_u16(&c);
+ if (argc == 0u || (size_t)argc > argv_cap || !argv_storage)
+ return BUILD_ERR;
+ out->argv = argv_storage;
+ out->argc = argc;
+ for (i = 0; i < out->argc; ++i) {
+ argv_storage[i] = get_slice(&c);
+ if (!token_valid(argv_storage[i], BUILD_URL_MAX)) return BUILD_ERR;
+ }
} else if (out->cmd == BUILD_CMD_NEED ||
out->cmd == BUILD_CMD_NEED_SUBMIT) {
uint16_t noverrides, argc;
@@ -257,6 +294,12 @@ int build_proto_encode_resp(const BuildReq* for_cmd, const BuildResp* resp,
} else {
return BUILD_ERR;
}
+ } else if (for_cmd->cmd == BUILD_CMD_FETCH) {
+ if (resp->status != BUILD_RESP_OK ||
+ !token_valid(resp->text, BUILD_PATH_MAX))
+ return BUILD_ERR;
+ put_bytes(&c, resp->id, BUILD_HASH_LEN);
+ put_slice(&c, resp->text);
} else if (for_cmd->cmd == BUILD_CMD_GLOB) {
if (resp->status == BUILD_RESP_GLOB_END) {
} else if (resp->status == BUILD_RESP_OK) {
@@ -323,6 +366,11 @@ int build_proto_decode_resp(const uint8_t* buf, size_t len, uint8_t cmd,
} else {
return BUILD_ERR;
}
+ } else if (cmd == BUILD_CMD_FETCH) {
+ if (out->status != BUILD_RESP_OK) return BUILD_ERR;
+ get_bytes(&c, out->id, BUILD_HASH_LEN);
+ out->text = get_slice(&c);
+ if (!token_valid(out->text, BUILD_PATH_MAX)) return BUILD_ERR;
} else if (cmd == BUILD_CMD_GLOB) {
if (out->status == BUILD_RESP_GLOB_END) {
} else if (out->status == BUILD_RESP_OK) {
diff --git a/src/build/protocol.h b/src/build/protocol.h
@@ -50,6 +50,7 @@ typedef enum BuildCmd {
BUILD_CMD_NEED = 4, /* arg: target + k=v[] + argv[] -> tree-id + path (blocking) */
BUILD_CMD_NEED_SUBMIT = 5, /* arg: target + k=v[] + argv[] -> token (non-blocking) */
BUILD_CMD_NEED_AWAIT = 6, /* token -> tree-id + path */
+ BUILD_CMD_FETCH = 7, /* arg: hex blob-id + url[] -> blob-id + path */
} BuildCmd;
typedef enum BuildRespStatus {
@@ -68,8 +69,8 @@ typedef struct BuildReq {
KitSlice arg; /* key | path | pattern | target (unused for AWAIT) */
const KitBuildKV* overrides; /* need/submit: propagated-config overlay */
size_t noverrides;
- const KitSlice* argv; /* need/submit: local argv (NULL or argc==0 => empty argv) */
- size_t argc;
+ const KitSlice* argv; /* need/submit: local argv; fetch: ordered URL hints */
+ size_t argc; /* need/submit argv count; fetch URL count */
uint64_t token; /* need-await: which submitted need to collect */
} BuildReq;
diff --git a/src/build/runner.c b/src/build/runner.c
@@ -656,6 +656,18 @@ int build_runner_service(KitBuildCoordinator* c, KitBuildConn* conn,
resp.text = kit_slice_cstr(full);
}
if (write_resp(c, conn, &req, &resp) != BUILD_OK) return BUILD_ERR;
+ } else if (req.cmd == BUILD_CMD_FETCH) {
+ uint8_t blob[BUILD_HASH_LEN];
+ char path[BUILD_PATH_MAX];
+ if (kit_hex_decode(blob, req.arg.s, BUILD_HASH_LEN) != KIT_OK ||
+ build_coord_fetch_blob(c, blob, req.argv, req.argc, path,
+ sizeof path) != BUILD_OK) {
+ resp_error(&resp, KIT_ERR, "fetch failed");
+ } else {
+ memcpy(resp.id, blob, BUILD_HASH_LEN);
+ resp.text = kit_slice_cstr(path);
+ }
+ if (write_resp(c, conn, &req, &resp) != BUILD_OK) return BUILD_ERR;
} else if (req.cmd == BUILD_CMD_GLOB) {
BuildGlobStream stream;
uint8_t result_hash[BUILD_HASH_LEN];
diff --git a/test/buildcoord/run.sh b/test/buildcoord/run.sh
@@ -22,7 +22,7 @@ kit_report_init
ws="$work/ws"
store="$work/store"
-mkdir -p "$ws/recipes" "$ws/src/depfile" "$ws/src/globset" \
+mkdir -p "$ws/recipes" "$ws/remote" "$ws/src/depfile" "$ws/src/globset" \
"$ws/src/tree/nested/deeper" "$store"
cat > "$ws/BUILD.kit" <<'EOF'
@@ -53,6 +53,8 @@ recipe recipes/depfile.sh
recipe recipes/env_probe.sh
[target //fail:probe]
recipe recipes/fail.sh
+[target //fetch:blob]
+recipe recipes/fetch.sh
[target //globset:probe]
recipe recipes/globset.sh
[target //globstar:probe]
@@ -122,6 +124,9 @@ EOF
cat > "$ws/src/tree/nested/deeper/leaf.txt" <<'EOF'
leaf
EOF
+cat > "$ws/remote/fetch.txt" <<'EOF'
+remote:v1
+EOF
cat > "$ws/recipes/lib.sh" <<'EOF'
#!/bin/sh
@@ -273,6 +278,16 @@ printf '%s\n' "$count" > fail.count
exit 1
EOF
+cat > "$ws/recipes/fetch.sh" <<'EOF'
+#!/bin/sh
+set -eu
+blob=$1
+shift
+p=$("$KIT" build fetch "$blob" "$@")
+IFS= read -r line < "$p"
+printf '%s\n' "$line" > "$KIT_BUILD_OUT/fetched.txt"
+EOF
+
cat > "$ws/recipes/globset.sh" <<'EOF'
#!/bin/sh
set -eu
@@ -1059,6 +1074,24 @@ fail_fixed_path=$(tree_path_from "$work/buildcoord-failure-fixed-runs.out")
contains "buildcoord-failure-fixed-output" "$fail_fixed_path/fixed.txt" \
"fixed"
+fetch_id=$("$KIT" hash -a blake2b "$ws/remote/fetch.txt" | awk '{print $1}')
+build_assert_ok buildcoord-fetch-mirror --stats --config env.PATH=/no/such \
+ //fetch:blob -- "$fetch_id" \
+ "file://$work/missing-fetch.txt" "file://$ws/remote/fetch.txt"
+contains "buildcoord-fetch-mirror-run" "$work/buildcoord-fetch-mirror.err" \
+ "recipes_run=1"
+fetch_path=$(tree_path_from "$work/buildcoord-fetch-mirror.out")
+contains "buildcoord-fetch-mirror-output" "$fetch_path/fetched.txt" \
+ "remote:v1"
+build_assert_ok buildcoord-fetch-cas-hit-bad-url --stats \
+ --config env.PATH=/no/such //fetch:blob -- \
+ "$fetch_id" "file://$work/still-missing-fetch.txt"
+contains "buildcoord-fetch-cas-hit-rerun" \
+ "$work/buildcoord-fetch-cas-hit-bad-url.err" "recipes_run=1"
+fetch_hit_path=$(tree_path_from "$work/buildcoord-fetch-cas-hit-bad-url.out")
+contains "buildcoord-fetch-cas-hit-output" "$fetch_hit_path/fetched.txt" \
+ "remote:v1"
+
run_fail "buildcoord-cycle-self-fails" \
"$KIT" build --store "$store" --root "$ws" --def BUILD.kit \
--config "env.KIT=$KIT" //cycle:self
