commit 191086c86039bef04b03180077d01a52acf86360
parent 25c31e015969ea48307092158c5aaf6be03485d0
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Thu, 18 Jun 2026 11:32:19 -0700
Record build fetch blobs in traces
Diffstat:
12 files changed, 328 insertions(+), 40 deletions(-)
diff --git a/doc/BUILD_COORDINATOR.md b/doc/BUILD_COORDINATOR.md
@@ -54,17 +54,17 @@ the resolution algorithm that drives it.
| **Test target request** | A request to execute a target as a test (`kit build test ...`). It uses the same recipe protocol and dependency discovery as a build request, but returns a typed test result. |
| **Test result tree** | The directory a test recipe produces, plus coordinator-injected `stdout` and `stderr` files, captured as a CAS tree (`result-tree-id`). |
| **PASS-only test trace** | A test trace is a cacheable claim only when the recipe exits 0. Nonzero exits return FAIL results for inspection but write no reusable trace. |
-| **Base inputs** | The *leaves* of the dependency graph, dynamically requested by recipes: **config values**, **source files**, and **globs**. |
+| **Base inputs** | The *leaves* of the dependency graph, dynamically requested by recipes: **config values**, **source files**, **globs**, and hash-pinned **fetch blobs**. |
| **Target dep** | A dependency of one target on the *output tree* of another. The graph's interior edges, created by `need`. |
| **Shallow trace** | A record of one build: its direct base inputs + its direct target deps (by output `tree-id`) → its output `tree-id`. |
| **Deep trace** | A record of one build: `(root config-id, argv)` + a pointer to the transitive input closure (a **deepset**) → its output `tree-id`. |
-| **Deepset** | One node of the transitive input-closure DAG (`deep-set-id` = `BLAKE2b(canonical body)`): a target's recipe-id + its direct source/glob leaves + its children's `deep-set-id`s. A self-verifying CAS blob; shared structurally across ancestors. |
+| **Deepset** | One node of the transitive input-closure DAG (`deep-set-id` = `BLAKE2b(canonical body)`): a target's recipe-id + its direct source/glob/fetch leaves + its children's `deep-set-id`s. A self-verifying CAS blob; shared structurally across ancestors. |
| **Tree cache** | On-disk *materialized* output directories, keyed by `tree-id`, ready to hand back as a filesystem path. |
-The three base-input kinds are exactly the dynamically-requested leaves: a recipe
-asks the coordinator for a config value, the bytes of a source file, or the
-expansion of a glob, and each request is logged as a dependency so a later
-rebuild knows precisely what to re-check.
+The base-input kinds are exactly the dynamically-requested leaves: a recipe asks
+the coordinator for a config value, the bytes of a source file, the expansion of
+a glob, or a hash-pinned fetched blob, and each request is logged as a dependency
+so a later rebuild knows precisely what to re-check.
## Build definitions
@@ -232,6 +232,8 @@ opt
src/main.c <blob-id>
[glob] ; sorted by pattern
src/*.c <glob-result-hash>
+[fetch] ; sorted by blob id
+<blob-id>
[dep] ; sorted by (dep-name, dep-config-id, dep-argv-id) — each row replays one `need`
//lib:core <dep-config-id> <dep-argv-id> <dep-output-tree-id>
```
@@ -303,6 +305,8 @@ src/main.c <blob-id>
config.h - ; an ABSENT leaf: "-" = the path was absent when read
[glob] ; this node's DIRECT glob leaves, sorted by pattern
src/*.c <glob-result-hash>
+[fetch] ; this node's DIRECT fetched blobs, sorted by blob id
+<blob-id>
[child] ; direct deps' deep-set-ids, sorted
<deep-set-id of //lib:core's node>
```
@@ -338,19 +342,24 @@ src/*.c <glob-result-hash>
pattern matched (sorted by path). It changes if any matched file is added,
removed, or edited — so one glob row covers the existence *and* content of its
whole match set, and files read through a glob need no separate `[source]` rows.
+- `[fetch]` rows record hash-pinned external blobs requested through the recipe
+ protocol. Only the blob id is cache identity; URL hints are deliberately not
+ recorded because they are transport choices. A matching trace requires the
+ blob to be present in the local CAS, and trace bundles include these blobs so
+ imported traces retain the real dependency.
- `[dep]` rows carry the **dep's config-id and argv-id** (and output `tree-id`);
together with the dep's own serialized config/argv blobs this re-resolves each
dep under the exact propagated config *and* argv it used, overlays included.
**Why the deep trace needs only `root-config` + the deepset closure.** If the
request's `config-id` equals `root-config`, its `argv-id` equals the recorded
-`argv`, *and* refreshing the deepset finds nothing moved — every source/glob leaf
+`argv`, *and* refreshing the deepset finds nothing moved — every source/glob/fetch leaf
still matches the live workspace **and** every node's `recipe-id` still recomputes
the same through the live definition — then every recipe in the closure has
byte-identical inputs *and* identical code, so every `need` overlay it computes is
identical, every effective config downstream is identical, and by determinism the
output is identical. Any input that could perturb a downstream overlay (a source a
-recipe branches on, a config value) is itself either a recorded source/glob leaf,
+recipe branches on, a config value) is itself either a recorded source/glob/fetch leaf,
a per-node `recipe-id`, or folded into `root-config`, so nothing escapes the
check. Using the whole `root-config` (rather than just
consumed keys) is a deliberate, conservative simplification: an *unconsumed*
@@ -490,14 +499,14 @@ resolve(T, cfg, argv, chain):
if S.argv != argv.id: continue # local config
M = config_by_id(S.config) # the built map (serialized blob)
if any consumed key in S.config-keys differs between cfg and M: continue
- if any direct source/glob leaf of S changed: continue
+ if any direct source/glob/fetch leaf of S changed: continue
ok = true ; child_nodes = []
for (dep, dep_cfg_id, dep_argv_id, recorded_tree) in S.deps: # may run in parallel
r = resolve(dep, config_by_id(dep_cfg_id), argv_by_id(dep_argv_id), chain') # replay the need by id
if r.output != recorded_tree: ok = false; break
child_nodes.push(r.leafset)
if ok:
- node = union(direct{recipe_id, source/glob leaves} of S, child_nodes) # builds+stores the deepset
+ node = union(direct{recipe_id, source/glob/fetch leaves} of S, child_nodes) # builds+stores the deepset
write_deep_trace(T, cfg.id, argv.id, node.id, S.output) # refresh the deep trace
p = materialize(S.output) # pure; ERR => fall through
if p ok: return done(S.output, p, node)
@@ -526,7 +535,7 @@ tree), and removes the resolve→runner edge.
This is the brief's flow, sharpened by config:
- *Phase 1* is the deep fast path: identical config-id **and argv-id** and an
- all-clear deepset refresh — every source/glob leaf unchanged **and** every
+ all-clear deepset refresh — every source/glob/fetch leaf unchanged **and** every
node's `recipe-id` still recomputing the same — ⇒ reuse. `refresh` hits the
per-process memos and the deep-set-id validity cache, so a shared subtree is
checked once and an unchanged subtree is skipped by id equality.
@@ -602,7 +611,7 @@ run_recipe(T, cfg, argv, chain):
on success:
output = kit_cas_add_tree_from_dir(out)
put serialized cfg map (cfg.id) and argv vector (argv.id) into the CAS
- node = union(direct{recipe_id, source/glob/absent leaves}, child deepset nodes)
+ node = union(direct{recipe_id, source/glob/fetch/absent leaves}, child deepset nodes)
# builds + stores the root deepset as a CAS blob, yields its deep-set-id
write_shallow_trace(T, recipe_id, argv.id, cfg.id, consumed key names,
direct leaves, dep edges with their config-ids and argv-ids, output)
@@ -678,6 +687,7 @@ command.
| `config-get <key>` | the propagated value (or *unset*) | config dep: key recorded as consumed; its value is the one in the effective `config-id` map (absent = unset) |
| `source <path>` | `blob-id` (or *absent*) + a path to read | source dep: `(path, blob-id)`, or `(path, absent)` |
| `glob <pattern>` | sorted list of matching paths (streamed) | glob dep: `(pattern, glob-result-hash)` |
+| `fetch <blob-id> <url>...` | local CAS path to the verified blob | fetch dep: `blob-id` |
| `need <target> [k=v…] [env…] [argv…]` | the dep's output `tree-id` + a path (blocks) | target dep edge: `(dep, dep-config-id, dep-argv-id, output-tree-id)` |
| `need-submit <target> [k=v…] [env…] [argv…]` | a **token** (does not block) | *(nothing yet — logged on await)* |
| `need-await <token>` | the submitted dep's output `tree-id` + a path | target dep edge: `(dep, dep-config-id, dep-argv-id, output-tree-id)` |
@@ -704,6 +714,11 @@ command.
recipe then reads the returned paths without further declaration. Matches are
**streamed** across as many frames as needed, so an arbitrarily large match set
is not bounded by a single frame.
+- **`fetch`** retrieves an externally stored blob by expected content id, trying
+ URL hints in order and installing only bytes that hash to that id. The logged
+ dependency is the blob id itself. URLs are not cache identity: changing mirrors
+ or URL order does not make a different build as long as the same blob id is
+ requested.
- **`need`** is the dynamic-dependency primitive. The optional `k=v` pairs
**overlay** propagated config for that sub-build; `--env NAME` /
`--env NAME=value` are accepted as env-prefixed config overlays; an optional
@@ -791,8 +806,9 @@ machinery wholesale:
trace bodies and the CAS blobs they reference — the **serialized config-map and
argv blobs** (required, else an imported shallow trace cannot replay its
`need`s) and the **deepset closure blobs** a deep trace points at (required,
- else an imported deep trace cannot refresh) — plus, optionally, the referenced
- output trees/blobs.
+ else an imported deep trace cannot refresh), including any **fetched blob**
+ leaves in shallow traces or deepsets — plus, optionally, the referenced output
+ trees/blobs.
- Trust is the DISTRIBUTE.md model unchanged: verify the minisign signature
against the trusted-keys file (`-p KEY`, the anchor file, or `--tofu`); the
signed trusted comment binds the signature to the manifest hash.
diff --git a/include/kit/build_coord.h b/include/kit/build_coord.h
@@ -431,10 +431,10 @@ KIT_API KitStatus kit_build_client_source(KitBuildClient*, KitSlice path,
uint8_t blob[KIT_BUILD_HASH_LEN],
KitSlice* realpath);
-/* Fetch a hash-pinned blob into the coordinator CAS, trying URL hints in order.
- * The URLs are transport hints only and are not cache identity; the expected
- * blob id must come from tracked recipe/config/source data. On success *path is
- * the local CAS blob path, borrowed until the next client call. */
+/* Fetch a hash-pinned blob into the coordinator CAS, trying URL hints in order
+ * and logging the blob id as a dependency. The URLs are transport hints only
+ * and are not cache identity. On success *path is the local CAS blob path,
+ * borrowed until the next client call. */
KIT_API KitStatus kit_build_client_fetch(
KitBuildClient*, const uint8_t blob[KIT_BUILD_HASH_LEN],
const KitSlice* urls, size_t nurls, KitSlice* path);
diff --git a/src/build/bundle.c b/src/build/bundle.c
@@ -354,23 +354,28 @@ static int BUILD_MAYBE_UNUSED parse_trace_header(const KitContext* ctx,
_Alignof(BuildSourceLeaf));
t.globs = (BuildGlobLeaf*)heap_array(ctx, rows, sizeof *t.globs,
_Alignof(BuildGlobLeaf));
+ t.fetches = (BuildFetchLeaf*)heap_array(ctx, rows, sizeof *t.fetches,
+ _Alignof(BuildFetchLeaf));
t.deps = (BuildDepEdge*)heap_array(ctx, rows, sizeof *t.deps,
_Alignof(BuildDepEdge));
- if (!t.config_keys || !t.sources || !t.globs || !t.deps) {
+ if (!t.config_keys || !t.sources || !t.globs || !t.fetches || !t.deps) {
heap_free_array(ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(ctx, t.deps, rows, sizeof *t.deps);
return BUILD_ERR;
}
t.cap_config_keys = rows;
t.cap_sources = rows;
t.cap_globs = rows;
+ t.cap_fetches = rows;
t.cap_deps = rows;
if (build_shallow_parse(data, len, &t, err, sizeof err) != BUILD_OK) {
heap_free_array(ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(ctx, t.deps, rows, sizeof *t.deps);
return BUILD_ERR;
}
@@ -380,6 +385,7 @@ static int BUILD_MAYBE_UNUSED parse_trace_header(const KitContext* ctx,
heap_free_array(ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(ctx, t.deps, rows, sizeof *t.deps);
return BUILD_OK;
}
@@ -395,23 +401,28 @@ static int BUILD_MAYBE_UNUSED parse_trace_header(const KitContext* ctx,
_Alignof(BuildSourceLeaf));
t.globs = (BuildGlobLeaf*)heap_array(ctx, rows, sizeof *t.globs,
_Alignof(BuildGlobLeaf));
+ t.fetches = (BuildFetchLeaf*)heap_array(ctx, rows, sizeof *t.fetches,
+ _Alignof(BuildFetchLeaf));
t.deps = (BuildDepEdge*)heap_array(ctx, rows, sizeof *t.deps,
_Alignof(BuildDepEdge));
- if (!t.config_keys || !t.sources || !t.globs || !t.deps) {
+ if (!t.config_keys || !t.sources || !t.globs || !t.fetches || !t.deps) {
heap_free_array(ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(ctx, t.deps, rows, sizeof *t.deps);
return BUILD_ERR;
}
t.cap_config_keys = rows;
t.cap_sources = rows;
t.cap_globs = rows;
+ t.cap_fetches = rows;
t.cap_deps = rows;
if (build_test_shallow_parse(data, len, &t, err, sizeof err) != BUILD_OK) {
heap_free_array(ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(ctx, t.deps, rows, sizeof *t.deps);
return BUILD_ERR;
}
@@ -421,6 +432,7 @@ static int BUILD_MAYBE_UNUSED parse_trace_header(const KitContext* ctx,
heap_free_array(ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(ctx, t.deps, rows, sizeof *t.deps);
return BUILD_OK;
}
@@ -618,14 +630,21 @@ static int export_deepset_closure(KitBuildCoordinator* c,
_Alignof(BuildSourceLeaf));
ds.globs = (BuildGlobLeaf*)heap_array(c->ctx, rows, sizeof *ds.globs,
_Alignof(BuildGlobLeaf));
+ ds.fetches = (BuildFetchLeaf*)heap_array(c->ctx, rows, sizeof *ds.fetches,
+ _Alignof(BuildFetchLeaf));
ds.children = (uint8_t(*)[BUILD_HASH_LEN])heap_array(
c->ctx, rows, sizeof *ds.children, _Alignof(uint8_t));
- if (!ds.sources || !ds.globs || !ds.children) goto out;
+ if (!ds.sources || !ds.globs || !ds.fetches || !ds.children) goto out;
ds.cap_sources = rows;
ds.cap_globs = rows;
+ ds.cap_fetches = rows;
ds.cap_children = rows;
if (build_deepset_parse(fd.data, fd.size, &ds, err, sizeof err) != BUILD_OK)
goto out;
+ for (i = 0u; i < ds.n_fetches; ++i)
+ if (payload_write_blob_from_cas(c, ds.fetches[i].blob, root, seen) !=
+ BUILD_OK)
+ goto out;
for (i = 0u; i < ds.n_children; ++i)
if (export_deepset_closure(c, ds.children[i], root, seen) != BUILD_OK)
goto out;
@@ -633,6 +652,7 @@ static int export_deepset_closure(KitBuildCoordinator* c,
out:
heap_free_array(c->ctx, ds.sources, rows, sizeof *ds.sources);
heap_free_array(c->ctx, ds.globs, rows, sizeof *ds.globs);
+ heap_free_array(c->ctx, ds.fetches, rows, sizeof *ds.fetches);
heap_free_array(c->ctx, ds.children, rows, sizeof *ds.children);
kit_cas_release(c->cas, &fd);
return ok;
@@ -658,14 +678,25 @@ static int import_deepset_closure(KitBuildCoordinator* c,
_Alignof(BuildSourceLeaf));
ds.globs = (BuildGlobLeaf*)heap_array(c->ctx, rows, sizeof *ds.globs,
_Alignof(BuildGlobLeaf));
+ ds.fetches = (BuildFetchLeaf*)heap_array(c->ctx, rows, sizeof *ds.fetches,
+ _Alignof(BuildFetchLeaf));
ds.children = (uint8_t(*)[BUILD_HASH_LEN])heap_array(
c->ctx, rows, sizeof *ds.children, _Alignof(uint8_t));
- if (!ds.sources || !ds.globs || !ds.children) goto out;
+ if (!ds.sources || !ds.globs || !ds.fetches || !ds.children) goto out;
ds.cap_sources = rows;
ds.cap_globs = rows;
+ ds.cap_fetches = rows;
ds.cap_children = rows;
if (build_deepset_parse(data, len, &ds, err, sizeof err) != BUILD_OK)
goto out;
+ for (i = 0u; i < ds.n_fetches; ++i) {
+ KitFileData dep_fd;
+ memset(&dep_fd, 0, sizeof dep_fd);
+ if (payload_install_blob(c, ds.fetches[i].blob, root, seen, NULL, NULL,
+ &dep_fd) != BUILD_OK)
+ goto out;
+ release_file(c, &dep_fd);
+ }
for (i = 0u; i < ds.n_children; ++i)
if (import_deepset_closure(c, ds.children[i], root, seen) != BUILD_OK)
goto out;
@@ -673,6 +704,7 @@ static int import_deepset_closure(KitBuildCoordinator* c,
out:
heap_free_array(c->ctx, ds.sources, rows, sizeof *ds.sources);
heap_free_array(c->ctx, ds.globs, rows, sizeof *ds.globs);
+ heap_free_array(c->ctx, ds.fetches, rows, sizeof *ds.fetches);
heap_free_array(c->ctx, ds.children, rows, sizeof *ds.children);
release_file(c, &fd);
return ok;
@@ -721,12 +753,16 @@ static int BUILD_MAYBE_UNUSED export_trace_refs(KitBuildCoordinator* c,
_Alignof(BuildSourceLeaf));
t.globs = (BuildGlobLeaf*)heap_array(c->ctx, rows, sizeof *t.globs,
_Alignof(BuildGlobLeaf));
+ t.fetches = (BuildFetchLeaf*)heap_array(c->ctx, rows, sizeof *t.fetches,
+ _Alignof(BuildFetchLeaf));
t.deps = (BuildDepEdge*)heap_array(c->ctx, rows, sizeof *t.deps,
_Alignof(BuildDepEdge));
- if (!t.config_keys || !t.sources || !t.globs || !t.deps) goto shallow_out;
+ if (!t.config_keys || !t.sources || !t.globs || !t.fetches || !t.deps)
+ goto shallow_out;
t.cap_config_keys = rows;
t.cap_sources = rows;
t.cap_globs = rows;
+ t.cap_fetches = rows;
t.cap_deps = rows;
if (build_shallow_parse(data, len, &t, err, sizeof err) != BUILD_OK)
goto shallow_out;
@@ -742,11 +778,16 @@ static int BUILD_MAYBE_UNUSED export_trace_refs(KitBuildCoordinator* c,
BUILD_OK)
goto shallow_out;
}
+ for (i = 0u; i < t.n_fetches; ++i)
+ if (payload_write_blob_from_cas(c, t.fetches[i].blob, root, seen) !=
+ BUILD_OK)
+ goto shallow_out;
ok = BUILD_OK;
shallow_out:
heap_free_array(c->ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(c->ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(c->ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(c->ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(c->ctx, t.deps, rows, sizeof *t.deps);
return ok;
}
@@ -764,12 +805,16 @@ shallow_out:
_Alignof(BuildSourceLeaf));
t.globs = (BuildGlobLeaf*)heap_array(c->ctx, rows, sizeof *t.globs,
_Alignof(BuildGlobLeaf));
+ t.fetches = (BuildFetchLeaf*)heap_array(c->ctx, rows, sizeof *t.fetches,
+ _Alignof(BuildFetchLeaf));
t.deps = (BuildDepEdge*)heap_array(c->ctx, rows, sizeof *t.deps,
_Alignof(BuildDepEdge));
- if (!t.config_keys || !t.sources || !t.globs || !t.deps) goto test_out;
+ if (!t.config_keys || !t.sources || !t.globs || !t.fetches || !t.deps)
+ goto test_out;
t.cap_config_keys = rows;
t.cap_sources = rows;
t.cap_globs = rows;
+ t.cap_fetches = rows;
t.cap_deps = rows;
if (build_test_shallow_parse(data, len, &t, err, sizeof err) != BUILD_OK)
goto test_out;
@@ -785,11 +830,16 @@ shallow_out:
BUILD_OK)
goto test_out;
}
+ for (i = 0u; i < t.n_fetches; ++i)
+ if (payload_write_blob_from_cas(c, t.fetches[i].blob, root, seen) !=
+ BUILD_OK)
+ goto test_out;
ok = BUILD_OK;
test_out:
heap_free_array(c->ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(c->ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(c->ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(c->ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(c->ctx, t.deps, rows, sizeof *t.deps);
return ok;
}
@@ -849,12 +899,16 @@ static int BUILD_MAYBE_UNUSED import_trace_refs(KitBuildCoordinator* c,
_Alignof(BuildSourceLeaf));
t.globs = (BuildGlobLeaf*)heap_array(c->ctx, rows, sizeof *t.globs,
_Alignof(BuildGlobLeaf));
+ t.fetches = (BuildFetchLeaf*)heap_array(c->ctx, rows, sizeof *t.fetches,
+ _Alignof(BuildFetchLeaf));
t.deps = (BuildDepEdge*)heap_array(c->ctx, rows, sizeof *t.deps,
_Alignof(BuildDepEdge));
- if (!t.config_keys || !t.sources || !t.globs || !t.deps) goto shallow_out;
+ if (!t.config_keys || !t.sources || !t.globs || !t.fetches || !t.deps)
+ goto shallow_out;
t.cap_config_keys = rows;
t.cap_sources = rows;
t.cap_globs = rows;
+ t.cap_fetches = rows;
t.cap_deps = rows;
if (build_shallow_parse(data, len, &t, err, sizeof err) != BUILD_OK)
goto shallow_out;
@@ -880,12 +934,19 @@ static int BUILD_MAYBE_UNUSED import_trace_refs(KitBuildCoordinator* c,
goto shallow_out;
release_file(c, &fd);
}
+ for (i = 0u; i < t.n_fetches; ++i) {
+ if (payload_install_blob(c, t.fetches[i].blob, root, seen, NULL, NULL,
+ &fd) != BUILD_OK)
+ goto shallow_out;
+ release_file(c, &fd);
+ }
ok = BUILD_OK;
shallow_out:
release_file(c, &fd);
heap_free_array(c->ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(c->ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(c->ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(c->ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(c->ctx, t.deps, rows, sizeof *t.deps);
return ok;
}
@@ -905,12 +966,16 @@ shallow_out:
_Alignof(BuildSourceLeaf));
t.globs = (BuildGlobLeaf*)heap_array(c->ctx, rows, sizeof *t.globs,
_Alignof(BuildGlobLeaf));
+ t.fetches = (BuildFetchLeaf*)heap_array(c->ctx, rows, sizeof *t.fetches,
+ _Alignof(BuildFetchLeaf));
t.deps = (BuildDepEdge*)heap_array(c->ctx, rows, sizeof *t.deps,
_Alignof(BuildDepEdge));
- if (!t.config_keys || !t.sources || !t.globs || !t.deps) goto test_out;
+ if (!t.config_keys || !t.sources || !t.globs || !t.fetches || !t.deps)
+ goto test_out;
t.cap_config_keys = rows;
t.cap_sources = rows;
t.cap_globs = rows;
+ t.cap_fetches = rows;
t.cap_deps = rows;
if (build_test_shallow_parse(data, len, &t, err, sizeof err) != BUILD_OK)
goto test_out;
@@ -936,12 +1001,19 @@ shallow_out:
goto test_out;
release_file(c, &fd);
}
+ for (i = 0u; i < t.n_fetches; ++i) {
+ if (payload_install_blob(c, t.fetches[i].blob, root, seen, NULL, NULL,
+ &fd) != BUILD_OK)
+ goto test_out;
+ release_file(c, &fd);
+ }
ok = BUILD_OK;
test_out:
release_file(c, &fd);
heap_free_array(c->ctx, t.config_keys, rows, sizeof *t.config_keys);
heap_free_array(c->ctx, t.sources, rows, sizeof *t.sources);
heap_free_array(c->ctx, t.globs, rows, sizeof *t.globs);
+ heap_free_array(c->ctx, t.fetches, rows, sizeof *t.fetches);
heap_free_array(c->ctx, t.deps, rows, sizeof *t.deps);
return ok;
}
diff --git a/src/build/coord.c b/src/build/coord.c
@@ -44,6 +44,7 @@ typedef struct BuildDeepSetMemo {
BuildLeafSet leaf;
BuildSourceLeaf* sources;
BuildGlobLeaf* globs;
+ BuildFetchLeaf* fetches;
const BuildLeafSet** children;
int valid_known;
int valid;
@@ -255,6 +256,9 @@ static void free_deepsets(KitBuildCoordinator* c) {
if (n->globs)
c->ctx->heap->free(c->ctx->heap, n->globs,
n->leaf.n_globs * sizeof *n->globs);
+ if (n->fetches)
+ c->ctx->heap->free(c->ctx->heap, n->fetches,
+ n->leaf.n_fetches * sizeof *n->fetches);
if (n->children)
c->ctx->heap->free(c->ctx->heap, n->children,
n->leaf.n_children * sizeof *n->children);
@@ -737,6 +741,13 @@ int build_coord_leafset_intern(KitBuildCoordinator* c, const BuildLeafSet* in,
if (!n->globs) goto oom;
memcpy(n->globs, in->globs, in->n_globs * sizeof *n->globs);
}
+ if (in->n_fetches) {
+ n->fetches = (BuildFetchLeaf*)c->ctx->heap->alloc(
+ c->ctx->heap, in->n_fetches * sizeof *n->fetches,
+ _Alignof(BuildFetchLeaf));
+ if (!n->fetches) goto oom;
+ memcpy(n->fetches, in->fetches, in->n_fetches * sizeof *n->fetches);
+ }
if (in->n_children) {
n->children = (const BuildLeafSet**)c->ctx->heap->alloc(
c->ctx->heap, in->n_children * sizeof *n->children,
@@ -746,6 +757,7 @@ int build_coord_leafset_intern(KitBuildCoordinator* c, const BuildLeafSet* in,
}
n->leaf.sources = n->sources;
n->leaf.globs = n->globs;
+ n->leaf.fetches = n->fetches;
n->leaf.children = n->children;
n->next = c->deepsets;
c->deepsets = n;
@@ -759,6 +771,9 @@ oom:
if (n->globs)
c->ctx->heap->free(c->ctx->heap, n->globs,
in->n_globs * sizeof *n->globs);
+ if (n->fetches)
+ c->ctx->heap->free(c->ctx->heap, n->fetches,
+ in->n_fetches * sizeof *n->fetches);
if (n->children)
c->ctx->heap->free(c->ctx->heap, n->children,
in->n_children * sizeof *n->children);
@@ -767,11 +782,12 @@ oom:
}
static void count_deepset_rows(const uint8_t* data, size_t len, size_t* ns,
- size_t* ng, size_t* nc) {
+ size_t* ng, size_t* nf, size_t* nc) {
size_t pos = 0;
int sec = 0;
*ns = 0;
*ng = 0;
+ *nf = 0;
*nc = 0;
while (pos < len) {
size_t start = pos;
@@ -788,10 +804,14 @@ static void count_deepset_rows(const uint8_t* data, size_t len, size_t* ns,
} else if (n == sizeof("[child]") - 1u &&
memcmp(data + start, "[child]", n) == 0) {
sec = 3;
+ } else if (n == sizeof("[fetch]") - 1u &&
+ memcmp(data + start, "[fetch]", n) == 0) {
+ sec = 4;
} else if (data[start] != '[' && data[start] != 'k') {
if (sec == 1) ++*ns;
if (sec == 2) ++*ng;
if (sec == 3) ++*nc;
+ if (sec == 4) ++*nf;
}
}
++pos;
@@ -807,9 +827,10 @@ int build_coord_deepset_load(KitBuildCoordinator* c,
BuildLeafSet leaf;
BuildSourceLeaf* sources = NULL;
BuildGlobLeaf* globs = NULL;
+ BuildFetchLeaf* fetches = NULL;
uint8_t(*child_ids)[BUILD_HASH_LEN] = NULL;
const BuildLeafSet** children = NULL;
- size_t ns, ng, nc, i;
+ size_t ns, ng, nf, nc, i;
int ok = BUILD_ERR;
if (!c || !deepset_id || !out) return BUILD_ERR;
for (memo = c->deepsets; memo; memo = memo->next) {
@@ -822,7 +843,7 @@ int build_coord_deepset_load(KitBuildCoordinator* c,
fd.size = 0;
fd.token = NULL;
if (kit_cas_get_blob(c->cas, deepset_id, &fd) != KIT_OK) return BUILD_ERR;
- count_deepset_rows(fd.data, fd.size, &ns, &ng, &nc);
+ count_deepset_rows(fd.data, fd.size, &ns, &ng, &nf, &nc);
if (ns) {
sources = (BuildSourceLeaf*)c->ctx->heap->alloc(
c->ctx->heap, ns * sizeof *sources, _Alignof(BuildSourceLeaf));
@@ -833,6 +854,11 @@ int build_coord_deepset_load(KitBuildCoordinator* c,
c->ctx->heap, ng * sizeof *globs, _Alignof(BuildGlobLeaf));
if (!globs) goto out_release;
}
+ if (nf) {
+ fetches = (BuildFetchLeaf*)c->ctx->heap->alloc(
+ c->ctx->heap, nf * sizeof *fetches, _Alignof(BuildFetchLeaf));
+ if (!fetches) goto out_release;
+ }
if (nc) {
child_ids = (uint8_t(*)[BUILD_HASH_LEN])c->ctx->heap->alloc(
c->ctx->heap, nc * sizeof *child_ids, _Alignof(uint8_t));
@@ -845,6 +871,8 @@ int build_coord_deepset_load(KitBuildCoordinator* c,
ds.cap_sources = ns;
ds.globs = globs;
ds.cap_globs = ng;
+ ds.fetches = fetches;
+ ds.cap_fetches = nf;
ds.children = child_ids;
ds.cap_children = nc;
if (build_deepset_parse(fd.data, fd.size, &ds, NULL, 0) != BUILD_OK)
@@ -861,6 +889,8 @@ int build_coord_deepset_load(KitBuildCoordinator* c,
leaf.n_sources = ds.n_sources;
leaf.globs = globs;
leaf.n_globs = ds.n_globs;
+ leaf.fetches = fetches;
+ leaf.n_fetches = ds.n_fetches;
leaf.children = children;
leaf.n_children = ds.n_children;
if (build_coord_leafset_intern(c, &leaf, out) != BUILD_OK) goto out_release;
@@ -871,6 +901,8 @@ out_release:
if (sources)
c->ctx->heap->free(c->ctx->heap, sources, ns * sizeof *sources);
if (globs) c->ctx->heap->free(c->ctx->heap, globs, ng * sizeof *globs);
+ if (fetches)
+ c->ctx->heap->free(c->ctx->heap, fetches, nf * sizeof *fetches);
if (child_ids)
c->ctx->heap->free(c->ctx->heap, child_ids, nc * sizeof *child_ids);
if (children)
diff --git a/src/build/coord.h b/src/build/coord.h
@@ -31,7 +31,7 @@
/* The in-memory deepset node: one node of the transitive input-closure DAG that
* a resolution returns and the targets memo caches (trace.h BuildDeepSet is its
- * on-disk/CAS-blob form). It carries this node's DIRECT source/glob leaves, its
+ * on-disk/CAS-blob form). It carries this node's DIRECT source/glob/fetch leaves, its
* target + recipe-id, and pointers to its children's (interned) nodes. `id` is
* the deep-set-id — the CAS blob id of the emitted node — so an unchanged
* subtree is recognized by id equality without descending. Folded into a parent
@@ -46,6 +46,8 @@ typedef struct BuildLeafSet {
size_t n_sources;
BuildGlobLeaf* globs;
size_t n_globs;
+ BuildFetchLeaf* fetches;
+ size_t n_fetches;
const struct BuildLeafSet** children; /* direct deps' interned nodes */
size_t n_children;
} BuildLeafSet;
diff --git a/src/build/resolve.c b/src/build/resolve.c
@@ -86,6 +86,9 @@ static int shallow_direct_match(KitBuildCoordinator* c,
return 0;
if (!build_id_eq(hash, st->globs[i].result_hash)) return 0;
}
+ for (i = 0; i < st->n_fetches; ++i) {
+ if (kit_cas_has_blob(c->cas, st->fetches[i].blob) != KIT_OK) return 0;
+ }
return 1;
}
@@ -139,6 +142,8 @@ static int try_shallow_trace(KitBuildCoordinator* c, KitSlice target,
log.n_sources = st->n_sources;
log.globs = st->globs;
log.n_globs = st->n_globs;
+ log.fetches = st->fetches;
+ log.n_fetches = st->n_fetches;
log.deps = deps_copy;
log.n_deps = st->n_deps;
log.child_leafsets = child_leafsets;
@@ -200,6 +205,8 @@ static int try_test_shallow_trace(KitBuildCoordinator* c, KitSlice target,
log.n_sources = st->n_sources;
log.globs = st->globs;
log.n_globs = st->n_globs;
+ log.fetches = st->fetches;
+ log.n_fetches = st->n_fetches;
log.deps = deps_copy;
log.n_deps = st->n_deps;
log.child_leafsets = child_leafsets;
@@ -432,6 +439,7 @@ scan_record:
BuildConfigKey keys[128];
BuildSourceLeaf sources[128];
BuildGlobLeaf globs[128];
+ BuildFetchLeaf fetches[128];
BuildDepEdge deps[128];
BuildShallowTrace st;
size_t rows;
@@ -450,6 +458,8 @@ scan_record:
st.cap_sources = sizeof sources / sizeof sources[0];
st.globs = globs;
st.cap_globs = sizeof globs / sizeof globs[0];
+ st.fetches = fetches;
+ st.cap_fetches = sizeof fetches / sizeof fetches[0];
st.deps = deps;
st.cap_deps = sizeof deps / sizeof deps[0];
if (rows <= sizeof keys / sizeof keys[0] &&
@@ -552,6 +562,7 @@ scan_record:
BuildConfigKey keys[128];
BuildSourceLeaf sources[128];
BuildGlobLeaf globs[128];
+ BuildFetchLeaf fetches[128];
BuildDepEdge deps[128];
BuildShallowTrace st;
size_t nrows;
@@ -570,6 +581,8 @@ scan_record:
st.cap_sources = sizeof sources / sizeof sources[0];
st.globs = globs;
st.cap_globs = sizeof globs / sizeof globs[0];
+ st.fetches = fetches;
+ st.cap_fetches = sizeof fetches / sizeof fetches[0];
st.deps = deps;
st.cap_deps = sizeof deps / sizeof deps[0];
if (nrows <= sizeof keys / sizeof keys[0] &&
@@ -661,6 +674,8 @@ int build_leafset_union(KitBuildCoordinator* c, const BuildLeafSet* direct,
ds.n_sources = direct->n_sources;
ds.globs = direct->globs;
ds.n_globs = direct->n_globs;
+ ds.fetches = direct->fetches;
+ ds.n_fetches = direct->n_fetches;
ds.children = child_ids;
ds.n_children = nchildren;
if (kit_writer_mem(c->ctx->heap, &w) != KIT_OK || !w) goto out;
@@ -680,6 +695,8 @@ int build_leafset_union(KitBuildCoordinator* c, const BuildLeafSet* direct,
leaf.n_sources = direct->n_sources;
leaf.globs = direct->globs;
leaf.n_globs = direct->n_globs;
+ leaf.fetches = direct->fetches;
+ leaf.n_fetches = direct->n_fetches;
leaf.children = child_ptrs;
leaf.n_children = nchildren;
if (build_coord_leafset_intern(c, &leaf, out) != BUILD_OK) goto out;
@@ -733,6 +750,10 @@ int build_leafset_refresh(KitBuildCoordinator* c, const BuildLeafSet* leafset,
return BUILD_ERR;
if (!build_id_eq(hash, leafset->globs[i].result_hash)) goto done;
}
+ for (i = 0; i < leafset->n_fetches; ++i) {
+ if (kit_cas_has_blob(c->cas, leafset->fetches[i].blob) != KIT_OK)
+ goto done;
+ }
for (i = 0; i < leafset->n_children; ++i) {
int child_match = 0;
if (build_leafset_refresh(c, leafset->children[i], &child_match) !=
diff --git a/src/build/resolve.h b/src/build/resolve.h
@@ -15,7 +15,7 @@
* §Resolution algorithm):
*
* Phase 1 deep fast path - same config-id AND argv-id, and an all-clear
- * refresh of the deepset closure (every source/glob
+ * refresh of the deepset closure (every source/glob/fetch
* leaf unchanged AND every node's recipe-id still
* recomputes the same) => reuse, no graph walk.
* Phase 2 shallow path - config, argv, and/or sources moved: guard on the
@@ -133,7 +133,7 @@ int build_materialize(KitBuildCoordinator*,
size_t cap);
/* Build a parent's deepset node from its DIRECT leaves and its children's nodes.
- * `direct` carries this node's target, recipe-id, and direct source/glob leaves
+ * `direct` carries this node's target, recipe-id, and direct source/glob/fetch leaves
* (its children fields empty); `children` are the deps' resolved nodes. Computes
* the deep-set-id, stores the canonical node as a CAS blob (kit_cas_add_blob —
* the children's blobs were stored when they were built, so this is incremental
@@ -144,7 +144,8 @@ int build_leafset_union(KitBuildCoordinator*, const BuildLeafSet* direct,
const BuildLeafSet** out);
/* Refresh a deepset node against the live workspace: per node, rehash its direct
- * source leaves (absent stays absent), reglob its glob leaves, recompute its
+ * source leaves (absent stays absent), reglob its glob leaves, check its fetched
+ * blobs are still present in the CAS, recompute its
* recipe-id through the LIVE definition (catching a dep repointed to a
* different-content recipe, treating same-content as unchanged), and recurse
* into children — all through the per-process memos and the deep-set-id validity
diff --git a/src/build/runner.c b/src/build/runner.c
@@ -96,6 +96,21 @@ static int append_glob(BuildDepLog* log, KitSlice pattern,
return BUILD_OK;
}
+static int append_fetch(BuildDepLog* log,
+ const uint8_t blob[BUILD_HASH_LEN]) {
+ BuildFetchLeaf* row;
+ size_t i;
+ if (!log || !blob) return BUILD_ERR;
+ for (i = 0; i < log->n_fetches; ++i) {
+ row = &log->fetches[i];
+ if (build_id_eq(row->blob, blob)) return BUILD_OK;
+ }
+ if (log->n_fetches >= log->cap_fetches) return BUILD_ERR;
+ row = &log->fetches[log->n_fetches++];
+ memcpy(row->blob, blob, BUILD_HASH_LEN);
+ return BUILD_OK;
+}
+
static int append_dep(BuildDepLog* log, KitSlice target,
const uint8_t overlay_id[BUILD_HASH_LEN],
const uint8_t config_id[BUILD_HASH_LEN],
@@ -335,6 +350,7 @@ static int build_run_recipe_impl(KitBuildCoordinator* c, KitSlice target,
BuildConfigKey config_keys[128];
BuildSourceLeaf sources[128];
BuildGlobLeaf globs[128];
+ BuildFetchLeaf fetches[128];
BuildDepEdge deps[128];
const BuildLeafSet* child_leafsets[128];
BuildPendingNeed pending[128];
@@ -369,6 +385,8 @@ static int build_run_recipe_impl(KitBuildCoordinator* c, KitSlice target,
log.cap_sources = sizeof sources / sizeof sources[0];
log.globs = globs;
log.cap_globs = sizeof globs / sizeof globs[0];
+ log.fetches = fetches;
+ log.cap_fetches = sizeof fetches / sizeof fetches[0];
log.deps = deps;
log.cap_deps = sizeof deps / sizeof deps[0];
log.child_leafsets = child_leafsets;
@@ -462,6 +480,7 @@ static int build_run_test_recipe_impl(KitBuildCoordinator* c, KitSlice target,
BuildConfigKey config_keys[128];
BuildSourceLeaf sources[128];
BuildGlobLeaf globs[128];
+ BuildFetchLeaf fetches[128];
BuildDepEdge deps[128];
const BuildLeafSet* child_leafsets[128];
BuildPendingNeed pending[128];
@@ -504,6 +523,8 @@ static int build_run_test_recipe_impl(KitBuildCoordinator* c, KitSlice target,
log.cap_sources = sizeof sources / sizeof sources[0];
log.globs = globs;
log.cap_globs = sizeof globs / sizeof globs[0];
+ log.fetches = fetches;
+ log.cap_fetches = sizeof fetches / sizeof fetches[0];
log.deps = deps;
log.cap_deps = sizeof deps / sizeof deps[0];
log.child_leafsets = child_leafsets;
@@ -661,7 +682,8 @@ int build_runner_service(KitBuildCoordinator* c, KitBuildConn* conn,
char path[BUILD_PATH_MAX];
if (kit_hex_decode(blob, req.arg.s, BUILD_HASH_LEN) != KIT_OK ||
build_coord_fetch_blob(c, blob, req.argv, req.argc, path,
- sizeof path) != BUILD_OK) {
+ sizeof path) != BUILD_OK ||
+ append_fetch(log, blob) != BUILD_OK) {
resp_error(&resp, KIT_ERR, "fetch failed");
} else {
memcpy(resp.id, blob, BUILD_HASH_LEN);
@@ -810,6 +832,8 @@ int build_runner_record_traces(KitBuildCoordinator* c, KitSlice target,
direct.n_sources = log->n_sources;
direct.globs = log->globs;
direct.n_globs = log->n_globs;
+ direct.fetches = log->fetches;
+ direct.n_fetches = log->n_fetches;
direct.children = log->child_leafsets;
direct.n_children = log->n_children;
if (build_leafset_union(c, &direct, log->child_leafsets, log->n_children,
@@ -829,6 +853,8 @@ int build_runner_record_traces(KitBuildCoordinator* c, KitSlice target,
shallow.n_sources = log->n_sources;
shallow.globs = log->globs;
shallow.n_globs = log->n_globs;
+ shallow.fetches = log->fetches;
+ shallow.n_fetches = log->n_fetches;
shallow.deps = log->deps;
shallow.n_deps = log->n_deps;
@@ -896,6 +922,8 @@ int build_runner_record_test_traces(KitBuildCoordinator* c, KitSlice target,
direct.n_sources = log->n_sources;
direct.globs = log->globs;
direct.n_globs = log->n_globs;
+ direct.fetches = log->fetches;
+ direct.n_fetches = log->n_fetches;
direct.children = log->child_leafsets;
direct.n_children = log->n_children;
if (build_leafset_union(c, &direct, log->child_leafsets, log->n_children,
@@ -915,6 +943,8 @@ int build_runner_record_test_traces(KitBuildCoordinator* c, KitSlice target,
shallow.n_sources = log->n_sources;
shallow.globs = log->globs;
shallow.n_globs = log->n_globs;
+ shallow.fetches = log->fetches;
+ shallow.n_fetches = log->n_fetches;
shallow.deps = log->deps;
shallow.n_deps = log->n_deps;
diff --git a/src/build/runner.h b/src/build/runner.h
@@ -52,6 +52,8 @@ typedef struct BuildDepLog {
size_t n_sources, cap_sources;
BuildGlobLeaf* globs; /* direct globs */
size_t n_globs, cap_globs;
+ BuildFetchLeaf* fetches; /* direct hash-pinned fetches */
+ size_t n_fetches, cap_fetches;
BuildDepEdge* deps; /* AWAITED `need` edges (recorded) */
size_t n_deps, cap_deps;
const BuildLeafSet** child_leafsets; /* one deepset node per awaited dep */
@@ -81,7 +83,7 @@ typedef struct BuildDepLog {
* dispatch them, appending each dependency to `log`. Drives the protocol.h
* codec over the transport. `chain` is this recipe's chain frame (its dispatched
* needs extend it). Commands:
- * config-get/source/glob — synchronous; log the dep, reply.
+ * config-get/source/glob/fetch — synchronous; log the dep, reply.
* need (fused) — build_resolve inline on this thread; log the edge.
* need-submit — build_coord_dispatch(dep) (concurrent, no block),
* stash a BuildPendingNeed, reply a token. Cycles and
diff --git a/src/build/trace.c b/src/build/trace.c
@@ -109,6 +109,10 @@ static int glob_cmp(const BuildGlobLeaf* a, const BuildGlobLeaf* b) {
return strcmp(a->pattern, b->pattern);
}
+static int fetch_cmp(const BuildFetchLeaf* a, const BuildFetchLeaf* b) {
+ return id_cmp(a->blob, b->blob);
+}
+
static int dep_cmp(const BuildDepEdge* a, const BuildDepEdge* b) {
int c = strcmp(a->name, b->name);
if (c != 0) return c;
@@ -225,6 +229,35 @@ static int emit_sorted_globs(const BuildGlobLeaf* rows, size_t n, KitWriter* out
return BUILD_OK;
}
+static int emit_fetch_row(KitWriter* out, const BuildFetchLeaf* row) {
+ char hex[BUILD_HEX_LEN];
+ hex_encode(hex, row->blob);
+ if (emit_bytes(out, hex) != BUILD_OK) return BUILD_ERR;
+ return emit_bytes(out, "\n");
+}
+
+static int emit_sorted_fetches(const BuildFetchLeaf* rows, size_t n,
+ KitWriter* out, char* err, size_t errcap) {
+ const BuildFetchLeaf* prev = NULL;
+ size_t emitted = 0;
+ if (n && !rows) return set_err(err, errcap, "missing fetch rows");
+ if (emit_section(out, "fetch") != BUILD_OK) return BUILD_ERR;
+ while (emitted < n) {
+ const BuildFetchLeaf* best = NULL;
+ size_t i;
+ for (i = 0; i < n; ++i) {
+ const BuildFetchLeaf* cur = &rows[i];
+ int after_prev = !prev || fetch_cmp(prev, cur) < 0;
+ if (after_prev && (!best || fetch_cmp(cur, best) < 0)) best = cur;
+ }
+ if (!best) return set_err(err, errcap, "duplicate fetch row");
+ if (emit_fetch_row(out, best) != BUILD_OK) return BUILD_ERR;
+ prev = best;
+ ++emitted;
+ }
+ return BUILD_OK;
+}
+
static int emit_dep_row(KitWriter* out, const BuildDepEdge* row) {
char ovr[BUILD_HEX_LEN], cfg[BUILD_HEX_LEN], argv[BUILD_HEX_LEN],
tree[BUILD_HEX_LEN];
@@ -367,6 +400,9 @@ static int shallow_emit_like(const BuildShallowTrace* t, KitWriter* out,
return BUILD_ERR;
if (emit_sorted_globs(t->globs, t->n_globs, out, err, errcap) != BUILD_OK)
return BUILD_ERR;
+ if (emit_sorted_fetches(t->fetches, t->n_fetches, out, err, errcap) !=
+ BUILD_OK)
+ return BUILD_ERR;
if (emit_sorted_deps(t->deps, t->n_deps, out, err, errcap) != BUILD_OK)
return BUILD_ERR;
return kit_writer_status(out) == KIT_OK ? BUILD_OK : BUILD_ERR;
@@ -422,6 +458,9 @@ int build_deepset_emit(const BuildDeepSet* t, KitWriter* out, char* err,
return BUILD_ERR;
if (emit_sorted_globs(t->globs, t->n_globs, out, err, errcap) != BUILD_OK)
return BUILD_ERR;
+ if (emit_sorted_fetches(t->fetches, t->n_fetches, out, err, errcap) !=
+ BUILD_OK)
+ return BUILD_ERR;
if (emit_sorted_children(t->children, t->n_children, out, err, errcap) !=
BUILD_OK)
return BUILD_ERR;
@@ -444,6 +483,7 @@ typedef enum TraceSection {
TRACE_SEC_CONFIG,
TRACE_SEC_SOURCE,
TRACE_SEC_GLOB,
+ TRACE_SEC_FETCH,
TRACE_SEC_DEP,
TRACE_SEC_CHILD,
TRACE_SEC_RECORD_ROWS
@@ -630,6 +670,42 @@ static int append_deepset_glob(BuildDeepSet* out, char** fields, size_t n,
return BUILD_OK;
}
+static int parse_fetch_leaf(char** fields, size_t n, BuildFetchLeaf* row,
+ char* err, size_t errcap) {
+ if (expect_field_count(n, 1u, err, errcap) != BUILD_OK) return BUILD_ERR;
+ if (hex_decode_strict(fields[0], row->blob) != BUILD_OK)
+ return set_err(err, errcap, "bad fetch hash");
+ return BUILD_OK;
+}
+
+static int append_shallow_fetch(BuildShallowTrace* out, char** fields, size_t n,
+ char* err, size_t errcap) {
+ BuildFetchLeaf* row;
+ if (out->n_fetches >= out->cap_fetches)
+ return set_err(err, errcap, "too many fetch rows");
+ row = &out->fetches[out->n_fetches];
+ if (parse_fetch_leaf(fields, n, row, err, errcap) != BUILD_OK)
+ return BUILD_ERR;
+ if (out->n_fetches && fetch_cmp(&out->fetches[out->n_fetches - 1u], row) >= 0)
+ return set_err(err, errcap, "non-canonical fetch rows");
+ ++out->n_fetches;
+ return BUILD_OK;
+}
+
+static int append_deepset_fetch(BuildDeepSet* out, char** fields, size_t n,
+ char* err, size_t errcap) {
+ BuildFetchLeaf* row;
+ if (out->n_fetches >= out->cap_fetches)
+ return set_err(err, errcap, "too many fetch rows");
+ row = &out->fetches[out->n_fetches];
+ if (parse_fetch_leaf(fields, n, row, err, errcap) != BUILD_OK)
+ return BUILD_ERR;
+ if (out->n_fetches && fetch_cmp(&out->fetches[out->n_fetches - 1u], row) >= 0)
+ return set_err(err, errcap, "non-canonical fetch rows");
+ ++out->n_fetches;
+ return BUILD_OK;
+}
+
static int append_dep(BuildShallowTrace* out, char** fields, size_t n, char* err,
size_t errcap) {
BuildDepEdge* row;
@@ -708,12 +784,14 @@ static int shallow_parse_like(const uint8_t* data, size_t len,
if (!data || !out) return set_err(err, errcap, "missing shallow trace");
if ((out->cap_config_keys && !out->config_keys) ||
(out->cap_sources && !out->sources) || (out->cap_globs && !out->globs) ||
+ (out->cap_fetches && !out->fetches) ||
(out->cap_deps && !out->deps))
return set_err(err, errcap, "missing trace storage");
memset(out->target, 0, sizeof out->target);
out->n_config_keys = 0;
out->n_sources = 0;
out->n_globs = 0;
+ out->n_fetches = 0;
out->n_deps = 0;
memset(&p, 0, sizeof p);
p.data = data;
@@ -741,7 +819,10 @@ static int shallow_parse_like(const uint8_t* data, size_t len,
sec = TRACE_SEC_SOURCE;
} else if (strcmp(line, "[glob]") == 0 && sec == TRACE_SEC_SOURCE) {
sec = TRACE_SEC_GLOB;
- } else if (strcmp(line, "[dep]") == 0 && sec == TRACE_SEC_GLOB) {
+ } else if (strcmp(line, "[fetch]") == 0 && sec == TRACE_SEC_GLOB) {
+ sec = TRACE_SEC_FETCH;
+ } else if (strcmp(line, "[dep]") == 0 &&
+ (sec == TRACE_SEC_GLOB || sec == TRACE_SEC_FETCH)) {
sec = TRACE_SEC_DEP;
} else {
return set_err(err, errcap, "bad shallow section");
@@ -785,6 +866,9 @@ static int shallow_parse_like(const uint8_t* data, size_t len,
} else if (sec == TRACE_SEC_GLOB) {
if (append_shallow_glob(out, fields, n, err, errcap) != BUILD_OK)
return BUILD_ERR;
+ } else if (sec == TRACE_SEC_FETCH) {
+ if (append_shallow_fetch(out, fields, n, err, errcap) != BUILD_OK)
+ return BUILD_ERR;
} else if (sec == TRACE_SEC_DEP) {
if (append_dep(out, fields, n, err, errcap) != BUILD_OK) return BUILD_ERR;
} else {
@@ -890,11 +974,13 @@ int build_deepset_parse(const uint8_t* data, size_t len, BuildDeepSet* out,
uint32_t seen = 0;
if (!data || !out) return set_err(err, errcap, "missing deepset");
if ((out->cap_sources && !out->sources) || (out->cap_globs && !out->globs) ||
+ (out->cap_fetches && !out->fetches) ||
(out->cap_children && !out->children))
return set_err(err, errcap, "missing deepset storage");
memset(out->target, 0, sizeof out->target);
out->n_sources = 0;
out->n_globs = 0;
+ out->n_fetches = 0;
out->n_children = 0;
memset(&p, 0, sizeof p);
p.data = data;
@@ -920,7 +1006,10 @@ int build_deepset_parse(const uint8_t* data, size_t len, BuildDeepSet* out,
sec = TRACE_SEC_SOURCE;
} else if (strcmp(line, "[glob]") == 0 && sec == TRACE_SEC_SOURCE) {
sec = TRACE_SEC_GLOB;
- } else if (strcmp(line, "[child]") == 0 && sec == TRACE_SEC_GLOB) {
+ } else if (strcmp(line, "[fetch]") == 0 && sec == TRACE_SEC_GLOB) {
+ sec = TRACE_SEC_FETCH;
+ } else if (strcmp(line, "[child]") == 0 &&
+ (sec == TRACE_SEC_GLOB || sec == TRACE_SEC_FETCH)) {
sec = TRACE_SEC_CHILD;
} else {
return set_err(err, errcap, "bad deepset section");
@@ -948,6 +1037,9 @@ int build_deepset_parse(const uint8_t* data, size_t len, BuildDeepSet* out,
} else if (sec == TRACE_SEC_GLOB) {
if (append_deepset_glob(out, fields, n, err, errcap) != BUILD_OK)
return BUILD_ERR;
+ } else if (sec == TRACE_SEC_FETCH) {
+ if (append_deepset_fetch(out, fields, n, err, errcap) != BUILD_OK)
+ return BUILD_ERR;
} else if (sec == TRACE_SEC_CHILD) {
if (append_child(out, fields, n, err, errcap) != BUILD_OK)
return BUILD_ERR;
diff --git a/src/build/trace.h b/src/build/trace.h
@@ -42,6 +42,13 @@ typedef struct BuildGlobLeaf {
uint8_t result_hash[BUILD_HASH_LEN];
} BuildGlobLeaf;
+/* One fetched blob leaf: a hash-pinned external object requested through
+ * `fetch`. URLs are transport hints only and are not cache identity; the blob id
+ * is the dependency. Sorted by blob id. */
+typedef struct BuildFetchLeaf {
+ uint8_t blob[BUILD_HASH_LEN];
+} BuildFetchLeaf;
+
/* One target-dep edge (shallow only): replays one `need`. Carries the explicit
* need overlay-id, the dep's effective config-id and argv-id, and the output
* tree-id it produced. Shallow replay recomputes the dep config as the current
@@ -82,6 +89,9 @@ typedef struct BuildShallowTrace {
BuildGlobLeaf* globs; /* [glob] section */
size_t n_globs;
size_t cap_globs;
+ BuildFetchLeaf* fetches; /* [fetch] section */
+ size_t n_fetches;
+ size_t cap_fetches;
BuildDepEdge* deps; /* [dep] section */
size_t n_deps;
size_t cap_deps;
@@ -91,8 +101,8 @@ typedef struct BuildShallowTrace {
* pointer to the TRANSITIVE input closure, held structurally as a deepset DAG
* (below) rather than inlined. If the request's config-id equals `root_config`,
* its argv-id equals `argv`, and refreshing the `deepset` closure finds nothing
- * moved (every source/glob leaf still matches AND every node's recipe-id still
- * recomputes the same), determinism makes the output identical — no recipe runs.
+ * moved (every source/glob/fetch leaf still matches AND every node's recipe-id
+ * still recomputes the same), determinism makes the output identical — no recipe runs.
* The deepset blob must be present to refresh; if absent (GC'd), the trace is
* treated as absent and resolution falls through (fail-safe). */
typedef struct BuildDeepTrace {
@@ -106,7 +116,7 @@ typedef struct BuildDeepTrace {
/* One node of the transitive input-closure DAG — the structural form of the old
* flat leafset ("compact deep traces"). One node per target build: its DIRECT
- * source/glob leaves, its recipe-id, and the deep-set-ids of its DIRECT deps.
+ * source/glob/fetch leaves, its recipe-id, and the deep-set-ids of its DIRECT deps.
* The fully-flattened closure is the recursive union, NEVER materialized: a
* subtree reached through many parents is one node, stored once — a Merkle DAG
* that structurally shares like the CAS. Building a parent is "concat my direct
@@ -126,6 +136,9 @@ typedef struct BuildDeepSet {
BuildGlobLeaf* globs; /* this node's DIRECT glob leaves */
size_t n_globs;
size_t cap_globs;
+ BuildFetchLeaf* fetches; /* this node's DIRECT fetched blobs */
+ size_t n_fetches;
+ size_t cap_fetches;
uint8_t (*children)[BUILD_HASH_LEN]; /* direct deps' deep-set-ids, sorted */
size_t n_children;
size_t cap_children;
diff --git a/test/buildcoord/run.sh b/test/buildcoord/run.sh
@@ -1083,6 +1083,13 @@ contains "buildcoord-fetch-mirror-run" "$work/buildcoord-fetch-mirror.err" \
fetch_path=$(tree_path_from "$work/buildcoord-fetch-mirror.out")
contains "buildcoord-fetch-mirror-output" "$fetch_path/fetched.txt" \
"remote:v1"
+find "$store/build/trace" -type f -exec grep -l "$fetch_id" {} + \
+ > "$work/fetch-traces.txt" 2> "$work/fetch-traces.err"
+if [ -s "$work/fetch-traces.txt" ]; then
+ ok "buildcoord-fetch-logged-in-trace"
+else
+ not_ok "buildcoord-fetch-logged-in-trace" "$work/fetch-traces.err"
+fi
build_assert_ok buildcoord-fetch-cas-hit-bad-url --stats \
--config env.PATH=/no/such //fetch:blob -- \
"$fetch_id" "file://$work/still-missing-fetch.txt"
