commit 5ae5d511771ef843bee75c53b3d6bbf27c94d493
parent 2a21dd256562d42cba804a07bc2c7a53e2fb891b
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Wed, 3 Jun 2026 10:00:40 -0700
build spike
Diffstat:
| M | .gitignore | | | 2 | +- |
| M | doc/plan/BUILD.md | | | 506 | +++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------- |
| A | doc/plan/BUILD_INTERNALS.md | | | 980 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | doc/plan/BUILD_TESTING.md | | | 192 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | include/kit/build_coord.h | | | 428 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| M | include/kit/cas.h | | | 18 | ++++++++++++++++++ |
| A | src/build/build.h | | | 124 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/bundle.h | | | 73 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/cfg.h | | | 94 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/coord.h | | | 229 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/defn.h | | | 62 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/protocol.h | | | 100 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/remote.h | | | 56 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/resolve.h | | | 141 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/runner.h | | | 115 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/store.h | | | 129 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/build/trace.h | | | 185 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
17 files changed, 3256 insertions(+), 178 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -1 +1 @@
-build/
+/build/
diff --git a/doc/plan/BUILD.md b/doc/plan/BUILD.md
@@ -13,6 +13,13 @@
> Bazel/Nix-style incremental build engine that compiles arbitrary projects by
> running recipes and caching their outputs by content. The two are unrelated
> beyond sharing the word "build."
+>
+> **Implementation breakdown:** [BUILD_INTERNALS.md](BUILD_INTERNALS.md) carves
+> this design into the `src/build/` modules — their responsibilities, dependency
+> graph, and pseudocode sketches — for parallel implementation.
+> **Testing strategy:** [BUILD_TESTING.md](BUILD_TESTING.md) specifies the
+> harness, per-module plan, integration scenarios, and cross-cutting property
+> tests (byte-stability, fail-safe parsing, minimal-rebuild, crash-safety).
## What it is
@@ -43,14 +50,15 @@ index, and the resolution algorithm that drives it.
| Term | Meaning |
|------|---------|
-| **Target** `T` | A named unit of work, e.g. `//app:server`. A build definition maps each target name to one recipe + local argv. |
+| **Target** `T` | A named unit of work, e.g. `//app:server`. A build definition maps each target name to one recipe; the target's local argv comes from the build request (empty if none). |
| **Recipe** | The executable the coordinator runs to produce a target's output. Identified by `recipe-id` = `BLAKE2b(recipe file bytes)`. |
-| **Configuration** | A key→value map of tunables (target triple, opt level, feature flags). Two scopes — *propagated* and *local* — see [below](#configuration-model). Identified by `config-id` = `BLAKE2b(canonical map)`. |
+| **Configuration** | Tunables a recipe reads (target triple, opt level, feature flags). Two scopes — *propagated* (a key→value map, `config-id`) and *local* (the argv vector, `argv-id`) — see [below](#configuration-model). Both content-addressed: `config-id`/`argv-id` = `BLAKE2b(canonical bytes)`. |
| **Output tree** | The directory a recipe produces, captured as a CAS tree (`tree-id`). The unit a build request returns. |
| **Base inputs** | The *leaves* of the dependency graph, dynamically requested by recipes: **config values**, **source files**, and **globs**. |
| **Target dep** | A dependency of one target on the *output tree* of another. The graph's interior edges, created by `need`. |
| **Shallow trace** | A record of one build: its direct base inputs + its direct target deps (by output `tree-id`) → its output `tree-id`. |
-| **Deep trace** | A record of one build: the *transitive* closure of source/glob leaves + the root `config-id` → its output `tree-id`. |
+| **Deep trace** | A record of one build: `(root config-id, argv)` + a pointer to the transitive input closure (a **deepset**) → its output `tree-id`. |
+| **Deepset** | One node of the transitive input-closure DAG (`deep-set-id` = `BLAKE2b(canonical body)`): a target's recipe-id + its direct source/glob leaves + its children's `deep-set-id`s. A self-verifying CAS blob; shared structurally across ancestors. |
| **Tree cache** | On-disk *materialized* output directories, keyed by `tree-id`, ready to hand back as a filesystem path. |
The three base-input kinds are exactly the dynamically-requested leaves: a recipe
@@ -70,12 +78,15 @@ Configuration has two scopes, distinguished by visibility:
text and content-addressed: `config-id = BLAKE2b(canonical map)`, stored as a
CAS blob so any recorded `config-id` resolves back to the actual map.
-- **Local configuration** is the target's **argv**, fixed by the build
- definition. It is visible *only* to that one target's recipe (delivered as the
+- **Local configuration** is the target's **argv**, supplied entirely by the
+ build request (both the top-level request and a `need` carry an optional argv);
+ when none is given it is the **empty** argv. The build definition carries no
+ argv. It is visible *only* to that one target's recipe (delivered as the
process's actual argv) and is **not** propagated to deps. Treating argv as
config keeps the model uniform: everything a recipe sees is configuration, some
- inherited, some local. Local config is recorded per trace but is not part of
- `config-id`. It too is serialized to a canonical CAS blob, `argv-id`.
+ inherited, some local. Local config is not part of `config-id`; it is serialized
+ to its own canonical CAS blob, `argv-id`, and — unlike propagated config —
+ `argv-id` *is* a first-class component of the resolution identity (below).
**Serialized for replay, not just hashed.** Both the effective config map and the
argv vector are persisted *by value* as canonical, content-addressed CAS blobs
@@ -89,30 +100,39 @@ original build definition or workspace. Source files and globs are the opposite
case: kept by hash only, because they are *verified* (did this still hash the
same?), never replayed, and their bytes already live in the CAS.
-The **resolution identity** of a build is therefore `(target-name, config-id)`,
-where `config-id` is the effective *propagated* config. Local argv is determined
-by the target name (via the build definition), so it does not widen the key. Two
-requests `(T, C1)` and `(T, C2)` — the same target under different propagated
-config — are genuinely distinct builds; both may be in flight at once.
-
-`recipe-id` covers only the recipe *file bytes*. Two other things that can change
-a target's behavior — *which* recipe it maps to, and its argv — live in the
-**build definition**, which the coordinator reads to resolve `T`. That read is
-itself a tracked source dependency of `T`, so editing a target's stanza
-(repointing its recipe, or changing argv) invalidates its traces through the
-ordinary source-leaf mechanism.
+The **resolution identity** of a build is therefore the triple
+`(target-name, config-id, argv-id)`, where `config-id` is the effective
+*propagated* config and `argv-id` is the effective *local* argv (the request's
+argv, or the empty argv when none is supplied — the build definition carries no
+argv). The target name is just an identifier — argv is not encoded in it; it is
+supplied by the request and then content-addressed. Two requests that differ in the propagated
+config **or** the argv — `(T, C1, A)` and `(T, C2, A)`, or `(T, C, A1)` and
+`(T, C, A2)` — are genuinely distinct builds; any number may be in flight at once.
+
+`recipe-id` covers only the recipe *file bytes*. *Which* recipe a target maps to
+lives in the **build definition**, which the coordinator reads to resolve `T`.
+The only definition-derived input recorded for `T` is its **resolved recipe-id**
+(a per-node scalar), refreshed by recomputing it through the *live* definition —
+not a hash of the whole definition file. So repointing `T` to a different-content
+recipe busts it; repointing to a same-content recipe does not (it is the same
+build); and editing an *unrelated* target's stanza leaves `defn_find(T)`
+unchanged, so it does not bust `T`. The *mapping* is not an independent input —
+only the resolved recipe content is. (argv is *not* in the definition — it comes
+from the request, and a different argv is a different `argv-id`, hence a distinct
+build.)
### Why two trace kinds
-The deep trace answers *"is this exact configuration unchanged and has any source
-moved?"* in one pass over a flat leaf set — no graph walk. When it matches, the
-output is determined and we skip straight to materialization. It is the inner
-dev-loop fast path: edit code, rebuild under the same config.
+The deep trace answers *"is this exact configuration unchanged and has any input
+moved?"* by refreshing the deepset closure — a DAG walk with structural sharing
+and an id-equality short-circuit, not a re-resolution. When it matches, the output
+is determined and we skip straight to materialization. It is the inner dev-loop
+fast path: edit code, rebuild under the same config.
The shallow trace handles everything the deep path defers — a changed source that
-might not actually move a dep's output, and *any* configuration change. It
+might not actually move a dep's output, and *any* configuration or argv change. It
rebuilds the *direct* deps (cheaply, via their own traces, threading the new
-config down through `need`), compares their resulting `tree-id`s against what it
+config and argv down through `need`), compares their resulting `tree-id`s against what it
recorded, and if they match, the recipe is still skipped. This is the payoff of
recording deps by *output* identity rather than by input identity: a subtree that
ignores a changed config key, or absorbs a comment-only edit, produces the same
@@ -126,11 +146,11 @@ immutable and self-verifying; exactly one class of object is mutable.
```
<store>/
cas/ # the shared content store, per DISTRIBUTE.md
- blob/<pp>/<blob-id> # raw file bytes (sources, config maps, argv vectors, trace bodies)
+ blob/<pp>/<blob-id> # raw bytes: sources, config maps, argv vectors, deepset nodes
tree/<pp>/<tree-id> # canonical directory manifests (output trees)
... # (chunk/, index/ as in DISTRIBUTE.md)
build/
- trace/<pp>/<trace-id> # IMMUTABLE canonical trace bodies (deep + shallow)
+ trace/<pp>/<trace-id> # IMMUTABLE canonical trace bodies (deep + shallow) — CLAIMS
target/<pp>/<target-key> # MUTABLE per-target trace set (the only mutable state)
cache/<pp>/<tree-id>/ # tree cache: materialized output directories
tmp/ # staging for atomic writes + recipe sandboxes
@@ -139,10 +159,19 @@ immutable and self-verifying; exactly one class of object is mutable.
`<pp>` is the first two lowercase hex chars of the relevant id (the DISTRIBUTE.md
convention). `trace-id` = `BLAKE2b` of the canonical trace body.
`target-key` = `BLAKE2b("kit build target v1" ‖ target-name)` — note it depends
-on the target *name only*, never on config or recipe-id, so a target's index
-entry is a stable handle while its inputs churn. Config maps and trace bodies are
-ordinary CAS blobs; only `target/` records and the `cache/` materializations are
-build-specific.
+on the target *name only*, never on config, argv, or recipe-id, so a target's
+index entry is a stable handle holding candidate traces across every
+`(config, argv)` it was recently built under, while its inputs churn.
+
+The line that sorts `cas/` from `build/` is **self-verifiability, not
+content-addressedness**. Config maps, argv vectors, and **deepset closure nodes**
+(below) are self-verifying content — hand over the bytes and the recipient
+recomputes the id and transitively checks every reference — so they are ordinary
+CAS blobs (trustless, fetchable from an untrusted mirror, bundled as plain
+blobs). A **trace body**, though equally content-addressed, is a *claim*
+(`inputs ⇒ output`) that cannot be verified without re-running the recipe, so it
+lives in `build/trace/` and is shared only as a *signed* bundle. Only `target/`
+records and the `cache/` materializations are otherwise build-specific.
### Trace bodies (immutable, content-addressed)
@@ -165,16 +194,20 @@ config <config-id> ; this target's effective propagated config (full
argv <argv-id> ; this target's local config (serialized vector blob)
[config] ; CONSUMED propagated key NAMES, sorted (values live in the map)
opt
-[source] ; sorted by path
+[source] ; sorted by path ("<path> -" marks an absent read)
src/main.c <blob-id>
-build.kit <blob-id> ; the build definition stanza is a tracked source
[glob] ; sorted by pattern
src/*.c <glob-result-hash>
-[dep] ; sorted by (dep-name, dep-config-id) — each row replays one `need`
-//lib:core <dep-config-id> <dep-output-tree-id>
+[dep] ; sorted by (dep-name, dep-config-id, dep-argv-id) — each row replays one `need`
+//lib:core <dep-config-id> <dep-argv-id> <dep-output-tree-id>
```
-**Deep trace** — root config-id plus the transitive source/glob closure, *no*
+The `recipe <recipe-id>` scalar is the only definition-derived input (recomputed
+through the live definition on refresh); the definition file itself is *not* a
+source leaf, so editing an unrelated target's stanza does not bust this trace.
+
+**Deep trace** — the `(root config-id, argv)` binding to an output, plus a single
+pointer to the transitive input closure (held as a **deepset** DAG, below), *no*
`[dep]` and *no* per-key `[config]` section:
```
@@ -184,13 +217,30 @@ recipe <recipe-id>
output <output-tree-id>
root-config <config-id> ; this target's effective propagated config (full map blob)
argv <argv-id> ; local config (serialized vector blob)
-[source] ; union over the whole closure, sorted, deduped
+deepset <deep-set-id> ; the transitive input closure, as a deepset DAG (a CAS blob)
+```
+
+**Deepset** — one node of the transitive input-closure DAG, a CAS blob. The deep
+trace inlines none of the closure; it points at the root deepset, and each node
+points at its children. A subtree reached through many parents is *one* node
+(structural sharing, like the CAS), so the closure is never re-listed per
+ancestor:
+
+```
+kit-build-deepset 1
+target //app:server ; carried so refresh can recompute this node's recipe-id
+recipe <recipe-id> ; the node's recipe content-hash (the only definition-derived input)
+[source] ; this node's DIRECT source leaves, sorted by path
src/main.c <blob-id>
-lib/core.c <blob-id> ; pulled in transitively from //lib:core
-[glob]
+config.h - ; an ABSENT leaf: "-" = the path was absent when read
+[glob] ; this node's DIRECT glob leaves, sorted by pattern
src/*.c <glob-result-hash>
+[child] ; direct deps' deep-set-ids, sorted
+<deep-set-id of //lib:core's node>
```
+`deep-set-id = BLAKE2b(canonical deepset body)`, which is exactly its CAS blob id.
+
- `config <config-id>` and `argv <argv-id>` reference the **serialized** effective
config map and argv vector — canonical CAS blobs, kept by value and bundled with
shared traces — so the target's invocation and every `need` it issues replay
@@ -201,23 +251,40 @@ src/*.c <glob-result-hash>
trace — no sentinel value needed. Matching compares, for each consumed key, the
request's value against the built map's value (absent == unset).
- `<blob-id>` = the source file's CAS blob id (`kit_blob_info`); source and glob
- leaves are kept by hash because they are *verified*, never replayed.
+ leaves are kept by hash because they are *verified*, never replayed. An absent
+ source is recorded as `<path> -` (a lone `-`), so *creating* the file later
+ busts the trace — the source analogue of a consumed-while-unset config key.
+- `recipe <recipe-id>` (on the shallow trace and on every deepset node) is the
+ target's **only** definition-derived input: `BLAKE2b` of the recipe *file*
+ bytes, refreshed by recomputing it through the *live* definition
+ (`defn_find(target) → recipe path → hash`). That catches a recipe edit *or* a
+ repoint to a different-content recipe at any depth, treats a repoint to a
+ same-content recipe as unchanged, and — because it goes through `defn_find` —
+ is unaffected by edits to *unrelated* stanzas. There is no whole-definition
+ source leaf.
+- `deepset <deep-set-id>` is the root of the closure DAG. Phase 1 loads it and
+ its children from the CAS to refresh; if any deepset blob is absent (e.g.
+ GC'd), the deep trace is treated as absent and resolution falls through
+ (fail-safe).
- `<glob-result-hash>` = `BLAKE2b` of the canonical `(path, blob-id)` listing the
pattern matched (sorted by path). It changes if any matched file is added,
removed, or edited — so one glob row covers the existence *and* content of its
whole match set, and files read through a glob need no separate `[source]` rows.
-- `[dep]` rows carry the **dep's config-id** (and output `tree-id`); together with
- the dep's own serialized config/argv this re-resolves each dep under the exact
- propagated config it used, overlays included.
-
-**Why the deep trace needs only `root-config` + source/glob leaves.** If the
-request's `config-id` equals `root-config` *and* every source/glob leaf still
-matches the live workspace, then every recipe in the closure has byte-identical
-inputs — so every `need` overlay it computes is identical, every effective config
-downstream is identical, and by determinism the output is identical. Any input
-that could perturb a downstream overlay (a source a recipe branches on, a config
-value) is itself either a recorded source/glob leaf or folded into `root-config`,
-so nothing escapes the check. Using the whole `root-config` (rather than just
+- `[dep]` rows carry the **dep's config-id and argv-id** (and output `tree-id`);
+ together with the dep's own serialized config/argv blobs this re-resolves each
+ dep under the exact propagated config *and* argv it used, overlays included.
+
+**Why the deep trace needs only `root-config` + the deepset closure.** If the
+request's `config-id` equals `root-config`, its `argv-id` equals the recorded
+`argv`, *and* refreshing the deepset finds nothing moved — every source/glob leaf
+still matches the live workspace **and** every node's `recipe-id` still recomputes
+the same through the live definition — then every recipe in the closure has
+byte-identical inputs *and* identical code, so every `need` overlay it computes is
+identical, every effective config downstream is identical, and by determinism the
+output is identical. Any input that could perturb a downstream overlay (a source a
+recipe branches on, a config value) is itself either a recorded source/glob leaf,
+a per-node `recipe-id`, or folded into `root-config`, so nothing escapes the
+check. Using the whole `root-config` (rather than just
consumed keys) is a deliberate, conservative simplification: an *unconsumed*
config change busts the deep fast path, but the [shallow path](#resolution-algorithm)
recovers — it re-runs only the recipes whose output actually changed, so unrelated
@@ -241,8 +308,9 @@ Each row points at a trace body in `build/trace/`. On every successful resolutio
the coordinator prepends the fresh trace, de-duplicates, and truncates each kind
to a small cap (`KIT_BUILD_RECORD_CAP`, e.g. 8) — a bounded MRU window. Older
traces age out and become GC-eligible. Multiple candidates exist because the same
-target may have been built under several distinct input/config combinations
-recently (e.g. two config values flipped back and forth).
+target may have been built under several distinct input, config, and argv
+combinations recently (e.g. two config values flipped back and forth, or two
+different argvs).
### Storage state machine (atomicity and crash safety)
@@ -292,7 +360,8 @@ hashed once:
| `source_hashes` | `path → blob-id` (+ stat). | Per-process memo. |
| `glob_results` | `pattern → (sorted paths, glob-result-hash)`. | Per-process memo. |
| `config_maps` | `config-id → map`, and overlay results. | Immutable; content-addressed. |
-| `targets` | `(target-name, config-id) → future` of `{output-tree-id, deep-leafset, path}`. | Memo **and** in-flight dedup. |
+| `deepsets` | `deep-set-id → (interned node, refresh-valid?)`. | Shares subtrees; skips unchanged ones by id. |
+| `targets` | `(target-name, config-id, argv-id) → future` of `{output-tree-id, deepset node, path}`. | Memo **and** in-flight dedup. |
| `cas` | Open `KitCas*` handle over `cas/`. | Plus optional remote (below). |
| `store` | Paths + target-record reader/writer over `build/`. | — |
| `jobs` | Semaphore bounding concurrent recipe processes. | The global parallelism limit. |
@@ -301,87 +370,105 @@ hashed once:
tables: the coordinator assumes the workspace does not change *under it*
mid-build.
-The `targets` table is keyed by the full `(target-name, config-id)` and holds a
-**future**, not a flag — so concurrent `need`s of the same `(T, config-id)` await
-one resolution rather than racing (see [Parallelism](#parallelism)). A resolved
+The `targets` table is keyed by the full `(target-name, config-id, argv-id)` and
+holds a **future**, not a flag — so concurrent `need`s of the same
+`(T, config-id, argv-id)` await one resolution rather than racing (see
+[Parallelism](#parallelism)). A resolved
build yields the **output `tree-id`** (to satisfy a `need`), a materialized
-**path**, and its **deep leafset** (the transitive source/glob closure, to fold
-into a parent's deep trace).
+**path**, and its **deepset node** (the root of its transitive input-closure DAG,
+to fold into a parent's deepset).
### Cycle detection
The `targets` memo alone cannot catch a cycle — a self-dependent target would
simply await its own unresolved future and deadlock. So each resolution carries an
-explicit **build chain**: the ordered list of `(target-name, config-id)` frames
-from the root request down to here. A `need` whose `(dep, config-id)` already
-appears on the chain is a dependency cycle; the chain *is* the error message
-(`//a → //b → //a`). The chain is per-path and distinct from the cross-path memo:
-the same `(T, config-id)` may appear in many chains (a shared dep) but never twice
-in one.
+explicit **build chain**: the ordered list of `(target-name, config-id, argv-id)`
+frames from the root request down to here. A `need` whose
+`(dep, config-id, argv-id)` already appears on the chain is a dependency cycle;
+the chain *is* the error message (`//a → //b → //a`). The chain is per-path and
+distinct from the cross-path memo: the same `(T, config-id, argv-id)` may appear
+in many chains (a shared dep) but never twice in one.
### Resolution algorithm
-A request is `resolve(T, cfg, chain)` where `cfg` is the effective propagated
-config (with `cfg.id` its config-id). The config-dependence is recorded in and
-verified against the traces; the *deep* path checks whole-config-id equality, the
-*shallow* path checks consumed keys.
+A request is `resolve(T, cfg, argv, chain)` where `cfg` is the effective
+propagated config (with `cfg.id` its config-id) and `argv` is the effective local
+argv (with `argv.id` its argv-id — the request's argv, or empty when none is
+supplied). The config/argv-dependence is recorded in and verified against the
+traces; the *deep* path checks whole-config-id and argv-id equality, the *shallow*
+path checks consumed keys and argv-id.
```
-resolve(T, cfg, chain):
- key = (T, cfg.id)
+resolve(T, cfg, argv, chain):
+ key = (T, cfg.id, argv.id)
if key in chain: error cycle(chain + key)
if key in targets: return await targets[key] # cross-path memo / in-flight dedup
targets[key] = new future ; chain' = chain + key
- # ---- Phase 1: deep fast path (same config, did sources move?) -------
+ if record(T) is empty and trace_remotes configured: # clean checkout: try shared traces
+ pull_once(T) # verify+install signed bundle, then re-scan
+
+ # ---- Phase 1: deep fast path (same config+argv, did anything move?) ---
for D in deep_traces(T), newest-first:
- if D.root_config == cfg.id
- and D.argv == argv_id(T)
- and refresh(D.source_glob_leaves) all match live: # memoized rehash / reglob
- return done(materialize(D.output, T, cfg), D.leafset)
+ if D.root_config == cfg.id and D.argv == argv.id:
+ node = deepset_load(D.deepset) # load DAG from CAS; absent => skip (fail-safe)
+ if node and refresh(node) all match live: # rehash/reglob + recompute recipe-id, memoized
+ p = materialize(D.output) # pure: cache -> CAS -> remote; ERR => fall through
+ if p ok: return done(D.output, p, node)
- # ---- Phase 2: shallow path (config and/or sources moved) ------------
+ # ---- Phase 2: shallow path (config, argv, and/or sources moved) ------
for S in shallow_traces(T), newest-first:
- if S.recipe != recipe_id(T): continue
- if S.argv != argv_id(T): continue # local config
+ if S.recipe != recipe_id(T): continue # recipe edit/repoint (live defn)
+ if S.argv != argv.id: continue # local config
M = config_by_id(S.config) # the built map (serialized blob)
if any consumed key in S.config-keys differs between cfg and M: continue
if any direct source/glob leaf of S changed: continue
- ok = true ; child_leafsets = []
- for (dep, dep_cfg_id, recorded_tree) in S.deps: # may run in parallel
- r = resolve(dep, config_by_id(dep_cfg_id), chain') # replay the need: map recovered by id
+ ok = true ; child_nodes = []
+ for (dep, dep_cfg_id, dep_argv_id, recorded_tree) in S.deps: # may run in parallel
+ r = resolve(dep, config_by_id(dep_cfg_id), argv_by_id(dep_argv_id), chain') # replay the need by id
if r.output != recorded_tree: ok = false; break
- child_leafsets.push(r.leafset)
+ child_nodes.push(r.leafset)
if ok:
- leafset = union(direct source/glob leaves of S, child_leafsets)
- write_deep_trace(T, cfg.id, leafset, S.output) # refresh the deep trace
- return done(materialize(S.output, T, cfg), leafset)
+ node = union(direct{recipe_id, source/glob leaves} of S, child_nodes) # builds+stores the deepset
+ write_deep_trace(T, cfg.id, argv.id, node.id, S.output) # refresh the deep trace
+ p = materialize(S.output) # pure; ERR => fall through
+ if p ok: return done(S.output, p, node)
# ---- Phase 3: run the recipe ----------------------------------------
- return run_recipe(T, cfg, chain')
+ return run_recipe(T, cfg, argv, chain')
```
-with the materialization ladder shared by every cache hit:
+with the materialization ladder shared by every cache hit — a **pure locator**
+that never runs a recipe:
```
-materialize(tree-id, T, cfg):
- if tree-id in build/cache/ : return that path # already on disk
+materialize(tree-id) -> path | MISS:
+ if tree-id in build/cache/ : return that path # already on disk
if tree-id in cas/ : restore into build/cache/, return path
if remote configured : fetch tree-id (+ blobs) into cas/, verify, restore
- else : return run_recipe(T, cfg) # bytes gone: rebuild
+ else : MISS # bytes gone everywhere
```
+On `MISS` the caller does not return — it falls through to the next candidate
+and ultimately Phase 3, which rebuilds and records the correct output. Keeping
+`materialize` recipe-free is what lets a cache hit's recorded `tree-id` always
+describe the bytes it returns (no rebuild can silently substitute a different
+tree), and removes the resolve→runner edge.
+
This is the brief's flow, sharpened by config:
-- *Phase 1* is the deep fast path: identical config-id and an all-clear source/glob
- refresh ⇒ reuse. `refresh` hits the per-process memo, so after the first target
- touches a file or glob the rest of the closure is free.
-- *Phase 2* is the shallow path, taken when Phase 1 finds nothing (config changed,
- or a source moved). It guards on the target's own consumed config and direct
- leaves, then re-resolves each recorded dep **under the dep's recorded config-id**
- and compares outputs. Re-using the recorded `dep_cfg_id` is correct: if the
- parent's consumed config were different, the consumed-key guard above would have
- failed first and the parent would re-run, recomputing its `need` overlays.
+- *Phase 1* is the deep fast path: identical config-id **and argv-id** and an
+ all-clear deepset refresh — every source/glob leaf unchanged **and** every
+ node's `recipe-id` still recomputing the same — ⇒ reuse. `refresh` hits the
+ per-process memos and the deep-set-id validity cache, so a shared subtree is
+ checked once and an unchanged subtree is skipped by id equality.
+- *Phase 2* is the shallow path, taken when Phase 1 finds nothing (config or argv
+ changed, or a source moved). It guards on the target's own argv-id, consumed
+ config, and direct leaves, then re-resolves each recorded dep **under the dep's
+ recorded config-id and argv-id** and compares outputs. Re-using the recorded
+ `(dep_cfg_id, dep_argv_id)` is correct: if the parent's consumed config or argv
+ were different, the guards above would have failed first and the parent would
+ re-run, recomputing its `need` overlays.
- A Phase-2 hit also writes a *fresh* deep trace from the now-known closure, so the
next request gets the Phase-1 fast path back.
- *Phase 3* runs the recipe only when no trace holds, or when a hit's bytes were
@@ -396,24 +483,30 @@ check; the shallow trace is the "did the churn actually reach me?" check.
### Running a recipe
```
-run_recipe(T, cfg, chain):
+run_recipe(T, cfg, argv, chain):
acquire jobs # global parallelism limit
sandbox = build/tmp/run-<n>/ ; out = sandbox/out/
- spawn recipe (argv = argv(T), recovered by argv_id) with env:
- { KIT_BUILD_SOCK, KIT_BUILD_OUT=out, KIT_BUILD_TARGET=T, workspace root }
+ spawn recipe (process argv = argv, the request's argv or empty when none given) with a
+ CLEAN env (host inherits nothing): { KIT_BUILD_SOCK, KIT_BUILD_OUT=out,
+ KIT_BUILD_TARGET=T, workspace root } + cfg's `env.*` keys as declared env vars
service the recipe's protocol requests, logging each as a dep (next section);
- config-get reads cfg; need recurses resolve(...) under cfg ⊕ overrides
+ config-get reads cfg; source returns the live workspace path + pinned blob-id;
+ need releases this recipe's job slot, recurses resolve(...) under cfg ⊕ overrides
+ and the need's own effective argv, then reacquires the slot (so a chain deeper
+ than `jobs` cannot deadlock)
release jobs
on nonzero exit: propagate failure, write NO trace
on success:
output = kit_cas_add_tree_from_dir(out)
- put serialized cfg map (cfg.id) and argv vector (argv_id(T)) into the CAS
- write_shallow_trace(T, recipe_id, argv_id, cfg.id, consumed key names,
- direct leaves, dep edges with their config-ids, output)
- write_deep_trace (T, cfg.id, argv_id, transitive source/glob closure, output)
+ put serialized cfg map (cfg.id) and argv vector (argv.id) into the CAS
+ node = union(direct{recipe_id, source/glob/absent leaves}, child deepset nodes)
+ # builds + stores the root deepset as a CAS blob, yields its deep-set-id
+ write_shallow_trace(T, recipe_id, argv.id, cfg.id, consumed key names,
+ direct leaves, dep edges with their config-ids and argv-ids, output)
+ write_deep_trace (T, cfg.id, argv.id, node.id, output) # references the deepset
prepend both to T's target record (dedup, truncate to cap)
install out in build/cache/<output>/
- return done({output, path}, leafset)
+ return done({output, path}, node)
```
Writing *both* trace kinds on every real build is what lets a later request take
@@ -423,12 +516,20 @@ whichever path fits the change it sees.
The coordinator owns all parallelism: it is the single process that launches and
manages every recipe, so the overall job limit is configured in it (the `jobs`
-semaphore). Independent `need`s resolve concurrently up to that limit; the
-`targets` futures guarantee a shared `(T, config-id)` is built once even when many
-parents request it at the same instant. Scheduling is at recipe granularity — a
-recipe may parallelize internally, but the coordinator counts it as one job.
-Cycle detection composes with concurrency through the per-path build chain
-described above; the global memo handles dedup, the chain handles cycles.
+semaphore). Parallelism comes from two places: **independent targets** (sibling
+deps, multi-target requests, Phase-2 dep re-resolution) and **a single recipe's
+fan-out** — a recipe issues `need-submit` for each dependency, the coordinator
+dispatches them onto workers concurrently, and the recipe `need-await`s the
+tokens. Both reduce to one primitive: a fresh `(T, config-id, argv-id)` is
+*dispatched* (driven on a worker thread), and the `targets` futures guarantee it
+is built once even when many parents request it at the same instant — a
+duplicate awaits the in-flight future rather than racing. The semaphore bounds
+*actively running* recipes, not in-flight ones: a recipe blocked in a `need`
+(or `need-await`) releases its slot and reacquires when the sub-build returns, so
+a chain deeper than `jobs` cannot deadlock. Cycle detection composes with
+concurrency through the per-path build chain (a parent-linked cactus stack, so
+each dispatched need extends a shared read-only prefix without copying): the
+global memo handles dedup, the chain handles cycles.
## Recipe protocol
@@ -439,9 +540,16 @@ length-prefixed request/response framing; the host supplies connect/read/write.
```c
typedef struct KitBuildTransport {
- /* Accept one recipe connection (the recipe dialed the endpoint named in
- * $KIT_BUILD_SOCK); read/write length-prefixed frames; close. */
- int (*accept)(void* user, KitBuildConn** out);
+ /* Server: create a uniquely-named endpoint for one recipe (its name, written
+ * into name_out, is passed to the child in $KIT_BUILD_SOCK); accept the
+ * recipe's one connection on it; tear the endpoint down when done. Per-recipe
+ * endpoints are what let many recipes run concurrently. */
+ int (*listen)(void* user, char* name_out, size_t cap, KitBuildListener** out);
+ int (*accept)(void* user, KitBuildListener* lst, KitBuildConn** out);
+ void (*close_listener)(void* user, KitBuildListener* lst);
+ /* Client: dial the endpoint named in $KIT_BUILD_SOCK; read/write length-
+ * prefixed frames; close. */
+ int (*dial)(void* user, KitSlice endpoint, KitBuildConn** out);
int (*read_frame)(void* user, KitBuildConn*, uint8_t* buf, size_t cap, size_t* n);
int (*write_frame)(void* user, KitBuildConn*, const uint8_t* buf, size_t n);
void (*close)(void* user, KitBuildConn*);
@@ -465,9 +573,11 @@ command.
| Command | Returns | Dependency logged |
|---------|---------|-------------------|
| `config-get <key>` | the propagated value (or *unset*) | config dep: key recorded as consumed; its value is the one in the effective `config-id` map (absent = unset) |
-| `source <path>` | `blob-id` + a path to read | source dep: `(path, blob-id)` |
-| `glob <pattern>` | sorted list of matching paths | glob dep: `(pattern, glob-result-hash)` |
-| `need <target> [k=v…]` | each dep's output `tree-id` + a path | target dep edge: `(dep, dep-config-id, output-tree-id)` |
+| `source <path>` | `blob-id` (or *absent*) + a path to read | source dep: `(path, blob-id)`, or `(path, absent)` |
+| `glob <pattern>` | sorted list of matching paths (streamed) | glob dep: `(pattern, glob-result-hash)` |
+| `need <target> [k=v…] [argv…]` | the dep's output `tree-id` + a path (blocks) | target dep edge: `(dep, dep-config-id, dep-argv-id, output-tree-id)` |
+| `need-submit <target> [k=v…] [argv…]` | a **token** (does not block) | *(nothing yet — logged on await)* |
+| `need-await <token>` | the submitted dep's output `tree-id` + a path | target dep edge: `(dep, dep-config-id, dep-argv-id, output-tree-id)` |
- **`config-get`** reads the target's *effective propagated* config and records the
value's hash, so changing a consumed key invalidates while changing an unconsumed
@@ -475,17 +585,43 @@ command.
*new* key next time, the old trace already fails to match on the changed key, so
the new key-set is discovered on the rebuild — self-correcting, never needing to
predict the input set ahead of time. Local config (argv) is *not* read here; it
- arrives as the process's argv.
-- **`source`** hands back a path inside the workspace (or, in a hermetic variant, a
- path staged into the sandbox) plus the blob-id the read is pinned to.
+ arrives as the process's argv. **Environment** is config too: keys under the
+ `env.` prefix (e.g. `env.PATH`) become the recipe's environment variables, so a
+ recipe's env is tracked, hermetic, and propagates like any config — the spawned
+ process otherwise gets a *clean* environment (nothing ambient inherited).
+- **`source`** hands back a path inside the **live workspace** (not a staged
+ copy) plus the blob-id the read is pinned to; an absent path is reported and
+ recorded as an *absent* leaf (creating it later busts the trace). Returning the
+ live path makes workspace immutability for the build's duration load-bearing
+ for soundness — see [Determinism](#determinism-and-hermeticity).
- **`glob`** records the whole match set's content via `glob-result-hash`; the
- recipe then reads the returned paths without further declaration.
+ recipe then reads the returned paths without further declaration. Matches are
+ **streamed** across as many frames as needed, so an arbitrarily large match set
+ is not bounded by a single frame.
- **`need`** is the dynamic-dependency primitive. The optional `k=v` pairs
- **overlay** propagated config for that sub-build (local argv never propagates).
- The coordinator resolves `(dep, cfg ⊕ overrides)` recursively, returns its output
- `tree-id` and a readable path, and records the edge with the dep's `config-id`
- so the shallow path can re-resolve identically. `need` is also where cycles are
- caught.
+ **overlay** propagated config for that sub-build; an optional argv vector sets
+ the dep's **local argv** (omitted ⇒ the empty argv; local argv never
+ propagates). The coordinator resolves `(dep, cfg ⊕ overrides, argv)`
+ recursively, returns its output `tree-id` and a readable path, and records the
+ edge with the dep's `config-id` and `argv-id` so the shallow path can re-resolve
+ identically. `need` is also where cycles are caught.
+- **`need-submit` / `need-await`** are the *future-based* form of `need`, for
+ building a recipe's deps **concurrently**. `need-submit` resolves nothing
+ inline — it dispatches `(dep, cfg ⊕ overrides, argv)` onto the coordinator's
+ workers and returns a **token** immediately (failing fast only on a cycle or
+ unknown target); the recipe submits its whole fan-out, then `need-await`s each
+ token to collect results. `need` is exactly `submit` + `await` fused into one
+ blocking round-trip (cheaper for the one-dep-at-a-time case, no worker
+ hand-off). Two rules keep the cache sound:
+ - **The dep edge is recorded on `await`, not `submit`.** A submitted-but-never-
+ awaited need is speculative (its output never reached the recipe), so it is
+ *not* a dependency; it is cancelled when the recipe exits.
+ - **`await` is always for a specific token** — there is deliberately no "await
+ whichever finishes first." Concurrency is a performance property only; the set
+ of needs a recipe awaits (hence its dependencies and output) must stay a
+ deterministic function of its inputs, never of completion order, or two runs
+ would record different dependencies. The recorded `[dep]` section is sorted,
+ so the trace is byte-identical regardless of submit/await order.
- **Outputs** need no command: the recipe writes under `$KIT_BUILD_OUT` and the
coordinator snapshots that directory on exit.
@@ -506,8 +642,9 @@ external-fetch templates. The fetched bytes land in `build/tmp/` and are
local CAS; a corrupt or malicious mirror fails the hash check and is discarded. So
the remote and the fetch recipe are *untrusted* — the existing self-verifying CAS
makes that safe. This adds one rung to the materialization ladder (tree cache →
-local CAS → remote fetch → run recipe) and lets a clean checkout *download* an
-output instead of rebuilding it.
+local CAS → remote fetch) and lets a clean checkout *download* an output instead
+of rebuilding it. A **deepset** node needed during a deep refresh is fetched the
+same way — it is a CAS blob (`{kind}=blob`), so no separate kind is required.
### Shared traces (trusted, as signed packages)
@@ -521,9 +658,11 @@ machinery wholesale:
- A bundle is a signed manifest (`kit-build-traces 1`, signed exactly like a
`kit-package` manifest) listing `(target-name, kind, trace-id, output-tree-id)`
claims, carried in a `.kpkg` (portable tar.gz or native kpkg) alongside the
- trace bodies and the **serialized config-map and argv blobs** they reference
- (required — without them an imported shallow trace cannot replay its `need`s),
- plus, optionally, the referenced output trees/blobs.
+ trace bodies and the CAS blobs they reference — the **serialized config-map and
+ argv blobs** (required, else an imported shallow trace cannot replay its
+ `need`s) and the **deepset closure blobs** a deep trace points at (required,
+ else an imported deep trace cannot refresh) — plus, optionally, the referenced
+ output trees/blobs.
- Trust is the DISTRIBUTE.md model unchanged: verify the minisign signature
against the trusted-keys file (`-p KEY`, the anchor file, or `--tofu`); the
signed trusted comment binds the signature to the manifest hash.
@@ -547,7 +686,15 @@ deterministically.
- **Declared-inputs-are-complete.** Reading an undeclared file is a hermeticity
violation. The protocol makes declaration the only *intended* way to get an
input, but **enforcement is deferred** (see below): for now it is a contract
- recipes must honor.
+ recipes must honor. The recipe runs with a **clean environment** (only the
+ KIT_BUILD_* vars and the build's declared `env.*` config), so ambient
+ PATH/locale/etc. cannot leak in as undeclared, untracked inputs.
+- **Workspace immutability is load-bearing.** `source` returns a live workspace
+ path, so the bytes a recipe reads are the bytes on disk *at read time*, not a
+ snapshot of what was hashed. If the workspace mutates mid-build the recorded
+ input hash and the bytes actually consumed can disagree, making a trace's claim
+ false — *unsound*, not merely stale. The coordinator therefore assumes the
+ workspace does not change for a build's duration; verify mode is the audit.
- **Determinism.** Timestamps, RNG, and unpinned network fetches break the model —
a second build's captured tree differs from the recorded output, and the cache
would serve a stale-but-believed-current result. Mitigations: declare such
@@ -562,20 +709,23 @@ deterministically.
`//app:server` depends on `//lib:core`; the recipe globs `src/*.c`, reads
`src/main.c`, and consumes config `opt`. The request is under config `C0` (so
-`config-id = c0`).
+`config-id = c0`). Neither request supplies argv, so every build's argv is empty
+(argv-id `a0`), held constant throughout; the full resolution key is the triple
+`(target, config-id, argv-id)`.
1. **Cold build.** No record. Phase 3 runs both recipes. `//lib:core` builds →
`tree L0`. `//app:server` logs `config opt`, `glob src/*.c`, `source src/main.c`,
- `need //lib:core` (under `c0` → `(//lib:core, c0)`); yields `tree A0`. Both trace
- kinds written for both targets. `build/cache/A0/` materialized, path returned.
+ `need //lib:core` (under `c0` → `(//lib:core, c0, a0)`); yields `tree A0`.
+ Both trace kinds written for both targets. `build/cache/A0/` materialized, path
+ returned.
2. **No-op rebuild (same `C0`).** Phase 1: `//app:server`'s deep trace has
- `root-config c0` (matches) and refreshes `opt` via config-id, `src/*.c`,
+ `root-config c0` and `argv a0` (both match) and refreshes `opt` via config-id, `src/*.c`,
`src/main.c`, and the folded `lib/core.c` — all memoized, all match. `A0` is in
the tree cache. Returns instantly, nothing re-run, no graph walk.
3. **Comment-only edit to `lib/core.c`.** `lib/core.c`'s blob-id moved → the deep
trace fails Phase 1 (a folded source leaf changed). Phase 2: direct leaves
(`opt`, `src/*.c`, `src/main.c`) unchanged, so probe the one dep —
- `resolve(//lib:core, c0)`. Its recipe re-runs (its source moved) but emits a
+ `resolve(//lib:core, c0, a0)`. Its recipe re-runs (its source moved) but emits a
byte-identical `L0` (comment stripped). `L0 == L0` ⇒ `//app:server`'s recipe is
**skipped**, `A0` restored, a refreshed deep trace written so step 2's fast path
returns next time.
@@ -583,10 +733,10 @@ deterministically.
`//app:server`'s deep trace has `root-config c0 ≠ c1`. Phase 2 under `c1`:
`opt` is a consumed key whose value differs between `c1` and the trace's built
map → no shallow trace holds → Phase 3 re-runs `//app:server`. Its `need //lib:core` carries no `opt` override,
- so the dep resolves as `(//lib:core, c1)`; but `//lib:core` never consumes
- `opt`, so its shallow trace under `c1` matches by consumed-keys (empty) and
- direct leaves, and `L0` is reused without re-running. Only the one recipe that
- actually depends on `opt` re-ran.
+ so the dep resolves as `(//lib:core, c1, a0)`; but `//lib:core` never
+ consumes `opt`, so its shallow trace under `c1` matches by argv-id (`a0`),
+ consumed-keys (empty), and direct leaves, and `L0` is reused without
+ re-running. Only the one recipe that actually depends on `opt` re-ran.
## Decided / deferred / open
@@ -601,7 +751,30 @@ deterministically.
content id (trustless).
- **Trace sharing** ships traces as signed `.kpkg` bundles over the DISTRIBUTE.md
trust model (trusted), because traces are claims, not self-verifying content.
-- **Public surface** is a public header `<kit/build.h>` plus a `kit build` driver
+ Self-verifying content (sources, output trees, config/argv blobs, **deepset
+ nodes**) lives in `cas/`; only claims live in `build/trace/`.
+- **Structural deepset closure (adopted).** The transitive closure is a Merkle
+ DAG of **deepset** nodes — `{ target, recipe-id, direct source/glob leaves,
+ [child-deep-set-id…] }`, `deep-set-id = BLAKE2b(canonical body)` — each a
+ self-verifying **CAS blob** (`deep-set-id` == blob-id). A deep trace inlines
+ none of the closure; it points at the root deepset, and a subtree reached
+ through many parents is one node (structural sharing like the CAS). Building a
+ parent is "concat direct leaves + child ids, hash" — the union *is* the
+ construction, never an N-way merge. Refresh memoizes `deep-set-id → valid?`, so
+ a shared subtree is checked once and an unchanged subtree is skipped by id
+ equality at its root without descending — change detection is O(changed
+ frontier), not O(closure). The cost accepted: refresh walks the DAG (rather than
+ one flat list) and the child deepset blobs must be present (GC-rooted; bundled
+ with shared traces; an absent one makes Phase 1 fall through, fail-safe).
+- **Recipe identity is content-only.** The sole definition-derived input is a
+ target's resolved `recipe-id` (a per-node scalar), refreshed by recomputing it
+ through the live definition. No whole-definition source leaf; the target→recipe
+ *mapping* is not an independent input.
+- **Recipe environment is clean** and declared via the `env.*` config namespace
+ (tracked); `source` reads are live (workspace immutability is load-bearing);
+ large globs stream across frames; `materialize` is a pure locator (no rebuild
+ rung); a clean-checkout miss tries one lazy signed-trace pull before Phase 3.
+- **Public surface** is a public header `<kit/build_coord.h>` plus a `kit build` driver
command, gated by `KIT_BUILD_ENABLED` (the `kit/cas.h` + `kit cas` precedent). A
thin CLI parses flags and supplies the transport/host vtables; the coordinator
and trace model live in the library.
@@ -611,34 +784,13 @@ deterministically.
- **Hermeticity enforcement.** No sandbox-deny or FS-trace at first; the protocol
defines the contract, enforcement comes later (deny undeclared reads, or
`strace`/FUSE detection that warns).
+- **In-process recipes** run on a `KitBuildSched` thread (the framed protocol
+ needs the recipe to run concurrently with the service loop, which a subprocess
+ gives for free). This is a supported configuration — it is the substrate for
+ the integration test harness ([BUILD_TESTING.md](BUILD_TESTING.md)). Only the
+ *sequential* (`sched == NULL`) in-process case is deferred; it would need a
+ direct-dispatch bypass of the service loop.
- **GC.** Deferred. When built: mark-and-sweep rooted at live target records →
- reachable trace bodies → referenced trees/blobs/config-maps/argv vectors; sweep
- unreachable `trace/`, `cas/`, and `cache/`, with LRU size bounds on the
- (re-derivable) tree cache and CAS.
-
-**Open — compact deep traces.** The flat deep leafset re-lists every transitive
-source in *every* ancestor: O(Σ closure sizes) bytes and O(closure) work to
-refresh. A **structural** representation fixes both and is the natural form of the
-"efficient set union" the closure already needs:
-
-- Store each target's deep leafset as a content-addressed object
- `deep-set = { direct source/glob leaves, [child-deep-set-id…] }`, with
- `deep-set-id = BLAKE2b(canonical(direct ‖ child-ids))`. The fully-flattened set
- is the recursive union, but it is never materialized — an ancestor stores only
- its own direct slice plus 32-byte pointers to its children's deep-sets. A leaf
- shared by many ancestors is stored once (a Merkle DAG, structurally sharing like
- the CAS itself). This *is* the union operation: building a parent's deep-set is
- "concatenate sorted direct leaves with child-set pointers," not "merge N flat
- arrays."
-- **Faster checking** falls out of the same structure. Refresh memoizes
- `deep-set-id → valid?` for the whole build: a subtree checked once is reused by
- every parent that points at it, and an *unchanged* subtree is recognized by
- pointer/hash equality at its root without descending to leaves. Change detection
- becomes O(changed frontier) instead of O(closure): recompute current deep-set-ids
- bottom-up (memoized, so O(distinct subtrees) = O(nodes)), and where a recomputed
- id equals the recorded one, the whole subtree is provably unchanged and skipped.
-- The cost is that refresh walks the DAG rather than streaming one flat list, and
- the child deep-set objects must be present (another store dependency, GC-rooted
- like traces). The flat form in this doc is the *semantic* definition; the
- structural form is the recommended physical representation once flat traces bloat
- — adopt it together with the in-memory union it mirrors.
+ reachable trace bodies → referenced output trees/blobs, config-maps, argv
+ vectors, **and the deepset-closure blobs**; sweep unreachable `trace/`, `cas/`,
+ and `cache/`, with LRU size bounds on the (re-derivable) tree cache and CAS.
diff --git a/doc/plan/BUILD_INTERNALS.md b/doc/plan/BUILD_INTERNALS.md
@@ -0,0 +1,980 @@
+# Build coordinator — internal module design
+
+> **Status: design, not yet built.** Companion to [BUILD.md](BUILD.md), which
+> specifies *what* the content-addressed build coordinator does (the trace model,
+> the storage state machine, the resolution algorithm, the recipe protocol). This
+> doc specifies *how* the implementation is carved into modules under `src/build/`:
+> for each module, what it is responsible for, which other modules it depends on,
+> and a pseudocode sketch dense enough to prove the interfaces in the `src/build/*.h`
+> spike are sufficient to implement every module **in parallel**. Where a sketch
+> exposed a gap, it is called out inline and collected in
+> [Interface findings](#interface-findings).
+
+The public surface is `<kit/build_coord.h>` (coordinator + recipe-side client) plus a
+`kit build` driver command, gated by `KIT_BUILD_ENABLED`, mirroring the
+`<kit/cas.h>` + `kit cas` precedent. Everything below is internal to `src/build/`
+except the composition root `src/api/build.c` and the driver glue.
+
+## Layering and the dependency graph
+
+The build layer sits entirely on the public kit surface — the content store
+(`<kit/cas.h>`), digests (`<kit/hash.h>`), the canonical writer/heap/slice core
+(`<kit/core.h>`), and signed packages (`<kit/package.h>`) — and adds exactly one
+thing the CAS lacks: a *mutable per-target trace index* plus the resolution
+algorithm that drives it. Every id it computes is a plain BLAKE2b-256 of canonical
+bytes, which **is** a CAS blob id (`kit_blob_info().id`), so config maps, argv
+vectors, and **deepset closure nodes** content-address themselves through the
+ordinary CAS as ordinary blobs (self-verifying content). Trace bodies are
+content-addressed too, but live in `build/trace/` rather than `cas/`, because a
+trace is a *claim* (not self-verifying) — the criterion that sorts the two stores
+is self-verifiability, not content-addressedness.
+
+Internal modules form a strict DAG (no cycles — verified against the `#include`
+sets in the spike). Arrows point "depends on":
+
+```
+ ┌─────────────────────────────────────────┐
+ L6 composition │ src/api/build.c (KitStatus, ctx->diag)│
+ └───────────────┬─────────────────────────┘
+ │ uses everything below
+ L5 recipe execution runner ───────────────► bundle (trace sharing)
+ │ │ │ │
+ L4 algorithm resolve │ │ │
+ │ │ │ │
+ L3 coordinator state coord ◄──────────────────┘
+ ┌──┴───┬────────┬─────────┐
+ L2 I/O + remote store remote (client ── recipe side, L1-only)
+ │
+ L1 pure value/byte ┌── trace ──┐ cfg defn protocol
+ logic (no I/O) │ │
+ L0 foundation └──────── build.h ───────────────────────┘
+ (constants, ids, result convention, diag)
+```
+
+| Layer | Modules | Property |
+|-------|---------|----------|
+| L0 foundation | `build` | constants, `BUILD_OK/ERR`, magics, `build_id_eq`, `build_target_key`, `build_diagf`. No internal deps. |
+| L1 pure logic | `cfg`, `defn`, `trace`, `protocol`, `client`† | byte/value transforms only: emit / parse / hash / encode / decode. **No I/O, no CAS, no coordinator.** Unit-testable in isolation. |
+| L2 store + remote | `store`, `remote` | the on-disk state machine and the untrusted fetch rung. Touch the filesystem (via host vtables) and the CAS; interpret no trace semantics beyond `trace`. |
+| L3 coordinator | `coord` | the `KitBuildCoordinator` context: process-lifetime memos, the jobs semaphore, per-target futures, base-input probes. |
+| L4 algorithm | `resolve` | the 3-phase `resolve()` + build chain + materialization ladder + leafset union/refresh. |
+| L5 execution | `runner`, `bundle` | Phase 3 (spawn + service a recipe, record both traces) and signed trace import/export. |
+| L6 composition | `src/api/build.c` | maps internal `int` → `KitStatus`, routes detail through `ctx->diag`, owns section-buffer sizing. The public functions in `<kit/build_coord.h>` live here. |
+
+† `client` is L1 by dependency (it only needs `protocol` + a transport) but is a
+*peer process role*, not part of the coordinator. It backs the public
+`kit_build_client_*` recipe-side API.
+
+**Result/error convention (inherited from `src/dist`).** Internal modules return
+`int` (`BUILD_OK` / `BUILD_ERR`); parse/emit modules additionally take a
+`char* err, size_t errcap` sink. Only `src/api/build.c` knows `KitStatus` and
+`ctx->diag`. All ids are lowercase hex on the wire and in every canonical body.
+
+**One id invariant the whole layer leans on:** for any canonical byte string `b`,
+`kit_blob_info(&info, b, len); info.id` equals the BLAKE2b-256 that names `b` as a
+CAS blob. So "compute the config-id/argv-id/trace-id" and "store the blob and read
+its id back" agree by construction — a trace can reference a config map by id and
+later recover the exact bytes with `kit_cas_get_blob`. Pure index keys that are
+*never* stored as content (`target-key`, `glob-result-hash`) use the same hash over
+a domain-separated buffer.
+
+---
+
+## L0 — `build` (foundation)
+
+**Role.** Shared vocabulary for every other module: hash length, the `BUILD_OK/ERR`
+convention, the canonical magics (`kit-build-shallow 1`, `…-deep 1`, `…-record 1`,
+`…-config 1`, `…-argv 1`, `…-traces 1`, `kit-build 1`), the fixed row capacities,
+byte-exact id equality, the per-target index-key derivation, and the diag shim.
+
+**Depends on.** `<kit/cas.h>` (hash length), `<kit/hash.h>`, `<kit/core.h>`. No
+internal deps.
+
+**Pseudocode.**
+
+```c
+int build_target_key(KitSlice name, uint8_t out[32]):
+ // target-key = BLAKE2b("kit build target v1" || name); a pure index key,
+ // never stored, so hash a domain-separated scratch buffer directly.
+ buf = stack[ sizeof(DOMAIN)-1 + name.len ] // bounded by BUILD_TARGET_MAX
+ memcpy(buf, BUILD_TARGET_KEY_DOMAIN, ...); append name
+ KitBlobInfo bi; kit_blob_info(&bi, buf, len); memcpy(out, bi.id, 32)
+ return BUILD_OK
+
+void build_diagf(ctx, fmt, ...): // mirror cas_diagf
+ if !ctx || !ctx->diag: return
+ va_start; format into a bounded stack buffer; ctx->diag(ctx->diag_user, msg); va_end
+```
+
+`build_id_eq` is the existing inline. **No interface gap.**
+
+---
+
+## L1 — `cfg` (propagated config + local argv)
+
+**Role.** The two configuration objects a build records, each serialized *by value*
+as a canonical content-addressed CAS blob (never an opaque one-way hash) so an
+imported trace can replay every `need` exactly. `BuildConfig` is the effective
+**propagated** map (`config-id`); `BuildArgv` is the **local** argv vector
+(`argv-id`). Pure value logic: init / set / get / overlay / canonical emit / parse /
+id. Entries hang off caller storage (the `DistTree` growable-buffer pattern, no VLAs).
+
+**Depends on.** `build` (L0), `<kit/core.h>` (`KitWriter`, `KitHeap`),
+`<kit/cas.h>` (`kit_blob_info`), `<kit/build_coord.h>` (`KitBuildKV` overlay pairs).
+
+**Pseudocode.**
+
+```c
+int build_config_set(cfg, key, val):
+ if key.len > BUILD_KEY_MAX-1 || val.len > BUILD_VAL_MAX-1: return BUILD_ERR
+ i = lower_bound(cfg.entries, key) // keep sorted, dedup-on-set
+ if i found and equal-key: overwrite value; else insert (grow if n==cap → ERR if no room)
+ return BUILD_OK
+
+int build_config_overlay(base, overrides[], n, out): // the `need` overlay
+ copy base into out (already canonical/sorted)
+ for kv in overrides: build_config_set(out, kv.key, kv.value) // re-sorts/dedups
+ return BUILD_OK
+
+int build_config_emit(cfg, w): // canonical, byte-stable
+ write "kit-build-config 1\n"
+ for e in sorted(cfg.entries): write e.key, ' ', e.value, '\n' // sort enforced here
+ return kit_writer_status(w)
+
+int build_config_id(heap, cfg, out[32]):
+ kit_writer_mem(heap, &w); build_config_emit(cfg, w)
+ bytes = kit_writer_mem_bytes(w, &len); KitBlobInfo bi; kit_blob_info(&bi, bytes, len)
+ memcpy(out, bi.id, 32); kit_writer_close(w); return BUILD_OK
+
+int build_config_parse(data, len, out, err, errcap):
+ require first line == BUILD_CONFIG_MAGIC else parse-error (→ caller treats absent)
+ for each "key value" line: enforce strictly-increasing key, no dups → build_config_set
+ // non-canonical ordering / unknown shape / dup = BUILD_ERR
+```
+
+`build_argv_*` mirror this but order is significant (positional vector, magic
+`kit-build-argv 1`, one arg per line, **no** sort). The empty argv has a fixed
+`argv-id` = hash of just the magic line — the default when a request supplies none.
+**No interface gap.**
+
+---
+
+## L1 — `defn` (build definition)
+
+**Role.** Parse the build-definition file (`kit-build 1`) mapping each target name →
+one recipe path, and expose a by-name lookup. The coordinator reads it to resolve a
+target. The **only** definition-derived input of a target is its resolved
+`recipe-id`, refreshed by recomputing it through the *live* definition
+(`build_defn_find` → recipe path → hash) — there is **no** whole-definition source
+leaf, so editing an unrelated stanza does not bust unrelated targets, and a repoint
+to a same-content recipe is correctly a no-op. The definition carries **no argv** —
+local argv comes entirely from the request.
+
+**Depends on.** `build` (L0), `<kit/core.h>`. Pure parse + lookup, no I/O.
+
+**Pseudocode.**
+
+```c
+int build_defn_parse(data, len, out, err, errcap):
+ out.bytes = data; out.len = len // retained so coord can hash file as a source leaf
+ require first line == "kit-build 1"
+ loop: expect "[target <name>]" then "recipe <path>"
+ validate name (BUILD_TARGET_MAX) and path (build path-valid), append stanza
+ enforce sorted-unique target names (canonical) else BUILD_ERR
+ return BUILD_OK
+
+const BuildTargetDefn* build_defn_find(defn, name):
+ binary-search defn.targets by name; return match or NULL
+```
+
+**Interface note.** `out.bytes` aliases the caller's buffer; the coordinator must
+keep the definition bytes alive for the coordinator's lifetime (it does:
+`coord.defn_bytes`). The recipe-id is *not* here — `coord` hashes the recipe file
+named by `recipe_path`. **No interface gap.**
+
+---
+
+## L1 — `trace` (trace bodies + target record)
+
+**Role.** The immutable, content-addressed heart of the cache: emit / parse / id for
+the shallow trace, the deep trace, the **deepset** closure node, and the mutable
+target record. Pure byte logic —
+strict, byte-stable INI-style text; **any** non-canonical input (unknown key, dup
+row, mis-ordering) is a parse error, and a parse error is treated as *absent*, never
+as a match (fail safe). Sections hang off caller-provided growable buffers.
+
+**Depends on.** `build` (L0), `<kit/core.h>`. No I/O — `store` persists/loads these,
+`resolve` matches against them.
+
+**Pseudocode.**
+
+```c
+int build_shallow_emit(t, w, err, errcap):
+ write "kit-build-shallow 1\n"
+ write "target ", t.target, '\n'; "recipe ", hex(t.recipe), '\n'
+ "output ", hex(t.output), '\n'; "config ", hex(t.config), '\n'; "argv ", hex(t.argv), '\n'
+ write "[config]\n"; for k in sort(config_keys): write k.name, '\n'
+ write "[source]\n"; for s in sort_by_path(sources): require path-valid; write s.path, ' ', s.absent?"-":hex(s.blob), '\n'
+ write "[glob]\n"; for g in sort_by_pattern(globs): write g.pattern, ' ', hex(g.result_hash), '\n'
+ write "[dep]\n"; for d in sort_by(name,cfg,argv): write d.name,' ',hex(d.config_id),' ',hex(d.argv_id),' ',hex(d.output_tree),'\n'
+ return kit_writer_status(w)
+
+int build_deep_emit(t, w, err, errcap): // no [dep], no [config], no inline closure
+ write "kit-build-deep 1\n"; target/recipe/output; "root-config ", hex(t.root_config)
+ "argv ", hex(t.argv); "deepset ", hex(t.deepset) // one pointer to the closure DAG
+
+int build_deepset_emit(t, w, err, errcap): // one node of the closure DAG
+ write "kit-build-deepset 1\n"; "target ", t.target; "recipe ", hex(t.recipe)
+ "[source]\n"; for s in sort_by_path(sources): write s.path, ' ', s.absent ? "-" : hex(s.blob), '\n'
+ "[glob]\n"; for g in sort_by_pattern(globs): write g.pattern, ' ', hex(g.result_hash), '\n'
+ "[child]\n"; for c in sort(children): write hex(c), '\n' // direct deps' deep-set-ids
+
+void build_trace_id(body, len, out[32]): KitBlobInfo bi; kit_blob_info(&bi, body, len); memcpy(out, bi.id, 32)
+// build_deepset_id is the same hash; a deepset is stored as an ordinary CAS blob, so deep-set-id == blob-id.
+
+int build_record_prepend(rec, row): // MRU, per-kind cap
+ drop any existing row equal in (kind, trace_id)
+ shift rows down; rec.rows[0] = row
+ truncate the run of rows with row.kind to KIT_BUILD_RECORD_CAP (other kind untouched)
+```
+
+Parsers are the strict inverse; `build_*_parse` reject non-canonical bytes with
+`BUILD_ERR`. **No interface gap** — emit takes `KitWriter`, parse fills caller
+buffers, exactly the `dist_tree`/`dist_manifest` shape.
+
+---
+
+## L1 — `protocol` (recipe wire format)
+
+**Role.** The recipe command set (`config-get`, `source`, `glob`, `need`) and its
+length-prefixed framing, as pure encode/decode over a `KitBuildConn`. No I/O, no
+transport: the coordinator's `runner` drives the server side, `client` drives the
+recipe side, both calling these codecs. Every request returns a value *and* logs a
+dependency; an input read but not requested is invisible to the cache.
+
+**Depends on.** `build` (L0), `<kit/core.h>`, `<kit/build_coord.h>` (`KitBuildKV`,
+`KitBuildConn`).
+
+**Pseudocode.**
+
+```c
+int build_proto_encode_req(req, buf, cap, n):
+ cursor c over buf (cap ≥ BUILD_FRAME_MAX)
+ put u8 req.cmd; put_slice(req.arg)
+ if cmd==NEED or cmd==NEED_SUBMIT: put u16 noverrides; for kv: put_slice(kv.key), put_slice(kv.value)
+ put u16 argc; for a: put_slice(a)
+ if cmd==NEED_AWAIT: put u64 req.token
+ *n = c.len; return (c overran) ? BUILD_ERR : BUILD_OK
+
+int build_proto_decode_resp(buf, len, cmd, out, glob_cb, glob_user):
+ out.status = get_u8
+ switch cmd:
+ CONFIG_GET: if status==UNSET return OK(present=0); else out.text = get_slice
+ SOURCE: get_bytes(out.id,32); out.text = get_slice // realpath
+ NEED, AWAIT: get_bytes(out.id,32); out.text = get_slice // tree-id + path
+ NEED_SUBMIT: out.token = get_u64 // coordinator-assigned
+ GLOB: u32 m = get_u32; repeat m: p = get_slice; if glob_cb(user, p) != 0 break // streamed
+ return malformed ? BUILD_ERR : BUILD_OK
+```
+
+`build_proto_decode_req` fills caller-provided `ovr_storage` / `argv_storage` for a
+`need`/`need-submit`, and reads `req.token` for `need-await`. Frames are bounded by
+`BUILD_FRAME_MAX`. **No interface gap.**
+
+---
+
+## L1 — `client` (recipe-side, `src/build/client.c`)
+
+**Role.** Implements the public `kit_build_client_*` API a recipe links (or reaches
+through the in-process transport / a `kit` helper subcommand). Each call encodes a
+request with `protocol`, writes it over the transport, reads the response, decodes
+it. It is the *mirror* of `runner`'s service loop and shares zero coordinator state.
+
+**Depends on.** `protocol` (L1), `<kit/build_coord.h>` (`KitBuildTransport`,
+`KitBuildClient`, `KitBuildResult`), `<kit/core.h>`.
+
+**Pseudocode.**
+
+```c
+struct KitBuildClient { ctx; const KitBuildTransport* t; KitBuildConn* conn; uint8_t frame[BUILD_FRAME_MAX]; }
+
+KitStatus kit_build_client_open(ctx, transport, out):
+ c = alloc; c.t = transport; endpoint = getenv(KIT_BUILD_ENV_SOCK)
+ transport->dial(transport->user, endpoint, &c.conn); *out = c; return KIT_OK
+
+KitStatus kit_build_client_source(c, path, blob[32], realpath):
+ build_proto_encode_req({SOURCE, path}, c.frame, ...); transport->write_frame(conn, frame, n)
+ transport->read_frame(conn, frame, cap, &n); build_proto_decode_resp(frame, n, SOURCE, &resp, 0,0)
+ if resp.status != OK: return KIT_<err>; memcpy(blob, resp.id, 32); *realpath = resp.text; return KIT_OK
+
+KitStatus kit_build_client_need(c, req, out): // req carries overlay + argv
+ encode NEED(req.target, req.config, req.argv); round-trip; decode → out.output_tree, out.path
+```
+
+**Interface note (gap → see findings #1).** `client` is a real translation unit with
+no internal header in the spike. That's fine (its interface *is* `<kit/build_coord.h>`),
+but the parallel-work plan must assign it an owner. Listed here so it isn't lost.
+
+---
+
+## L2 — `store` (on-disk state machine)
+
+**Role.** The build store under `<store>/build/` and its crash-safe rules:
+immutable content objects (`trace/`, materialized `cache/`) written tmp→fsync→atomic-
+rename; the one mutable, ordering-sensitive object (the target `record/`) updated by
+read-modify-write into tmp then atomic rename; recipe sandboxes under `tmp/`. Owns
+content-keyed path layout (`<pp>/<id>`), atomicity, and the record RMW. It interprets
+*no* trace semantics beyond calling `trace`'s emit/parse.
+
+**Depends on.** `build` (L0), `trace` (L1, for `BuildTargetRecord` and record
+emit/parse), `<kit/cas.h>` (the shared CAS, blob/tree ops), `<kit/build_coord.h>`
+(`KitBuildStoreIo` atomicity extras, `KitCasHost` for `file_io`).
+
+**Pseudocode.**
+
+```c
+int build_store_put_trace(s, body, len, out_id[32]): // idempotent content write
+ build_trace_id(body, len, out_id)
+ build_store_trace_path(s, out_id, final, …)
+ if exists(final): return BUILD_OK // re-derivation is a no-op
+ io->make_temp_dir(build/tmp, tmpdir); tmp = tmpdir/"t"
+ cas_host->file_io->open_writer(tmp, &w); w.write(body,len); w.close()
+ io->sync_path(tmp); io->rename(tmp, final); io->sync_path(parent(final))
+ return BUILD_OK
+
+int build_store_record_update(s, key[32], name, kind, trace_id[32]):
+ L = io->lock ? io->lock(key) : NULL // optional advisory lock
+ build_store_record_load(s, key, name, &rec) // missing/garbage → empty rec (fail safe)
+ build_record_prepend(&rec, {kind, trace_id}) // MRU + cap, from trace.h
+ kit_writer_mem(heap,&w); build_record_emit(&rec,w,…); bytes = mem_bytes(w,&n)
+ write bytes to build/tmp/rec-XXXX; sync; build_store_target_path(s,key,final); io->rename(tmp, final); sync parent
+ if L: io->unlock(L); return BUILD_OK
+
+int build_store_cache_materialize(s, tree_id[32], path_out, cap): // CAS → cache/
+ if kit_cas_verify_tree(cas, tree_id) != KIT_OK: return BUILD_ERR // bytes absent → caller climbs ladder
+ io->make_temp_dir(build/tmp, stage)
+ kit_cas_materialize_tree(cas, tree_id, stage) // verifies each blob, applies modes
+ build_store_cache_path(s, tree_id, final, cap); io->rename(stage, final); io->sync_path(parent)
+ copy final into path_out; return BUILD_OK
+
+int build_store_ingest_output(s, out_dir, out_tree_id[32], path_out, cap):
+ kit_cas_add_tree_from_dir(cas, out_dir, out_tree_id) // hashes+stores every file → tree id
+ return build_store_cache_materialize(s, out_tree_id, path_out, cap) // install into cache/ (idempotent)
+
+int build_store_sandbox_new(s, sandbox_out, scap, out_dir_out, ocap):
+ io->make_temp_dir(build/tmp, sandbox_out) // run-<n>/
+ cas_host->mkdir_p(sandbox_out + "/out"); copy "<sandbox>/out" → out_dir_out; return BUILD_OK
+```
+
+**Interface notes.**
+- `build_store_open` must create the `build/{trace,target,cache,tmp}/` skeleton —
+ it has the `KitCasHost` (`mkdir_p`) and `store_root` to do so. OK.
+- `build_store_cache_lookup` is a pure existence probe (`io->list_dir` or a stat via
+ `file_io`); it returns `BUILD_ERR` on a miss so the caller climbs the ladder.
+- A subtlety worth a comment in the `.c`: `cache/<pp>/<tree-id>/` is a *directory*
+ rename, so `make_temp_dir` + populate + `rename` must be directory-atomic on every
+ supported host (POSIX `rename(2)` over an empty/no dest is; the Windows adapter must
+ emulate). This is a host-contract note, not a header change. **No header gap.**
+
+---
+
+## L2 — `remote` (untrusted object fetch)
+
+**Role.** The trustless rung of the materialization ladder: when an object is absent
+locally and an object-remote is configured, render the user's fetch-recipe argv
+template, spawn it via the exec host into a temp file, **verify** the bytes against
+the requested content id, and only then install into the CAS. A corrupt/malicious
+mirror fails the hash check and is discarded. Remotes are tried in order.
+
+**Depends on.** `build` (L0), `<kit/build_coord.h>` (`KitBuildExec`,
+`KitBuildObjectRemote`), `<kit/cas.h>` (install + verify).
+
+**Pseudocode.**
+
+```c
+int build_remote_fetch(ctx, exec, remotes[], n, cas, tmp_dir, kind, id[32]):
+ if n == 0: return BUILD_ERR
+ kit_hex_encode(hex, id, 32); pp = hex[0:2]
+ for r in remotes:
+ out = tmp_dir + "/fetch"
+ argv = render(r.fetch_argv_template, {kind: kind==BLOB?"blob":"tree", pp, id:hex, out})
+ exec->spawn(argv, …, &proc); exec->wait(proc, &code); if code != 0: continue
+ bytes = file_io->read_all(out)
+ if kind==BLOB:
+ kit_blob_info(&bi, bytes.data, bytes.len); if !id_eq(bi.id, id): discard; continue
+ kit_cas_add_blob(cas, bytes.data, bytes.len, &bi) // re-derives same id
+ else: // tree manifest: store as blob, then it can be verified/materialized by id
+ store tree manifest bytes; if resulting tree-id != id: discard; continue
+ return BUILD_OK
+ return BUILD_ERR
+```
+
+**Interface note (minor → findings #3).** The tree case needs an "ingest a *tree
+manifest* (not a directory) whose id I already know, and verify it equals `id`" CAS
+operation. `kit_cas_add_blob` stores raw bytes (the manifest *is* canonical bytes);
+verifying the stored tree then needs its referenced blobs present too. In practice a
+tree fetch is followed by blob fetches for its entries on demand during
+`materialize`. The fetch primitive itself only needs to install + id-check the
+manifest bytes; `materialize` already verifies the whole tree before use. The header
+suffices; the `.c` documents the two-step (tree manifest, then per-blob) fetch.
+
+---
+
+## L3 — `coord` (coordinator context, memos, concurrency)
+
+**Role.** Owns the `KitBuildCoordinator` and all process-lifetime state — no globals.
+Memoizes every base-input probe (source hash, glob, config-by-id, argv-by-id,
+recipe-id), every interned **deepset node** (by deep-set-id, with a refresh-valid
+cache so a shared subtree is checked once), and every target resolution, so a
+diamond builds once and a file hashes once. Owns the jobs semaphore, the per-target
+**futures** (cross-path memo *and* in-flight dedup) built on the optional
+`KitBuildSched`, and the per-target *pulled* set that makes the lazy trace-remote
+pull idempotent. Does *not* contain the resolution algorithm (that's `resolve`) or
+recipe execution (`runner`).
+
+**Depends on.** `build`, `cfg`, `defn`, `store`, `trace` (all included by the spike
+header), `<kit/build_coord.h>` (host vtables, options), `<kit/cas.h>`.
+
+**Pseudocode.**
+
+```c
+KitStatus build_coord_open(ctx, host, store_root, opts, out):
+ c = alloc; c.ctx_storage = *ctx; c.ctx = &c.ctx_storage; c.host = *host; c.opts = *opts
+ kit_cas_open(ctx, host->cas_host, store_root + "/cas", &c.cas)
+ build_store_open(ctx, c.cas, host->store_io, host->cas_host, store_root, &c.store)
+ c.defn_bytes = file_io->read_all(opts->build_def_path) // tracked source bytes
+ build_defn_parse(c.defn_bytes.data, .len, &c.defn, err, …)
+ c.jobs_limit = (host->sched && opts->jobs > 0) ? opts->jobs : (host->sched ? host_default : 1)
+ init memos (source/glob/config/argv/targets hash maps over ctx->heap)
+ if host->sched: c.lock = sched->mutex_new(); c.jobs_sem = sem_new(c.jobs_limit)
+ *out = c; return KIT_OK
+
+int build_coord_source_hash(c, path, out_blob[32], present): // memoized
+ lock; if path in c.sources: copy; unlock; return OK
+ fd = file_io->read_all(workspace_root + "/" + path)
+ if absent: *present = 0; memo (path → ABSENT); else kit_blob_info(&bi, fd.data, fd.len); out_blob = bi.id; *present = 1
+ memo insert; unlock; return OK
+
+int build_coord_glob(c, pattern, out_result_hash[32], cb, cb_user): // memoized
+ lock; if pattern in c.globs: replay matches to cb; copy hash; unlock; return OK
+ matches = host glob over workspace (sorted); for each: source_hash(match)
+ result_hash = build_glob_result_hash(sorted (path, blob) listing) // BLAKE2b of canonical listing
+ memo insert; replay to cb; unlock; return OK
+
+int build_coord_config_by_id(c, id[32], out): // replay path
+ lock; memo hit? copy. else: kit_cas_get_blob(cas, id, &fd); build_config_parse(fd.data, .len, out, …)
+ kit_cas_release(cas, &fd); memo; unlock
+ // build_coord_argv_by_id mirrors with build_argv_parse
+
+int build_coord_recipe_id(c, target, out[32]):
+ d = build_defn_find(&c.defn, target); if !d: return BUILD_ERR
+ build_coord_source_hash(c, d->recipe_path, out, &present) // recipe-id = hash(recipe file bytes)
+ return present ? BUILD_OK : BUILD_ERR
+
+int build_coord_deepset_load(c, deepset_id[32], out_node): // Phase-1 replay
+ if id in c.deepsets: *out_node = memoized; return BUILD_OK
+ kit_cas_get_blob(cas, deepset_id, &fd) // (remote blob fetch first if absent + remote)
+ else return BUILD_ERR // absent => Phase 1 falls through
+ build_deepset_parse(fd.data, .len, &ds, ...)
+ for child_id in ds.children: build_coord_deepset_load(c, child_id, &child) or return ERR // recurse+intern
+ node = intern(target, recipe, direct leaves, child nodes); c.deepsets[id] = node; *out_node = node
+
+int build_coord_trace_remote_pull_once(c, target, out_pulled_now): // lazy shared-trace pull
+ if c.opts.n_trace_remotes == 0 or target in c.pulled: *out_pulled_now = 0; return BUILD_OK
+ c.pulled.add(target); *out_pulled_now = (build_trace_remote_pull(c, target) == BUILD_OK)
+ return BUILD_OK
+
+// ---- concurrency ----
+void build_coord_jobs_acquire(c): if c.jobs_sem: sem_wait(c.jobs_sem) // no-op in sequential mode
+void build_coord_jobs_release(c): if c.jobs_sem: sem_post(c.jobs_sem)
+
+int build_coord_target_intern(c, target, cfg_id, argv_id, out_future, is_fresh):
+ key = (target, cfg_id, argv_id); lock
+ if key in c.targets: *out_future = existing; *is_fresh = 0
+ else: f = new future(state=PENDING); c.targets[key] = f; *out_future = f; *is_fresh = 1
+ unlock; return OK
+
+int build_coord_target_await(c, f, out): // futures over sched cond
+ lock; while f.state == PENDING: sched->cond_wait(f.cond, c.lock)
+ ok = (f.state == DONE); if ok: *out = f.result; unlock; return ok ? OK : ERR
+void build_coord_target_complete(c, f, res): lock; f.result=*res; f.state=DONE; cond_broadcast(f.cond); unlock
+void build_coord_target_fail(c, f): lock; f.state=FAILED; cond_broadcast(f.cond); unlock
+```
+
+In **sequential mode** (`host->sched == NULL`): `jobs_*` are no-ops, the lock is a
+no-op, and a fresh future is completed inline before any other caller can observe it,
+so `await` never blocks — futures degrade to a plain memo exactly as the public
+header promises.
+
+**Interface notes.**
+- `build_glob_result_hash` (the canonical `(path, blob-id)` listing → 32-byte hash)
+ is used by both `coord` and `trace` matching. Today it is implied inside
+ `build_coord_glob`. **Findings #4:** give it a name in `build.h` (a pure helper) so
+ `trace` matching and `coord` agree byte-for-byte.
+- The jobs semaphore is built *in `coord`* on `KitBuildSched`'s mutex+cond (the
+ vtable has no semaphore primitive — correct; a counting semaphore is trivially a
+ mutex+cond+int). `c.jobs_sem` is `void*` for that reason. OK.
+
+---
+
+## L4 — `resolve` (the three-phase algorithm)
+
+**Role.** `build_resolve(T, cfg, argv, chain) → {output-tree, path, leafset}`, doing
+the least work necessary: Phase 1 deep fast path (same config-id+argv-id, did sources
+move?), Phase 2 shallow path (config/argv/sources moved — guard consumed keys + direct
+leaves, re-resolve recorded deps by their recorded ids, compare *output* tree-ids),
+Phase 3 run the recipe (delegated to `runner`). Owns the per-path build **chain**
+(cycle detection), the materialization ladder, and leafset union/refresh.
+
+**Depends on.** `coord` (L3, memos + futures + store access), `cfg` (L1), `trace`
+(via `coord`/`store`), `runner` (Phase 3 — see note), `build` (L0).
+
+**Pseudocode.**
+
+```c
+int build_resolve(c, T, cfg, argv, chain, out): // INLINE (top-level + fused need)
+ if build_chain_extend(c, chain, T, cfg.id, argv.id, &chain', err) != OK: diag(err); return BUILD_ERR // cycle
+ build_coord_target_intern(c, T, cfg.id, argv.id, &f, &fresh)
+ if !fresh: return build_coord_target_await(c, f, out) // memo / dedup (no pop — chain is immutable)
+
+ rc = resolve_phases(c, T, cfg, argv, chain', out) // 1 → 2 → 3 below, on THIS thread
+ if rc == OK: build_coord_target_complete(c, f, out) else build_coord_target_fail(c, f)
+ return rc
+
+int build_dispatch(c, T, cfg, argv, chain, &f_out, err): // ASYNC (need-submit)
+ if build_chain_extend(c, chain, T, cfg.id, argv.id, &chain', err) != OK: return BUILD_ERR // cycle, fail fast
+ build_coord_target_intern(c, T, cfg.id, argv.id, &f, &fresh)
+ if fresh:
+ if build_coord_spawn(c, worker, pack(c,T,cfg,argv,chain',f)) != OK: // no sched → resolve inline now
+ rc = resolve_phases(c,T,cfg,argv,chain',&r); rc==OK ? complete(f,r) : fail(f)
+ *f_out = f; return BUILD_OK // returns WITHOUT awaiting
+ // worker(args): rc = resolve_phases(...); rc==OK ? build_coord_target_complete(f,r) : build_coord_target_fail(f)
+
+resolve_phases(c, T, cfg, argv, chain, out):
+ key = build_target_key(T); rec = store_record_load(key, T)
+ if rec empty: build_coord_trace_remote_pull_once(c, T, &pulled); if pulled: rec = store_record_load(key, T)
+
+ // ---- Phase 1: deep fast path -------------------------------------------------
+ for trace_id in rec where kind==DEEP, newest-first:
+ D = load+parse deep trace (absent/garbage → skip)
+ if id_eq(D.root_config, cfg.id) and id_eq(D.argv, argv.id):
+ if build_coord_deepset_load(c, D.deepset, &node) != OK: continue // blob absent → fall through
+ build_leafset_refresh(c, node, &all_match) // rehash/reglob + recompute recipe-id
+ if all_match and build_materialize(c, D.output, out->path, …) == OK:
+ out->output_tree = D.output; out->leafset = node; return BUILD_OK
+ // materialize MISS → keep scanning, ultimately Phase 3 (no rebuild inside materialize)
+
+ // ---- Phase 2: shallow path ---------------------------------------------------
+ for trace_id in rec where kind==SHALLOW, newest-first:
+ S = load+parse shallow trace (absent/garbage → skip)
+ if !id_eq(S.recipe, recipe_id(T)): continue // recipe edit/repoint (live defn)
+ if !id_eq(S.argv, argv.id): continue // local config differs
+ M = config_by_id(S.config)
+ if any name in S.config_keys differs between cfg and M (absent==unset): continue
+ if any direct source/glob leaf of S changed (refresh): continue
+ ok = 1; children = []
+ for (dep, dcfg_id, dargv_id, recorded_tree) in S.deps: // may fan out on threads
+ dcfg = config_by_id(dcfg_id); dargv = argv_by_id(dargv_id)
+ r = build_resolve(c, dep, dcfg, dargv, chain, &child) // replay the need exactly
+ if r != OK or !id_eq(child.output_tree, recorded_tree): ok = 0; break
+ children.push(child.leafset) // child deepset nodes
+ if ok:
+ build_leafset_union(c, direct_node(S), children, &node) // builds + stores the root deepset
+ write_deep_trace(c, T, cfg.id, argv.id, node.id, S.output) // refresh → next call hits Phase 1
+ if build_materialize(c, S.output, out->path, …) != OK: continue // bytes gone → Phase 3
+ out->output_tree = S.output; out->leafset = node; return BUILD_OK
+
+ // ---- Phase 3: run the recipe -------------------------------------------------
+ return build_run_recipe(c, T, cfg, argv, chain, out) // runner.h
+
+int build_materialize(c, tree_id, path_out, cap): // PURE locator — never runs a recipe
+ if build_store_cache_lookup(store, tree_id, path_out, cap) == OK: return OK // on disk
+ if build_store_cache_materialize(store, tree_id, path_out, cap) == OK: return OK // local CAS → cache
+ if opts.n_object_remotes and build_remote_fetch(…, tree_id) == OK: // remote (trustless)
+ return build_store_cache_materialize(store, tree_id, path_out, cap)
+ return BUILD_ERR // bytes gone → caller falls to Phase 3
+
+int build_chain_extend(c, parent, T, cfg_id, argv_id, &out, err, errcap): // cactus stack, no pop
+ for fr = parent; fr; fr = fr->parent: // walk to the root
+ if fr==(T,cfg_id,argv_id): render "//a → //b → //a" into err; return BUILD_ERR // cycle
+ f = arena_alloc(c.build_arena, BuildChainFrame); *f = {parent, T, cfg_id, argv_id}
+ *out = f; return BUILD_OK
+ // Immutable + parent-linked: concurrent dispatched siblings each allocate their own frame
+ // over the shared read-only `parent`; nothing is copied and nothing is popped.
+```
+
+**Interface notes.**
+- `resolve` and `runner` form one **strongly-connected component**, not a DAG edge:
+ `resolve` calls `build_run_recipe` (Phase 3), and `runner`'s service loop calls
+ `build_resolve` for each `need`. The "strict DAG" elsewhere holds; this single
+ edge is an acknowledged cycle, broken at the *header* level only: `build_run_recipe`
+ is declared in `resolve.h` (its Phase-3 continuation) and implemented in `runner.c`,
+ so `resolve.c` never includes `runner.h`. `materialize` no longer participates —
+ it is a pure locator (below), so the only resolve→runner call is the Phase-3 tail.
+- `build_materialize` is **pure** (cache → CAS → remote, no `target/cfg/argv/chain`):
+ on a total miss it returns `BUILD_ERR` and `resolve` falls through to Phase 3. This
+ is what removes the old `materialize → run_recipe` rung and guarantees a cache hit's
+ recorded `tree-id` always describes the bytes returned.
+- The `chain` is a **parent-linked cactus stack** of `BuildChainFrame`s (no VLAs),
+ allocated off a per-build arena `kit_build` owns and resets per request. It
+ replaces the old fixed-depth array: concurrent dispatched needs each extend a
+ shared read-only parent (no copy, no pop), and a cycle is detected by walking
+ parent links. Parents outlive their children by construction (the parent only
+ returns once all its `need-await`s complete), so the arena lifetime is sound.
+- `build_dispatch` is the async sibling of `build_resolve` (the `need-submit`
+ engine): same intern + 3-phase body, but driven on a `build_coord_spawn` worker
+ so submit returns without blocking. `build_resolve` = inline drive; the recipe
+ fan-out = N×`build_dispatch` then N×`build_coord_target_await`. Both share
+ `resolve_phases`; futures dedup means a repeated dep spawns no second worker.
+
+---
+
+## L5 — `runner` (Phase 3: spawn + service a recipe)
+
+**Role.** Run a recipe under `(cfg, argv)`: acquire a job slot, stage a sandbox,
+spawn via the exec host with `KIT_BUILD_SOCK/OUT/TARGET` in the env, **service** its
+protocol connection (logging every request as a dep, recursing into `build_resolve`
+for each `need`), then on success ingest the output tree, write *both* trace kinds,
+prepend them to the record, install the output in the cache, and return. A nonzero
+exit propagates failure and writes **no** trace.
+
+**Depends on.** `coord` (L3), `resolve` (L4, for `need` recursion + materialize),
+`protocol` (L1, wire codec — `runner.c` includes it), `trace` (L1), `cfg` (L1),
+`store` (via `coord`), `<kit/build_coord.h>` (exec + transport vtables).
+
+**Pseudocode.**
+
+```c
+int build_run_recipe(c, T, cfg, argv, chain, out):
+ if c.opts.verify and a trace says "unchanged": run anyway, compare fresh tree-id, diag on mismatch // verify mode
+ build_coord_jobs_acquire(c) // bounds *actively running* recipes
+ build_store_sandbox_new(store, sandbox, out_dir)
+ listener = transport->listen(&endpoint_name) // per-recipe endpoint ← findings #5
+ env = CLEAN: { KIT_BUILD_SOCK=endpoint_name, KIT_BUILD_OUT=out_dir, KIT_BUILD_TARGET=T,
+ workspace_root } + cfg's `env.`-prefixed keys (KIT_BUILD_ENV_PREFIX) as env vars
+ exec->spawn(recipe_argv = [recipe_path] ++ argv.args, env, cwd=workspace_root, &proc) // host inherits nothing
+ log = {}; transport->accept(listener, &conn)
+ build_runner_service(c, conn, T, cfg, chain, &log) // drives protocol; recurses for `need`
+ transport->close(conn); transport->close_listener(listener)
+ exec->wait(proc, &code)
+ build_coord_jobs_release(c)
+ if code != 0: build_store_sandbox_done(store, sandbox); return BUILD_ERR // NO trace
+ build_store_ingest_output(store, out_dir, &output_tree, out->path, …)
+ build_store_sandbox_done(store, sandbox)
+ put serialized cfg map (→ cfg.id) and argv vector (→ argv.id) into the CAS (kit_cas_add_blob)
+ build_runner_record_traces(c, T, cfg, argv, &log, output_tree, &node) // builds+stores deepset, both kinds
+ out->output_tree = output_tree; out->leafset = node; return BUILD_OK
+
+int build_runner_service(c, conn, T, cfg, chain, log):
+ loop:
+ transport->read_frame(conn, frame, cap, &n); if EOF/closed: return BUILD_OK // recipe done
+ build_proto_decode_req(frame, n, &req, ovr_storage, …, argv_storage, …)
+ switch req.cmd:
+ CONFIG_GET: build_config_get(cfg, req.arg, &v, &present)
+ append req.arg to log.config_keys (consumed, set or unset)
+ reply {present?OK:UNSET, text=v}
+ SOURCE: build_coord_source_hash(c, req.arg, blob, &present)
+ append (req.arg, blob, absent=!present) to log.sources
+ reply present ? {OK, id=blob, text=realpath} : {OK absent, no path} // live workspace path
+ GLOB: build_coord_glob(c, req.arg, result_hash, collect_paths, &acc)
+ append (req.arg, result_hash) to log.globs
+ reply matches in OK batch frames, then a GLOB_END frame // streamed, multi-frame
+ NEED: dcfg = build_config_overlay(cfg, req.overrides, …); dargv = build_argv_set(req.argv, …)
+ build_coord_jobs_release(c) // ← yield slot while blocked (findings #6)
+ build_resolve(c, req.arg, &dcfg, &dargv, chain, &r) // inline (no worker)
+ build_coord_jobs_acquire(c) // ← reclaim before resuming work
+ record_dep(log, req.arg, dcfg.id, dargv.id, &r) // append edge + push child node
+ reply {OK, id=r.output_tree, text=r.path}
+ NEED_SUBMIT: dcfg = overlay; dargv = set
+ if build_dispatch(c, req.arg, &dcfg, &dargv, chain, &f, err) != OK: reply {ERROR} // cycle/unknown
+ tok = log.next_token++; log.pending.push({tok, req.arg, dcfg.id, dargv.id, f})
+ reply {OK, token=tok} // does NOT block
+ NEED_AWAIT: p = find_pending(log, req.token); if none: reply {ERROR}
+ build_coord_jobs_release(c); build_coord_target_await(c, p.future, &r); build_coord_jobs_acquire(c)
+ if r failed: reply {ERROR}
+ record_dep(log, p.dep, p.config_id, p.argv_id, &r) // edge logged HERE, on await
+ remove p from log.pending; reply {OK, id=r.output_tree, text=r.path}
+ // on EOF: for each still-pending need, exec->kill its in-flight recipes (best-effort); record NO edge
+
+int build_runner_record_traces(c, T, cfg, argv, log, output[32], &out_node):
+ recipe_id = build_coord_recipe_id(c, T)
+ build_leafset_union(c, direct_node{T, recipe_id, log.sources, log.globs}, log.child_leafsets, &node)
+ // builds the root deepset node, emits it, stores it as a CAS blob, yields node.id (= deep-set-id)
+ S = shallow{T, recipe_id, output, cfg.id, argv.id, log.config_keys, log.sources, log.globs, log.deps}
+ emit S → bytes; build_store_put_trace(store, bytes, &sid); build_store_record_update(store, key(T), T, SHALLOW, sid)
+ D = deep{T, recipe_id, output, cfg.id, argv.id, deepset=node.id} // one pointer, not an inline closure
+ emit D → bytes; build_store_put_trace(store, bytes, &did); build_store_record_update(store, key(T), T, DEEP, did)
+ *out_node = node; return BUILD_OK
+```
+
+**Interface notes.**
+- **Findings #5 (transport listener).** Spawning *concurrent* recipes needs each to
+ have its own endpoint so `accept` is unambiguous. The current
+ `KitBuildTransport.accept(user, out)` has no endpoint and there is no "create a
+ uniquely-named endpoint" call, while the client side already `dial`s the name in
+ `$KIT_BUILD_SOCK`. Proposal: add a per-recipe listener to the vtable —
+ `listen(user, char* name_out, size_t cap, KitBuildListener** out)`,
+ `accept(user, KitBuildListener*, KitBuildConn** out)`, `close_listener`. The runner
+ creates a listener per recipe, passes `name_out` as `KIT_BUILD_SOCK`, accepts once,
+ then closes. The in-process transport returns a synthetic name and a direct conn.
+- **Findings #6 (jobs slot while blocked on `need`).** A parent recipe servicing a
+ `need` is idle CPU-wise while the child builds; if it keeps its job slot, a build
+ with `jobs < max-chain-depth` deadlocks (parent holds the only slot; child can never
+ acquire one). The fix is the release/reacquire around `build_resolve` shown above —
+ the jobs semaphore bounds *actively running* recipes, not *in-flight* ones. In
+ sequential mode `jobs_*` are no-ops so this is free. This belongs in BUILD.md's
+ Parallelism section too.
+- `runner.c` includes `protocol.h`; `runner.h` doesn't need to (no protocol types in
+ its signatures). The leafset interning (so `out->leafset` lives for the process) is a
+ `coord` responsibility — add `build_coord_leafset_intern` (findings #7) rather than
+ leaking heap ownership into `runner`.
+
+---
+
+## L5 — `bundle` (signed trace sharing)
+
+**Role.** Export/import traces as **signed** `.kpkg` bundles, reusing the
+`<kit/package.h>` manifest + minisign + trust machinery wholesale. A trace is a
+*claim* (not self-verifying), so import is gated by signature/trust; the output bytes
+it references stay trustless (verified by tree-id/blob-id on use). Also pulls trace
+bundles from configured trace-remotes (the trusted counterpart to `remote`).
+
+**Depends on.** `coord` (L3), `store`/`trace` (via `coord`), `<kit/package.h>`
+(`kit_pkg_create`/`kit_pkg_verify`/`kit_minisig_*`/`kit_trust_*`), `<kit/build_coord.h>`.
+
+**Pseudocode.**
+
+```c
+int build_bundle_export(c, opts):
+ stage = make_temp_dir() // assemble the bundle payload tree
+ claims = []
+ for T in opts.targets:
+ rec = store_record_load(key(T), T)
+ for row in rec (newest candidates): copy build/trace/<row.trace_id> → stage/trace/<id>
+ parse the body → collect referenced config-id, argv-id (and output-tree for the claim row)
+ claims.push({T, row.kind, row.trace_id, output_tree})
+ for each referenced config-id/argv-id: kit_cas_get_blob → stage/cas/blob/<id> // required for replay
+ if opts.include_outputs: also copy referenced output trees + their blobs into stage/cas/
+ build_bundle_manifest_emit(claims, &w) → stage/manifest "kit-build-traces 1"
+ kit_pkg_create({ root_dir: stage, format: opts.format, sk: opts.sk, keyid: opts.keyid,
+ name:"traces", out_path: opts.out_path }) // signs the whole tree
+ return BUILD_OK
+
+int build_bundle_import(c, opts, result):
+ unpack = make_temp_dir()
+ kit_pkg_verify({ pkg_data, pkg_len, format, pubkey_bytes, trusted_keys, tofu, unpack_dir: unpack }, &vr)
+ if !verified: return BUILD_ERR // signature/trust failure
+ read unpack/manifest → build_bundle_manifest_parse → claims
+ for blob in unpack/cas/blob/*: kit_cas_add_blob(cas, bytes) // config/argv/(outputs), hash-checked
+ for c in claims:
+ install unpack/trace/<c.trace_id> via build_store_put_trace // id re-checked on store
+ build_store_record_update(store, key(c.target), c.target, c.kind, c.trace_id) // prepend
+ result.n_traces = |claims|; result.keyid = vr.keyid; result.tofu_pin = vr.tofu_pin; result.tofu_pk = vr.tofu_pk
+ return BUILD_OK
+
+int build_trace_remote_pull(c, T): // trusted fetch, mirrors remote.c
+ for tr in opts.trace_remotes:
+ out = tmp/"bundle"; argv = render(tr.fetch_argv_template, {target: T, out}); exec->spawn; exec->wait
+ if code != 0: continue
+ build_bundle_import(c, { pkg_data: read(out), trusted_keys: tr.trusted_keys, tofu: tr.tofu }, &res)
+ if ok: return BUILD_OK
+ return BUILD_ERR
+
+// build_bundle_manifest_emit/parse: canonical "kit-build-traces 1" rows
+// <target> <kind> <trace-id> <output-tree-id>, sorted, mirroring dist_manifest_*
+```
+
+**Interface notes.**
+- The "signed `kit-build-traces` manifest" is realized by packaging a staging *tree*
+ that contains the traces manifest file (plus bodies + blobs); `kit_pkg_create`'s own
+ `kit-package 3` signature covers that whole tree, so trusting the package signature
+ transitively trusts the `kit-build-traces` manifest. No new signing primitive is
+ needed. **No header gap** — the manifest emit/parse are pure (`bundle.h` already has
+ them).
+- Import correctness leans on the storage state machine: `build_store_put_trace`
+ re-derives and re-checks the trace-id, and `kit_cas_add_blob` re-derives blob ids, so
+ even a mis-built bundle cannot install a body under the wrong key.
+
+---
+
+## L6 — `src/api/build.c` (composition root)
+
+**Role.** Implements the public `<kit/build_coord.h>` coordinator + export/import entry
+points by calling the internal modules, mapping `BUILD_OK/ERR` (and the `err`
+buffers) to `KitStatus` + `ctx->diag`, and owning the sizing of every caller-provided
+section buffer (the trace/record/config growable arrays) so no inner module allocates
+policy. Gated by `KIT_BUILD_ENABLED`; a stubbed-out build compiles these to
+`KIT_UNSUPPORTED` like `config_stubs.c`.
+
+**Depends on.** Everything (it is the root). The recipe-side `kit_build_client_*` are
+implemented in `client.c` but composed here under the same `KIT_BUILD_ENABLED` gate.
+
+**Pseudocode.**
+
+```c
+KitStatus kit_build(coord, req, out):
+ BuildConfig cfg over stack/heap storage; for kv in req.config: build_config_set(cfg, kv.key, kv.value)
+ build_config_id(heap, &cfg, &cfg.id)
+ BuildArgv argv; build_argv_set(&argv, req.argv, req.argc); build_argv_id(heap, &argv, &argv.id)
+ reset coord per-build arena (chain frames + dispatched-worker tracking live here)
+ BuildResolved r; rc = build_resolve(coord, req.target, &cfg, &argv, /*chain=*/NULL, &r) // root: no parent
+ join any outstanding dispatched workers; if rc != OK: return build_status_from(coord->ctx, rc)
+ memcpy(out->output_tree, r.output_tree, 32); strcpy(out->path, r.path); return KIT_OK
+```
+
+`kit_build_traces_export/import` thinly wrap `build_bundle_export/import`;
+`kit_build_client_*` live in `client.c`.
+
+---
+
+## Concurrency model (cross-cutting)
+
+- **One host thread per in-flight resolution.** `runner` services a recipe's
+ connection with blocking transport reads on its thread; a fresh `(T,config,argv)`
+ is *dispatched* onto a `KitBuildSched` worker (`build_coord_spawn`). Parallelism
+ comes from two places now: **independent targets** (multi-target requests, Phase-2
+ dep re-resolution) *and* **a single recipe's fan-out** — `need-submit` dispatches
+ each dep concurrently, the recipe `need-await`s the tokens. The fused `need`
+ resolves inline (no worker) for the one-at-a-time case.
+- **`jobs` semaphore bounds actively-running recipes**, not in-flight ones. A recipe
+ blocked inside a `need`/`need-await` releases its slot and reacquires after
+ (findings #6), preventing chain-depth deadlock.
+- **Futures dedup shared targets.** `build_coord_target_intern` returns a shared
+ future for a repeated `(T, config-id, argv-id)`; concurrent submits/needs await
+ one resolution and spawn at most one worker. The memo and the futures are the
+ *same* table.
+- **Determinism under concurrency.** `need-await` takes a *specific* token (no
+ await-any), and a dep edge is recorded on await, so the awaited set — hence the
+ recorded deps and output — is a deterministic function of inputs, never of
+ completion order. The sorted `[dep]` section is byte-identical regardless of
+ submit/await order.
+- **Cycle detection is per-path**, on the parent-linked `BuildChainFrame` cactus
+ stack; the global memo handles dedup, the chain handles cycles. A shared dep
+ appears in many chains, never twice in one; concurrent siblings share a
+ read-only parent prefix.
+- **Sequential degrade.** `host->sched == NULL` ⇒ `jobs_limit = 1`, `jobs_*` and the
+ lock are no-ops, fresh futures complete inline, no thread is ever spawned — and
+ every module above is otherwise identical.
+
+---
+
+## Interface findings
+
+Concrete adjustments the pseudocode surfaced. **All applied** to the `src/build/*.h`
+spike, the public `<kit/build_coord.h>`, and `BUILD.md` as of this revision; the entries
+below are the rationale of record. None block starting the L0/L1 modules.
+
+1. **`client` module ownership.** The recipe-side `kit_build_client_*` need a real
+ translation unit (`src/build/client.c`) with no internal header — assigned an owner
+ in [Wave B](#suggested-parallel-implementation-order) below so it isn't mistaken for
+ part of `coord`. *(No header change; planning note only.)*
+2. **`build_run_recipe` prototype placement.** *(Applied — `resolve.h`, `runner.h`.)*
+ `resolve` ↔ `runner` is a deliberate mutual recursion. `build_run_recipe` is now
+ declared in `resolve.h` (the Phase-3 continuation) and implemented in `runner.c`;
+ `runner.h` carries only `build_runner_service` / `build_runner_record_traces`. Avoids
+ an awkward `resolve.h → runner.h` include.
+3. **Tree fetch in `remote`.** *(Applied — `remote.h` comment.)* The two-step tree
+ fetch (manifest bytes id-checked + installed, then per-blob fetch on demand during
+ `materialize`) is documented; the function signature is sufficient.
+4. **`build_glob_result_hash` in `build.h`.** *(Applied — `build.h`.)* Added as a pure
+ helper over a new foundation `BuildPathBlob` pair type, so the sole producer of the
+ `[glob]` value is separately unit-testable and byte-stable.
+5. **Transport listener (public header change).** *(Applied — `<kit/build_coord.h>`.)* Added
+ the per-recipe endpoint to `KitBuildTransport`:
+ `listen(user, char* name_out, size_t cap, KitBuildListener** out)`,
+ `accept(user, KitBuildListener*, KitBuildConn** out)`, `close_listener`. Concurrent
+ recipes each get a uniquely-named endpoint; the client `dial` side was already there.
+6. **Jobs slot released while blocked on `need`.** *(Applied — `coord.h`, `runner.h`,
+ `BUILD.md §Parallelism` + the `run_recipe` sketch.)* The semaphore bounds *actively
+ running* recipes; a recipe blocked on a `need` yields its slot, preventing a
+ `jobs < chain-depth` deadlock.
+7. **`build_coord_leafset_intern`.** *(Applied — `coord.h`.)* `runner`/`resolve` hand
+ back a process-lived `BuildLeafSet*` (`BuildResolved.leafset` is
+ borrowed-for-the-process); the interning entry point lives in `coord` rather than
+ leaking heap ownership upward.
+
+**Design-review decisions** (a second pass that pressed on correctness and seams;
+all applied to the spike headers, `<kit/build_coord.h>`, and `BUILD.md`):
+
+8. **Deep-trace soundness vs. recipe changes.** *(Applied — `trace.h`, `coord.h`,
+ `resolve.h`, `defn.h`.)* The flat deep trace checked source/glob leaves but never
+ recipe identity, so a recipe edit (T's own or any descendant's) with unchanged
+ sources produced a **stale** Phase-1 hit. Fix: `recipe-id` is a per-node scalar on
+ the deepset, refreshed by recomputing through the *live* definition. Content-only
+ (same-content repoint = no-op) and transitive (a descendant's recipe change busts
+ the root). The whole-definition source leaf is removed — only the resolved
+ recipe-id is a definition-derived input.
+9. **Structural deepset closure adopted now.** *(Applied — `trace.h` `BuildDeepSet`,
+ `coord.h` node + `deepsets` memo + `build_coord_deepset_load`, `resolve.h`
+ union/refresh, `build.h` `BUILD_DEEPSET_MAGIC`.)* The closure is a Merkle DAG of
+ `kit-build-deepset 1` nodes stored as **CAS blobs** (`deep-set-id` == blob-id —
+ self-verifying content, so `cas/`, not `build/`). The deep trace holds one
+ `deepset <id>` pointer. Refresh is O(changed frontier) via the validity memo.
+ `BuildLeafSet` is now that in-memory node (was a flat source/glob list).
+10. **Absent source leaf.** *(Applied — `trace.h` `BuildSourceLeaf.absent`,
+ protocol/runner.)* An absent `source` read is recorded (`<path> -`) so creating
+ the file later busts the trace — the source analogue of consumed-while-unset.
+11. **`materialize` is a pure locator.** *(Applied — `resolve.h`.)* Dropped
+ `target/cfg/argv/chain`; returns `BUILD_ERR` on a total miss and `resolve` falls
+ through to Phase 3. Removes the `materialize → runner` edge and the
+ rebuild-returns-a-different-tree hazard.
+12. **Lazy trace-remote pull.** *(Applied — `coord.h`
+ `build_coord_trace_remote_pull_once` + `pulled` set; `resolve.h` Phase-0 hook.)*
+ A clean-checkout record miss attempts one signed-bundle pull (idempotent per
+ target) before Phase 3, so shared traces can actually be hit.
+13. **Clean recipe environment + `env.*`.** *(Applied — `<kit/build_coord.h>` `KitBuildExec`
+ + `KIT_BUILD_ENV_PREFIX`, `runner.h`.)* `spawn`'s `env` is the recipe's COMPLETE
+ environment (host inherits nothing); declared env comes from tracked `env.`-prefixed
+ config, so it is hermetic and cache-visible.
+14. **Glob responses span frames.** *(Applied — `protocol.h` `BUILD_RESP_GLOB_END`.)*
+ A glob match set larger than one 64 KB frame streams as batch frames terminated by
+ an END frame, instead of silently truncating.
+15. **Future-based `need` (concurrent fan-out).** *(Applied — `<kit/build_coord.h>`
+ `KitBuildNeedToken` + `need_submit`/`need_await`; `protocol.h`
+ `NEED_SUBMIT`/`NEED_AWAIT` + token fields; `coord.h` `build_coord_spawn`;
+ `resolve.h` `build_dispatch` + cactus `BuildChainFrame`; `runner.h`
+ `BuildPendingNeed`.)* A recipe submits its fan-out (each `need-submit` →
+ dispatched on a worker, returns a token) then `need-await`s the tokens, so
+ independent deps build concurrently; the fused blocking `need` stays for the
+ one-at-a-time case. Three correctness constraints: the dep edge is logged on
+ **await** (un-awaited submits are speculative, cancelled at exit); await is
+ **token-specific** (no await-any) so the awaited set stays input-deterministic;
+ and the chain becomes a **parent-linked cactus stack** so concurrent siblings
+ extend a shared read-only prefix without copying. Sequential mode resolves
+ submit eagerly inline (transparent).
+
+- **Source reads stay live.** *(Decision, not a code change.)* `source` returns the
+ live workspace path (no staging); workspace immutability for a build's duration is
+ therefore load-bearing for *soundness*, documented in `BUILD.md §Determinism`.
+- **In-process recipes run on a `sched` thread.** *(Supported — the integration
+ test substrate; see [BUILD_TESTING.md](BUILD_TESTING.md).)* The framed protocol
+ needs the recipe to run concurrently with the service loop; an in-process recipe
+ gets that from a `KitBuildSched` thread. Only the *sequential* (`sched == NULL`)
+ in-process case is deferred (it would need a direct-dispatch bypass).
+- **`KitBuildStats` observability.** *(Applied — `<kit/build_coord.h>`, `coord.h`.)* Six
+ cumulative counters (deep/shallow/recipe, materialize-miss, object-fetch,
+ trace-pull) on the coordinator, bumped under `lock` by resolve/runner/remote/
+ bundle and read via `kit_build_stats`. Tests snapshot-and-diff to assert which
+ path served each request; doubles as user build introspection.
+
+---
+
+## Suggested parallel implementation order
+
+The DAG makes the seams obvious; teams can take whole layers independently behind the
+spike headers.
+
+- **Wave A (no deps, fully unit-testable now):** `build`, `cfg`, `defn`, `trace`,
+ `protocol`. Each gets a `test/build/` corpus mirroring `test/dist` (round-trip
+ emit→parse, non-canonical-input rejection, id stability). Resolve findings #4 first.
+- **Wave B (on Wave A):** `store` (state-machine tests with a fake `KitBuildStoreIo`),
+ `remote` (fake `KitBuildExec` returning good/corrupt bytes), `client` (loopback
+ in-process transport against a scripted server).
+- **Wave C (on B):** `coord` (memo + future + sequential-mode tests with
+ `sched == NULL`). Settle findings #5/#7 here.
+- **Wave D (on C):** `resolve` then `runner` together (they share the recursion seam,
+ findings #2/#6) and `bundle`. End-to-end smoke via the in-process transport: cold
+ build → no-op rebuild → comment-edit (Phase 2 hit) → config flip — exactly the
+ [worked example](BUILD.md#worked-example).
+- **Wave E:** `src/api/build.c` composition + the `kit build` driver command +
+ `KIT_BUILD_ENABLED` gating, then the Makefile test targets.
diff --git a/doc/plan/BUILD_TESTING.md b/doc/plan/BUILD_TESTING.md
@@ -0,0 +1,192 @@
+# Build coordinator — testing strategy
+
+> **Status: design, not yet built.** Companion to [BUILD.md](BUILD.md) (the
+> design) and [BUILD_INTERNALS.md](BUILD_INTERNALS.md) (the module breakdown).
+> This doc specifies *how the subsystem is tested* — the shared harness, the
+> per-module plan, the integration scenarios, and the cross-cutting properties
+> (byte-stability, fail-safe parsing, minimal-rebuild, crash-safety,
+> concurrency, determinism). It mirrors how the layer below is tested: the dist
+> subsystem has no `test/dist`; it is covered by `test/cas` + `test/pkg` (per
+> public surface) plus INTERNAL unit tests. `kit/build_coord.h` gets `test/build`.
+
+## Why the subsystem is testable by construction
+
+Two properties — both deliberate — carry the plan:
+
+1. **L0–L1 are pure** (no I/O): `build`, `cfg`, `defn`, `trace`, `protocol`,
+ `client` are byte/value transforms. They get straight INTERNAL unit tests and
+ golden corpus, exactly like the ISA/DWARF units.
+2. **Every side effect is a host vtable**: `KitBuildStoreIo`, `KitBuildExec`,
+ `KitBuildTransport`, `KitBuildSched`, and `KitCasHost`. So every L2+ test
+ injects a fake and stays hermetic and deterministic — no real sockets,
+ processes, clock, or network, and no flakiness.
+
+Guiding principle: **test what ships.** The integration harness drives the *real*
+protocol codec, service loop, runner, resolver, and coordinator; only the OS
+socket and process spawn are stubbed (see [Integration](#integration)).
+
+## Repo conventions this plugs into
+
+- **C unit tests** use `test/lib/kit_unit.h` (stack-resident `KitUnit`, `CU_CHECK`
+ /`CU_EXPECT`, no globals), registered in `test/lib/unit.mk` in one of two
+ regimes:
+ - **INTERNAL** (`-Isrc`, links `LIB_OBJS`): for modules with no public API —
+ `build`, `cfg`, `defn`, `trace`, `protocol`, `store`, `remote`, `coord`,
+ `resolve`, `runner`. They call internal symbols (`build_shallow_emit`, …).
+ - **PUBLIC** (`-Iinclude`, links `LIB_AR`): for the public surface —
+ `client`, `bundle`, and the `kit_build*` coordinator API. Proves the public
+ header is self-sufficient.
+- **Driver / end-to-end** is a `run.sh` driven through the `kit` binary with
+ `test/lib/kit_sh_assert.sh`, exactly like `test/cas/run.sh` / `test/pkg/run.sh`,
+ wired as a `test-build-driver` target in `test/test.mk`.
+- **Granular targets.** Per the project rule (prefer targeted runs), every module
+ gets its own `test-build-<module>`, aggregated by `test-build`. A failing
+ resolver test never forces a full-suite rerun.
+
+## Shared harness — `test/build/lib/`
+
+Written once, reused everywhere. The harness *is* the test investment.
+
+| Fake | Backs | Capability |
+|------|-------|-----------|
+| `FakeStoreIo` | `KitBuildStoreIo` | rename/remove/temp/sync/lock over a real temp dir, **plus fault injection**: fail or half-apply the *N*th op, to drive crash-safety |
+| `FakeExec` | `KitBuildExec` | runs **scripted recipes** (`target → {sources, needs, config-gets, env reads, output files, exit code}`) and **counts invocations per target**; a corrupt-bytes mode backs remote-fetch |
+| `FakeWorkspace` | `KitCasHost.file_io` | an in-memory file set a test **mutates between builds** (edit / delete / create) to drive invalidation; the source of `source`/`glob` bytes |
+| `ipc_transport` | `KitBuildTransport` | **in-memory framed transport** over cond-var queues; the recipe callback runs on a `KitBuildSched` thread (the *real* protocol path) |
+| `thread_sched` | `KitBuildSched` | real pthread-backed threads/mutex/cond for integration + concurrency; a `NULL` sched exercises sequential degrade |
+| `count_source` | (a `FakeWorkspace` view) | counts rehashes/reglobs so refresh tests assert a **shared subtree is checked once** |
+
+Every integration test wraps its `kit_build` in a **watchdog timeout** (a
+deadlock-bug must fail, not wedge CI) — the one cost of the framed-over-thread
+harness (decision below).
+
+## Per-module plan
+
+| Layer | Module | Regime | Faked | Key cases |
+|---|---|---|---|---|
+| L0 | `build` | INTERNAL | — | `build_target_key` / `build_glob_result_hash` id-stability vectors; `build_id_eq` |
+| L1 | `cfg` | INTERNAL | — | set/overlay/get; canonical sort+dedup; empty-argv fixed id; emit→parse→emit identity; bad-body rejection |
+| L1 | `trace` | INTERNAL | — | shallow/deep/**deepset**/record emit↔parse; **absent leaf** (`<path> -`); record MRU dedup+cap; byte-stable golden + known id |
+| L1 | `defn` | INTERNAL | — | parse + `find`; sorted-unique enforcement; **no whole-file dep** (find(T) stable under unrelated edits) |
+| L1 | `protocol` | INTERNAL | — | req/resp encode↔decode; `need` overlay+argv; **multi-frame glob + `GLOB_END`**; frame-bound + malformed rejection |
+| L1 | `client` | PUBLIC | loopback | each `kit_build_client_*` round-trips against a scripted server |
+| L2 | `store` | INTERNAL | FakeStoreIo + real CAS | put/get idempotence; record RMW dedup+cap; cache materialize verifies; **deepset stored as a CAS blob**; **crash-safety** (below) |
+| L2 | `remote` | INTERNAL | FakeExec | verified install; **corrupt mirror discarded**; remotes tried in order; tree=manifest-then-blobs; deepset via `{kind}=blob` |
+| L3 | `coord` | INTERNAL | all | memo (file hashed once); **futures dedup**; **sequential degrade** (`sched==NULL`); `deepset_load` interns+recurses, absent→ERR; `pull_once` idempotent; **stat counters** |
+| L4 | `resolve` | INTERNAL/PUBLIC | ipc + FakeExec | the **scenario spine** + soundness scenarios |
+| L5 | `runner` | INTERNAL | ipc + FakeExec | service loop logs every dep; nonzero exit → **no trace**; both traces + deepset written; **jobs slot released across `need`** (chain deeper than `jobs` completes) |
+| L5 | `bundle` | PUBLIC | real pkg | export→import round-trip; **untrusted signer rejected**; tofu pin; deepset + config/argv blobs present so an imported trace refreshes/replays |
+| L6 | `api` + driver | PUBLIC + `run.sh` | ipc / real | `kit build` end-to-end; `kit_build_stats` deltas |
+
+## Integration
+
+### The scenario spine
+
+The [worked example](BUILD.md#worked-example) is the backbone. Each step asserts
+**(output tree-id, `KitBuildStats` delta)** — the stats diff names the path taken:
+
+1. **cold** → `recipes_run += 2`;
+2. **no-op rebuild** → `deep_hits += 1`, `recipes_run += 0`;
+3. **comment-edit a dep source** → dep `recipes_run += 1`, parent served by
+ `shallow_hits` (Phase-2 output-compare), parent recipe *not* run;
+4. **flip a consumed config key** → only the consuming recipe runs.
+
+### Soundness scenarios (the regression suite for the design review)
+
+These are the tests that would have caught the bugs found while pressing on the
+design — each maps to a [finding](BUILD_INTERNALS.md#interface-findings):
+
+- **recipe edit at depth** (#8): edit `//lib:core`'s recipe, sources unchanged →
+ `//app` **rebuilds** (the original deep trace would have stale-hit).
+- **repoint** (#8): same-content recipe → no rebuild; different-content → rebuild;
+ unrelated stanza edit → **nothing busts**.
+- **absent → create** (#10): `source` a missing file, then create it → busts.
+- **unconsumed config change**: subtree that ignores it is **not** re-run.
+- **deepset sharing/refresh** (#9): a diamond yields **one** shared child node
+ (assert via `count_source`: rehashed once); an absent deepset blob → Phase 1
+ falls through (fail-safe), not a wrong answer.
+- **materialize miss** (#11): evict a hit's output bytes → `materialize_misses`
+ bumps and the recipe re-runs (no stale tree-id returned).
+- **verify mode**: a recipe that emits a counter → flagged on the re-run compare.
+- **concurrent fan-out** (#15): a recipe `need-submit`s N deps then `need-await`s
+ them. Assert each distinct dep built **once** (futures dedup via
+ `KitBuildStats`/`FakeExec` counts), the recorded `[dep]` set is **identical**
+ whether the deps complete in order or reverse order (drive completion order via a
+ scripted-recipe delay in `FakeExec` — determinism must not depend on it), and a
+ submitted-but-never-awaited need contributes **no** edge and is cancelled. A
+ cycle introduced through `need-submit` fails fast at submit; `jobs = 1` with a
+ wide fan-out still completes (await releases the slot).
+
+### Tiers
+
+- **Framed-over-thread (primary).** The `ipc_transport` harness runs the real
+ codec + service loop + runner + resolver + coordinator with the recipe on a
+ sched thread. Fast, hermetic, deterministic *outcomes* (not interleavings).
+ Carries the scenario + soundness suites.
+- **Subprocess + real socket (smoke).** A small `run.sh` through `kit build` with
+ real shell recipes and a real unix-socket transport, per platform — proves the
+ real transport/exec/spawn that the framed harness stubs. The fidelity backstop.
+
+## Cross-cutting properties
+
+- **Byte-stability across platforms.** Golden trace/deepset/config/record vectors
+ are checked in; CI on every platform must reproduce identical bytes (a
+ divergence silently breaks trace sharing). Paired with **round-trip fuzz**
+ (`kit_differential.sh` style): random valid structures satisfy emit→parse→emit
+ identity; random bytes either reject or re-emit identically (canonical
+ idempotence) — the fail-safe property at scale, and the security guard that a
+ parse error is *never* a match.
+- **Minimal-rebuild is the invariant.** Every scenario asserts the recipe-run
+ *count* (via `FakeExec` / `KitBuildStats`), not just the output — a cache that's
+ correct but re-runs everything is the failure mode that matters here.
+- **Crash-safety** (decision: fault injection). `FakeStoreIo` aborts the sequence
+ before/after each step; the test reopens the store and asserts the invariants:
+ content is **complete-or-absent** under its key (never torn — only orphaned
+ `tmp/`); the record is **prior-or-next**, never torn; a missing record entry
+ re-derives. The vtable lets us abort between *any* two FS effects, which is
+ exactly the granularity the state machine reasons about.
+- **Concurrency** (real threads + **ThreadSanitizer**). A wide diamond (many
+ parents, one shared dep): assert the dep's recipe runs **exactly once**
+ (futures dedup) and that `jobs = 1` with a deep chain **completes** (the
+ release-slot-across-`need` rule; no deadlock). A deterministic cooperative
+ fake-sched for specific interleavings is a stretch goal, not v1.
+- **Sequential degrade.** The same scenario suite runs with `sched == NULL`:
+ `jobs_*` no-op, futures complete inline, no thread spawned — identical outputs
+ and stats. (In-process *recipes* still need a thread, so the sequential suite
+ uses subprocess or pre-seeded traces; see BUILD.md §Deferred.)
+
+## Observability — `KitBuildStats`
+
+Six cumulative counters on the coordinator (`deep_hits`, `shallow_hits`,
+`recipes_run`, `materialize_misses`, `object_fetches`, `trace_pulls`), bumped
+under the coordinator lock and read via `kit_build_stats`. Tests **snapshot
+before / diff after** each `kit_build` to assert which path served the request
+and how much work ran — turning "did the recipe run?" and "deep or shallow hit?"
+into exact assertions. The same surface is user-facing build introspection.
+
+## Decisions (this round)
+
+- **Integration harness = framed over a thread.** The in-process transport runs
+ the recipe on a `KitBuildSched` thread and exercises the real protocol/service
+ loop — tests what ships, at the cost of per-test watchdog timeouts. Direct-
+ dispatch (synchronous, sequential) is deferred unless in-process recipes become
+ a product feature.
+- **Observability = `KitBuildStats` counters** (public, snapshot-and-diff) rather
+ than inferring from recipe-run counts alone.
+- **Crash-safety = fault-injecting `FakeStoreIo`** (deterministic, in-process); a
+ real fork+kill harness is deferred.
+
+## Map to the implementation waves
+
+Tests land with their modules ([Wave order](BUILD_INTERNALS.md#suggested-parallel-implementation-order)):
+
+- **Wave A** (`build`,`cfg`,`defn`,`trace`,`protocol`): unit + golden + fuzz; no
+ harness needed. The byte-stability and fail-safe-parse corpus starts here.
+- **Wave B** (`store`,`remote`,`client`): `FakeStoreIo` (+ fault injection),
+ `FakeExec`, loopback transport land here.
+- **Wave C** (`coord`): memo / futures / sequential-degrade / stats tests; the
+ `thread_sched` and `FakeWorkspace` fakes land here.
+- **Wave D** (`resolve`,`runner`,`bundle`): the `ipc_transport` harness, the
+ scenario spine, and the soundness suite — the bulk of the value.
+- **Wave E** (`api` + driver): the `run.sh` subprocess smoke + `test-build`
+ aggregation in `test.mk`/`unit.mk`.
diff --git a/include/kit/build_coord.h b/include/kit/build_coord.h
@@ -0,0 +1,428 @@
+#ifndef KIT_PUBLIC_BUILD_COORD_H
+#define KIT_PUBLIC_BUILD_COORD_H
+
+#include <kit/cas.h>
+#include <kit/core.h>
+#include <kit/package.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/*
+ * Content-addressed build coordinator. A long-lived object that turns a build
+ * request (target T under configuration C) into a materialized output tree on
+ * disk, doing the least work necessary by caching every recipe result keyed by
+ * the exact set of inputs that produced it. See doc/plan/BUILD.md.
+ *
+ * Layering. The coordinator is built entirely on the public kit surface: the
+ * content store (<kit/cas.h>) holds source bytes, output trees, config maps,
+ * argv vectors, and trace bodies; signed packages (<kit/package.h>) carry
+ * shared trace bundles; digests (<kit/hash.h>) key the mutable index. The only
+ * mutable state the coordinator adds over the CAS is a per-target trace index.
+ *
+ * Side effects via the host. As elsewhere in kit, the library sources no
+ * entropy and performs no I/O, process control, or concurrency itself: every
+ * such capability arrives through the KitBuildHost vtables below. The library
+ * defines the recipe command set, the trace model, and the resolution
+ * algorithm; the host supplies sockets/pipes, process spawning, filesystem
+ * atomicity, and (optionally) threads. Operational failures return a KitStatus
+ * and emit detail through ctx->diag.
+ *
+ * Strings. Borrowed text crosses this API as KitSlice (the core fat-pointer
+ * type), never as a bare char*. Output paths are returned in caller-owned fixed
+ * buffers (KitBuildResult), as elsewhere in the public API.
+ */
+
+#define KIT_BUILD_HASH_LEN KIT_CAS_HASH_LEN
+#define KIT_BUILD_PATH_MAX 1024u
+
+/* Environment a recipe is launched with (see KitBuildExec). The coordinator
+ * creates a per-recipe endpoint (KitBuildTransport.listen) and passes its name
+ * in KIT_BUILD_SOCK; the recipe dials that name back to the coordinator and
+ * writes its output tree under KIT_BUILD_OUT. KIT_BUILD_TARGET names the target
+ * being built. */
+#define KIT_BUILD_ENV_SOCK "KIT_BUILD_SOCK"
+#define KIT_BUILD_ENV_OUT "KIT_BUILD_OUT"
+#define KIT_BUILD_ENV_TARGET "KIT_BUILD_TARGET"
+
+/* Propagated-config keys under this prefix declare the recipe's pass-through
+ * environment: a key "env.PATH" sets $PATH (to its config value) in the recipe's
+ * otherwise-clean environment. Because they are ordinary tracked config, the
+ * declared environment is hermetic and cache-visible. KIT_BUILD_* are always
+ * injected and are not drawn from config. */
+#define KIT_BUILD_ENV_PREFIX "env."
+
+typedef struct KitBuildCoordinator KitBuildCoordinator;
+
+/* One key=value pair: a propagated-config entry, a `need` overlay, or a recipe
+ * environment variable. */
+typedef struct KitBuildKV {
+ KitSlice key;
+ KitSlice value;
+} KitBuildKV;
+
+/* ------------------------------------------------------------------ *
+ * Host vtables
+ * ------------------------------------------------------------------ */
+
+/* One recipe<->coordinator connection. Opaque; created by the transport. */
+typedef struct KitBuildConn KitBuildConn;
+
+/* A server-side listening endpoint, one per in-flight recipe. Opaque; created
+ * by KitBuildTransport.listen. Concurrent recipes each get a uniquely-named
+ * endpoint so accept() is unambiguous. */
+typedef struct KitBuildListener KitBuildListener;
+
+/* The framed request/response transport between the coordinator (server) and a
+ * recipe (client). The library defines the length-prefixed framing and the
+ * command set; the host supplies the carrier. Defaults: a unix socket or
+ * anonymous pipe pair, a Windows named pipe, or an in-process callback table.
+ * The command set is identical across all of them. Each call returns 0 on
+ * success, non-zero on failure.
+ *
+ * Server lifecycle, per recipe: `listen` creates a fresh endpoint and reports
+ * its name (the coordinator puts that name in the child's $KIT_BUILD_SOCK);
+ * `accept` waits for the recipe to dial in; the connection is serviced; then
+ * `close` drops the connection and `close_listener` tears down the endpoint.
+ * Per-recipe endpoints are what let many recipes run concurrently. */
+typedef struct KitBuildTransport {
+ /* Server: create a uniquely-named endpoint for one recipe. The name to hand
+ * the child (via $KIT_BUILD_SOCK) is written NUL-terminated into name_out
+ * (cap bytes). */
+ int (*listen)(void* user, char* name_out, size_t cap, KitBuildListener** out);
+ /* Server: accept the recipe's one connection on `lst`. */
+ int (*accept)(void* user, KitBuildListener* lst, KitBuildConn** out);
+ /* Server: tear down a listener once its connection is closed. */
+ void (*close_listener)(void* user, KitBuildListener* lst);
+ /* Client: connect to the endpoint named in $KIT_BUILD_SOCK. */
+ int (*dial)(void* user, KitSlice endpoint, KitBuildConn** out);
+ int (*read_frame)(void* user, KitBuildConn*, uint8_t* buf, size_t cap,
+ size_t* n);
+ int (*write_frame)(void* user, KitBuildConn*, const uint8_t* buf, size_t n);
+ void (*close)(void* user, KitBuildConn*);
+ void* user;
+} KitBuildTransport;
+
+/* A spawned host process (recipe or fetch recipe). Opaque. */
+typedef struct KitBuildProc KitBuildProc;
+
+/* Process control for recipes and the remote fetch recipe. `argv` is the
+ * process argument vector; `env` is the process's COMPLETE environment — the
+ * host sets exactly these pairs (formatting each as "KEY=VALUE") and does NOT
+ * inherit its own ambient environment. A clean env is what makes a recipe's
+ * inputs complete: ambient PATH/locale/etc. would be undeclared inputs that
+ * break determinism and cross-machine trace reuse. The coordinator builds the
+ * set from KIT_BUILD_* plus the build's declared `env.`-prefixed config
+ * (KIT_BUILD_ENV_PREFIX), which are tracked, so the environment is cache-
+ * visible. Each spawn counts as one job against the parallelism limit. */
+typedef struct KitBuildExec {
+ int (*spawn)(void* user, const KitSlice* argv, size_t argc,
+ const KitBuildKV* env, size_t nenv, KitSlice cwd,
+ KitBuildProc** out);
+ /* Block until `proc` exits; *exit_code receives its status; frees `proc`. */
+ int (*wait)(void* user, KitBuildProc* proc, int* exit_code);
+ /* Best-effort terminate (cancel on cycle/abort); `wait` still reaps. */
+ void (*kill)(void* user, KitBuildProc* proc);
+ void* user;
+} KitBuildExec;
+
+/* Reports one directory entry name during a store sweep. Return non-zero to
+ * abort the walk. */
+typedef int (*KitBuildDirFn)(void* user, KitSlice name);
+
+/* An advisory per-target-key lock around the record read-modify-write.
+ * Optional; opaque. */
+typedef struct KitBuildLock KitBuildLock;
+
+/* Filesystem services the build store needs beyond KitCasHost: atomic rename,
+ * removal, temp-dir staging, durability (fsync), directory listing for GC, and
+ * an optional advisory lock. Reads and writes still go through the CAS host's
+ * KitFileIO; these are the extras that make the storage state machine
+ * crash-safe. Each returns 0 on success unless noted. Output paths are written
+ * into the caller's buffer (out, cap). */
+typedef struct KitBuildStoreIo {
+ int (*rename)(void* user, KitSlice from, KitSlice to); /* atomic */
+ int (*remove)(void* user, KitSlice path, int recursive);
+ int (*make_temp_dir)(void* user, KitSlice parent, char* out, size_t cap);
+ int (*sync_path)(void* user, KitSlice path); /* fsync file or dir */
+ int (*list_dir)(void* user, KitSlice path, KitBuildDirFn cb,
+ void* cb_user); /* optional (GC); NULL if absent */
+ int (*lock)(void* user, KitSlice key, KitBuildLock** out); /* optional */
+ void (*unlock)(void* user, KitBuildLock*);
+ void* user;
+} KitBuildStoreIo;
+
+/* Opaque concurrency primitives. */
+typedef struct KitBuildThread KitBuildThread;
+typedef struct KitBuildMutex KitBuildMutex;
+typedef struct KitBuildCond KitBuildCond;
+
+/* Optional concurrency substrate. The coordinator owns all parallelism (the
+ * jobs semaphore and the per-target futures); it implements them on these
+ * primitives, running one host thread per in-flight recipe so blocking
+ * transport reads compose with cross-target dedup. When `sched` is NULL on the
+ * host, the coordinator forces jobs=1 and resolves sequentially: futures
+ * degrade to a plain memo and no thread is ever spawned. */
+typedef struct KitBuildSched {
+ int (*thread_spawn)(void* user, void (*fn)(void*), void* arg,
+ KitBuildThread** out);
+ int (*thread_join)(void* user, KitBuildThread*);
+ KitBuildMutex* (*mutex_new)(void* user);
+ void (*mutex_lock)(void* user, KitBuildMutex*);
+ void (*mutex_unlock)(void* user, KitBuildMutex*);
+ void (*mutex_free)(void* user, KitBuildMutex*);
+ KitBuildCond* (*cond_new)(void* user);
+ void (*cond_wait)(void* user, KitBuildCond*, KitBuildMutex*);
+ void (*cond_broadcast)(void* user, KitBuildCond*);
+ void (*cond_free)(void* user, KitBuildCond*);
+ void* user;
+} KitBuildSched;
+
+/* Everything the coordinator needs from the host. `cas_host` and `store_io`
+ * are required; `exec` and `transport` are required to run recipes (a
+ * cache-only coordinator may omit them); `sched` is optional (see above). */
+typedef struct KitBuildHost {
+ const KitCasHost* cas_host;
+ const KitBuildStoreIo* store_io;
+ const KitBuildExec* exec;
+ const KitBuildTransport* transport;
+ const KitBuildSched* sched;
+ void* user;
+} KitBuildHost;
+
+/* ------------------------------------------------------------------ *
+ * Remotes (content fetch and trace sharing)
+ * ------------------------------------------------------------------ */
+
+/* An untrusted object source (the trustless rung of the materialization
+ * ladder). When a cache hit's bytes are absent locally, the coordinator renders
+ * `fetch_argv_template` and runs it via the exec host to fetch one object into
+ * a temp file, then VERIFIES the bytes against the requested content id before
+ * installing them in the CAS — so a corrupt or malicious mirror fails the hash
+ * check. Tokens: {kind} (blob|tree), {pp}, {id}, {out} (destination path).
+ * Multiple may be configured; the coordinator tries them in order. */
+typedef struct KitBuildObjectRemote {
+ KitSlice fetch_argv_template;
+} KitBuildObjectRemote;
+
+/* A trusted source of shared traces. A trace is a CLAIM, not self-verifying
+ * content, so importing one is gated by signature/trust (the <kit/package.h>
+ * model): `fetch_argv_template` pulls a signed trace bundle, and `trusted_keys`
+ * (the trusted-keys file bytes) or `tofu` anchors who may sign it. Tokens:
+ * {target}, {out}. Multiple may be configured; tried in order. */
+typedef struct KitBuildTraceRemote {
+ KitSlice fetch_argv_template;
+ KitSlice trusted_keys; /* trusted-keys file bytes, or empty */
+ int tofu; /* trust-on-first-use */
+} KitBuildTraceRemote;
+
+/* ------------------------------------------------------------------ *
+ * Coordinator lifecycle and requests
+ * ------------------------------------------------------------------ */
+
+typedef struct KitBuildOptions {
+ KitSlice workspace_root; /* recipe cwd and source root */
+ KitSlice build_def_path; /* build-definition file (tracked as a source) */
+ int jobs; /* max concurrent recipes; <=0 => host default */
+ int verify; /* verify mode: re-run cache hits and compare */
+ /* Untrusted content mirrors, tried in order (trustless, hash-verified). */
+ const KitBuildObjectRemote* object_remotes;
+ size_t n_object_remotes;
+ /* Trusted trace sources, tried in order (signature/trust-gated). */
+ const KitBuildTraceRemote* trace_remotes;
+ size_t n_trace_remotes;
+} KitBuildOptions;
+
+/* Open a coordinator over a build store rooted at `store_root` (which contains
+ * or sits beside a `cas/`). `host` and `opts` are borrowed for the handle's
+ * lifetime; a private copy of `ctx` is retained. */
+KIT_API KitStatus kit_build_coordinator_open(const KitContext* ctx,
+ const KitBuildHost* host,
+ KitSlice store_root,
+ const KitBuildOptions* opts,
+ KitBuildCoordinator** out);
+KIT_API void kit_build_coordinator_close(KitBuildCoordinator*);
+
+/* A build request: a target, a propagated-configuration map, and an optional
+ * local argv. The resolution identity is the triple (target, config-id,
+ * argv-id): two requests for the same target that differ in propagated config
+ * OR in argv are distinct builds. Used by both kit_build (the top-level
+ * request, where `config` SEEDS the propagated map) and kit_build_client_need
+ * (a sub-build, where `config` OVERLAYS the inherited propagated map).
+ *
+ * argv is the target's LOCAL config: visible only to that one recipe (delivered
+ * as its process argv) and never propagated to deps. It is NOT encoded in the
+ * target name, and the build definition does NOT carry argv — it is supplied
+ * entirely by the request. `argv == NULL` (or argc == 0) is the EMPTY argv, the
+ * default when none is given; otherwise the supplied vector is the argv. Either
+ * way the argv is content-addressed to an argv-id, the third dimension of the
+ * key — so a different argv is a genuinely distinct build. Only PROPAGATED
+ * configuration travels in `config`. */
+typedef struct KitBuildRequest {
+ KitSlice target; /* e.g. "//app:server" */
+ const KitBuildKV*
+ config; /* propagated config (seed at top level, overlay in need) */
+ size_t nconfig;
+ const KitSlice* argv; /* local argv; NULL or argc==0 => empty argv */
+ size_t argc;
+} KitBuildRequest;
+
+typedef struct KitBuildResult {
+ uint8_t output_tree[KIT_BUILD_HASH_LEN];
+ char path[KIT_BUILD_PATH_MAX]; /* materialized output directory */
+} KitBuildResult;
+
+/* Resolve `req->target` under the supplied propagated config, returning its
+ * output tree id and a materialized directory path. Runs the least work
+ * necessary: a deep/shallow cache hit returns without re-running the recipe; a
+ * miss runs it. */
+KIT_API KitStatus kit_build(KitBuildCoordinator*, const KitBuildRequest* req,
+ KitBuildResult* out);
+
+/* Cumulative resolution counters since the coordinator was opened — build
+ * introspection for users and the assertion surface for tests (which snapshot
+ * before a kit_build and diff after to assert *which* path served a request and
+ * how much work ran). Monotonic; updated under the coordinator's lock. */
+typedef struct KitBuildStats {
+ uint64_t deep_hits; /* Phase 1: deepset refresh matched */
+ uint64_t shallow_hits; /* Phase 2: dep outputs matched, recipe skipped */
+ uint64_t recipes_run; /* Phase 3: a recipe actually executed */
+ uint64_t materialize_misses; /* a cache hit whose bytes were gone everywhere */
+ uint64_t object_fetches; /* objects pulled + verified from an object-remote */
+ uint64_t trace_pulls; /* signed trace bundles imported from a trace-remote */
+} KitBuildStats;
+
+KIT_API void kit_build_stats(const KitBuildCoordinator*, KitBuildStats* out);
+
+/* ------------------------------------------------------------------ *
+ * Shared traces (signed bundles)
+ * ------------------------------------------------------------------ */
+
+typedef struct KitBuildExportOptions {
+ const KitSlice* targets; /* targets whose recorded traces to bundle */
+ size_t ntargets;
+ int include_outputs; /* also pack referenced output trees/blobs */
+ KitPkgFormat format; /* KPKG or TARGZ (not AUTO) */
+ const uint8_t* sk; /* minisign signing secret key (KIT_PKG_SK_LEN) */
+ const uint8_t* keyid; /* signing key id (KIT_PKG_KEYID_LEN) */
+ KitSlice out_path;
+} KitBuildExportOptions;
+
+/* Bundle the named targets' trace bodies (plus the serialized config-map and
+ * argv blobs they reference, required for replay, and optionally the referenced
+ * output trees/blobs) into a signed `.kpkg`, using the <kit/package.h> trust
+ * model. */
+KIT_API KitStatus kit_build_traces_export(KitBuildCoordinator*,
+ const KitBuildExportOptions*);
+
+typedef struct KitBuildImportOptions {
+ const uint8_t* pkg_data;
+ size_t pkg_len;
+ KitPkgFormat format;
+ const uint8_t* pubkey_bytes; /* -p key file content, or NULL */
+ size_t pubkey_len;
+ int tofu; /* trust-on-first-use */
+ const uint8_t* trusted_keys; /* trusted-keys file content, or NULL */
+ size_t trusted_keys_len;
+} KitBuildImportOptions;
+
+typedef struct KitBuildImportResult {
+ uint32_t n_traces;
+ uint8_t keyid[KIT_PKG_KEYID_LEN]; /* signer */
+ int tofu_pin; /* 1 -> caller should persist tofu_pk */
+ uint8_t tofu_pk[KIT_PKG_PK_LEN];
+} KitBuildImportResult;
+
+/* Verify a trace bundle's signature against the trust inputs, then install its
+ * trace bodies into the store and prepend them to the relevant target records;
+ * referenced config/argv blobs (and any packed outputs) land in the CAS,
+ * hash-verified on use. */
+KIT_API KitStatus kit_build_traces_import(KitBuildCoordinator*,
+ const KitBuildImportOptions*,
+ KitBuildImportResult*);
+
+/* ------------------------------------------------------------------ *
+ * Recipe-side client
+ * ------------------------------------------------------------------ */
+
+/* The client a recipe uses to talk back to the coordinator. Every call both
+ * returns a value and logs a dependency: an input the recipe reads but does not
+ * request through one of these is invisible to the cache. A C recipe links this
+ * directly; a shell recipe reaches it through a small `kit` helper subcommand;
+ * an in-process recipe uses the in-process transport. */
+typedef struct KitBuildClient KitBuildClient;
+
+/* Open a client over `transport`, dialing the endpoint in $KIT_BUILD_SOCK. */
+KIT_API KitStatus kit_build_client_open(const KitContext*,
+ const KitBuildTransport* transport,
+ KitBuildClient** out);
+KIT_API void kit_build_client_close(KitBuildClient*);
+
+/* Read a propagated config value (logs a config dep on `key`). On return
+ * *present is 0 when the key is unset; *value is borrowed until the next
+ * client call. */
+KIT_API KitStatus kit_build_client_config_get(KitBuildClient*, KitSlice key,
+ KitSlice* value, int* present);
+
+/* Resolve a source path to its pinned blob id and a readable path (logs a
+ * source dep). *realpath points into the LIVE workspace (not a staged snapshot),
+ * so the recipe must read it without the workspace changing underneath: the
+ * coordinator's caching is sound only while the workspace is immutable for the
+ * build's duration. An absent path is reported via the protocol as an absent
+ * source dep (recorded, so creating it later busts the trace). *realpath is
+ * borrowed until the next client call. */
+KIT_API KitStatus kit_build_client_source(KitBuildClient*, KitSlice path,
+ uint8_t blob[KIT_BUILD_HASH_LEN],
+ KitSlice* realpath);
+
+/* Report one matched path during a glob expansion. Return non-zero to stop. */
+typedef int (*KitBuildGlobFn)(void* user, KitSlice path);
+
+/* Expand `pattern`, invoking `cb` for each sorted match (logs a glob dep
+ * covering the whole match set's existence and content). */
+KIT_API KitStatus kit_build_client_glob(KitBuildClient*, KitSlice pattern,
+ KitBuildGlobFn cb, void* cb_user);
+
+/* Build a dependency target and block for its result (the one-shot form).
+ * Mirrors kit_build: `req` names the dep and carries the propagated-config
+ * OVERLAY for that sub-build (req->config overlays the inherited config; local
+ * argv never propagates), and *out receives the dep's output tree id and a
+ * materialized path. Logs a target-dep edge. Equivalent to need_submit followed
+ * immediately by need_await, but resolved on the coordinator's service thread
+ * without a worker hand-off — cheapest for the common one-dep-at-a-time pattern. */
+KIT_API KitStatus kit_build_client_need(KitBuildClient*,
+ const KitBuildRequest* req,
+ KitBuildResult* out);
+
+/* A handle to a submitted-but-not-yet-awaited need. A by-value token (no
+ * allocation, hold an array of them across a fan-out); carries a
+ * coordinator-assigned id. */
+typedef struct KitBuildNeedToken {
+ uint64_t id;
+} KitBuildNeedToken;
+
+/* Submit a dependency build WITHOUT blocking and receive a token. The
+ * coordinator begins resolving it concurrently; submit returns as soon as the
+ * need is registered (it still fails fast on an immediate error — an unknown
+ * target or a dependency cycle). A recipe submits its whole fan-out, then awaits
+ * each token, so independent deps build in parallel. `req` is as in
+ * kit_build_client_need (config overlay + optional argv). */
+KIT_API KitStatus kit_build_client_need_submit(KitBuildClient*,
+ const KitBuildRequest* req,
+ KitBuildNeedToken* out_token);
+
+/* Block until the submitted need identified by `token` completes, returning its
+ * output tree id and materialized path. The target-dep edge is logged HERE (on
+ * await), so a submitted-but-never-awaited need is speculative and is NOT a
+ * recorded dependency — it is cancelled when the recipe exits.
+ *
+ * Await is always for a SPECIFIC token: there is deliberately no "await whichever
+ * finishes first". Concurrency is a performance property only; the set of needs a
+ * recipe awaits (hence its dependencies and output) must stay a deterministic
+ * function of its inputs, never of completion order — otherwise the cache would
+ * record different dependencies across runs. Awaiting the same token twice
+ * returns the same result (the edge is deduped). */
+KIT_API KitStatus kit_build_client_need_await(KitBuildClient*,
+ KitBuildNeedToken token,
+ KitBuildResult* out);
+
+#endif
diff --git a/include/kit/cas.h b/include/kit/cas.h
@@ -75,6 +75,24 @@ KIT_API void kit_cas_close(KitCas* cas);
KIT_API KitStatus kit_cas_add_blob(KitCas* cas, const uint8_t* data, size_t len,
KitBlobInfo* out);
+/* Report whether a blob is present locally (KIT_OK present, KIT_NOT_FOUND
+ * absent). A light existence probe — it does not read or re-verify the bytes.
+ */
+KIT_API KitStatus kit_cas_has_blob(KitCas* cas,
+ const uint8_t id[KIT_CAS_HASH_LEN]);
+
+/* Read a stored blob's bytes by id. On success *out borrows the bytes; release
+ * them with kit_cas_release. Returns KIT_NOT_FOUND on a miss. This is the
+ * by-id read path that lets a recorded config-id / argv-id resolve back to the
+ * actual serialized map or vector (the build coordinator's `need` replay). */
+KIT_API KitStatus kit_cas_get_blob(KitCas* cas,
+ const uint8_t id[KIT_CAS_HASH_LEN],
+ KitFileData* out);
+
+/* Release bytes borrowed from kit_cas_get_blob (via the store's host file_io).
+ * Idempotent for a zero-initialized KitFileData. */
+KIT_API void kit_cas_release(KitCas* cas, KitFileData* fd);
+
/* Incrementally build a tree. Each add hashes its bytes and stores the blob;
* finish sorts and validates the entries, stores the canonical tree manifest,
* and yields its id. Free the builder when done (finish does not free it). */
diff --git a/src/build/build.h b/src/build/build.h
@@ -0,0 +1,124 @@
+#ifndef KIT_BUILD_BUILD_H
+#define KIT_BUILD_BUILD_H
+
+#include <kit/cas.h>
+#include <kit/core.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/*
+ * Shared substrate for the content-addressed build coordinator
+ * (`<kit/build_coord.h>`, `kit build`). See doc/plan/BUILD.md for the design.
+ *
+ * This layer sits *on top of* the public content store (`<kit/cas.h>`), the
+ * digest surface (`<kit/hash.h>`), and the signed-package machinery
+ * (`<kit/package.h>`). It deliberately does not reach into src/dist internals:
+ * every id it computes is a plain BLAKE2b-256 of canonical bytes, which is
+ * exactly a CAS blob id (`kit_blob_info`), so config maps, argv vectors, and
+ * trace bodies content-address themselves through the ordinary CAS.
+ *
+ * Naming/result conventions mirror src/dist: internal modules return `int`
+ * (BUILD_OK / BUILD_ERR), parse/emit modules also take a `char* err, size_t
+ * errcap` sink, and the public composition layer (src/api/build.c) maps the
+ * result to KitStatus and routes human-readable detail through ctx->diag. All
+ * ids are lowercase hex on the wire and in every canonical text body.
+ */
+
+/* All content ids are BLAKE2b-256, shared with the CAS. */
+#define BUILD_HASH_LEN KIT_CAS_HASH_LEN
+#define BUILD_HEX_LEN (2u * BUILD_HASH_LEN + 1u)
+
+/* Result convention for internal modules: 0 = ok, non-zero = error. */
+#define BUILD_OK 0
+#define BUILD_ERR 1
+
+/* Canonical text magics (version lines), in the `kit-tree 1` / `kit-package 3`
+ * family: a version line, ordered scalar fields, then sorted sections. */
+#define BUILD_SHALLOW_MAGIC "kit-build-shallow 1"
+#define BUILD_DEEP_MAGIC "kit-build-deep 1"
+#define BUILD_DEEPSET_MAGIC \
+ "kit-build-deepset 1" /* one node of the transitive input-closure DAG */
+#define BUILD_RECORD_MAGIC "kit-build-record 1"
+#define BUILD_CONFIG_MAGIC \
+ "kit-build-config 1" /* serialized propagated map \
+ */
+#define BUILD_ARGV_MAGIC "kit-build-argv 1" /* serialized local argv vector */
+#define BUILD_TRACES_MAGIC \
+ "kit-build-traces 1" /* signed trace-bundle manifest */
+#define BUILD_DEFN_MAGIC "kit-build 1" /* build-definition file */
+
+/* The resolution identity of a build is the triple (target-name, config-id,
+ * argv-id): config-id is the effective propagated config, argv-id the effective
+ * local argv. Two requests for one target that differ in either are distinct
+ * builds. Traces record all three and match on all three.
+ *
+ * Domain-separation prefix for the per-target index key:
+ * target-key = BLAKE2b("kit build target v1" || target-name).
+ * The index key depends on the target NAME only — never config, argv, or
+ * recipe-id — so a target's mutable record is a stable handle holding candidate
+ * traces across every (config, argv) it was recently built under, while its
+ * inputs churn. */
+#define BUILD_TARGET_KEY_DOMAIN "kit build target v1"
+
+/* Fixed string capacities. A target name and config key/value are bounded; a
+ * workspace/source path is bounded by BUILD_PATH_MAX. These cap a single
+ * canonical row, never an aggregate. */
+#define BUILD_TARGET_MAX 256u /* e.g. "//app:server" */
+#define BUILD_KEY_MAX 64u /* config key name */
+#define BUILD_VAL_MAX 512u /* config value */
+#define BUILD_PATH_MAX 1024u /* workspace-relative or absolute fs path */
+#define BUILD_PATTERN_MAX 256u /* glob pattern */
+
+/* The two kinds of trace a build records. Written together on every real
+ * build; resolution picks whichever fits the change it sees. */
+typedef enum BuildTraceKind {
+ BUILD_TRACE_DEEP = 0,
+ BUILD_TRACE_SHALLOW = 1,
+} BuildTraceKind;
+
+/* MRU window size of the mutable per-target record (each kind capped
+ * independently). Older candidates age out and become GC-eligible. */
+#define KIT_BUILD_RECORD_CAP 8u
+
+/* The first two lowercase-hex chars of an id form its fan-out dir, as in
+ * doc/DISTRIBUTE.md (`<pp>/<id>`). */
+#define BUILD_PP_LEN 2u
+
+/* Byte-exact id equality. */
+static inline int build_id_eq(const uint8_t a[BUILD_HASH_LEN],
+ const uint8_t b[BUILD_HASH_LEN]) {
+ size_t i;
+ for (i = 0; i < BUILD_HASH_LEN; ++i)
+ if (a[i] != b[i]) return 0;
+ return 1;
+}
+
+/* Compute target-key = BLAKE2b(BUILD_TARGET_KEY_DOMAIN || target-name). */
+int build_target_key(KitSlice target_name, uint8_t out[BUILD_HASH_LEN]);
+
+/* One (path, blob-id) pair in a glob's canonical match listing. Structurally a
+ * source leaf, but defined at the foundation so the glob-result hash stays a
+ * pure, separately-testable helper — it is the SOLE producer of the value the
+ * deep/shallow traces store and match on under [glob], so coord (which expands
+ * the glob) and any future consumer agree on its bytes by construction. */
+typedef struct BuildPathBlob {
+ char path[BUILD_PATH_MAX];
+ uint8_t blob[BUILD_HASH_LEN];
+} BuildPathBlob;
+
+/* glob-result-hash = BLAKE2b of the canonical listing of a glob's matches: one
+ * "<path> <hex blob-id>\n" row per entry, entries pre-sorted by path and
+ * deduped by the caller. One row thus covers the existence AND content of the
+ * whole match set, so files read through a glob need no separate source rows.
+ * Pure: emits the canonical bytes through a scratch writer off `heap`, then
+ * hashes (kit_blob_info), so the result equals the BLAKE2b that would name the
+ * listing as a CAS blob. */
+int build_glob_result_hash(KitHeap* heap, const BuildPathBlob* entries,
+ size_t n, uint8_t out[BUILD_HASH_LEN]);
+
+/* Emit a human-readable operational error through ctx->diag (no source
+ * location), mirroring cas_diagf. No-op when ctx/diag is NULL. */
+void build_diagf(const KitContext* ctx, const char* fmt, ...);
+
+#endif
diff --git a/src/build/bundle.h b/src/build/bundle.h
@@ -0,0 +1,73 @@
+#ifndef KIT_BUILD_BUNDLE_H
+#define KIT_BUILD_BUNDLE_H
+
+#include <kit/build_coord.h>
+#include <kit/core.h>
+#include <kit/package.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+#include "coord.h"
+
+/*
+ * Shared traces, as signed bundles. A trace is a CLAIM ("these inputs => this
+ * output tree"), not self-verifying content: confirming it means re-running the
+ * assumed-deterministic recipe, the very work we avoid. So importing someone
+ * else's trace requires trusting the claimant. Bundles reuse the
+ * <kit/package.h> manifest + minisign + trust machinery wholesale: a signed
+ * `kit-build-traces 1` manifest lists (target, kind, trace-id, output-tree-id)
+ * claims, carried in a `.kpkg` alongside the trace bodies and the CAS blobs they
+ * reference — the serialized config-map / argv blobs (required, else an imported
+ * shallow trace cannot replay its `need`s) AND the deepset closure blobs a deep
+ * trace points at (required, else an imported deep trace cannot refresh) — plus
+ * optionally the referenced output trees/blobs.
+ *
+ * The security split is precise: a local build can deep/shallow-HIT on a remote
+ * builder's trace (obtaining the output without running the recipe) while
+ * trusting only the signed claim; the output bytes remain trustless, verified
+ * by tree-id/blob-id on use.
+ */
+
+/* One claim row in a bundle manifest. */
+typedef struct BuildTraceClaim {
+ char target[BUILD_TARGET_MAX];
+ uint8_t kind; /* BuildTraceKind */
+ uint8_t trace_id[BUILD_HASH_LEN];
+ uint8_t output_tree[BUILD_HASH_LEN];
+} BuildTraceClaim;
+
+/* Gather the named targets' recorded traces (newest candidates), collect the
+ * trace bodies + referenced config/argv blobs + the deepset-closure blobs each
+ * deep trace transitively points at (+ optional output trees/blobs), emit and
+ * sign a kit-build-traces manifest, and write the `.kpkg` to out_path.
+ * Signing/format/key come from `opts`. */
+int build_bundle_export(KitBuildCoordinator*,
+ const KitBuildExportOptions* opts);
+
+/* Verify a bundle's signature against the trust inputs (the unchanged
+ * DISTRIBUTE.md model: -p key, the trusted-keys anchor, or --tofu; the signed
+ * trusted comment binds the signature to the manifest hash), then install the
+ * trace bodies into build/trace/, install referenced config/argv (and any
+ * packed outputs) into the CAS, and prepend the imported traces to the relevant
+ * target records. Output bytes are hash-verified on use. */
+int build_bundle_import(KitBuildCoordinator*, const KitBuildImportOptions* opts,
+ KitBuildImportResult* result);
+
+/* Try to obtain a trace for `target` from the configured trace remotes
+ * (opts.trace_remotes), in order: render each remote's template, fetch the
+ * signed bundle via the exec host, verify it against that remote's trust anchor
+ * (trusted_keys / tofu), and import its traces. Returns BUILD_OK once a bundle
+ * imports, BUILD_ERR if none is configured or all fail. The trusted counterpart
+ * to build_remote_fetch (content is trustless; trace claims are trusted). */
+int build_trace_remote_pull(KitBuildCoordinator*, KitSlice target);
+
+/* Emit / parse the canonical kit-build-traces manifest body (the bytes that get
+ * signed). Pure logic, mirroring dist_manifest_*. */
+int build_bundle_manifest_emit(const BuildTraceClaim*, size_t n,
+ KitWriter* out);
+int build_bundle_manifest_parse(const uint8_t* data, size_t len,
+ BuildTraceClaim* out, size_t cap, size_t* n,
+ char* err, size_t errcap);
+
+#endif
diff --git a/src/build/cfg.h b/src/build/cfg.h
@@ -0,0 +1,94 @@
+#ifndef KIT_BUILD_CFG_H
+#define KIT_BUILD_CFG_H
+
+#include <kit/build_coord.h> /* KitBuildKV (the `need` overlay pair) */
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+
+/*
+ * The two configuration objects a build records, both serialized BY VALUE as
+ * canonical, content-addressed CAS blobs (never reduced to an opaque hash) so
+ * that an imported trace can replay every `need` exactly:
+ *
+ * BuildConfig - the effective PROPAGATED config: a key->value map that flows
+ * down the subtree, canonicalized to byte-stable text. Its
+ * blob id IS the `config-id`. Overlays applied along a `need` path produce a
+ * new map (and new id). BuildArgv - the LOCAL config: the target's argv
+ * vector, fixed by the build definition, visible only to that one recipe and
+ * never propagated. Its blob id is the `argv-id`.
+ *
+ * Pure value logic: canonicalize, hash, parse, look up, overlay. No I/O — the
+ * store persists/loads the resulting blobs through the CAS. Entries hang off a
+ * caller-provided, growable buffer (the DistTree pattern) so there are no VLAs
+ * and the api layer owns sizing.
+ */
+
+typedef struct BuildConfigEntry {
+ char key[BUILD_KEY_MAX];
+ char value[BUILD_VAL_MAX];
+} BuildConfigEntry;
+
+/* A propagated config map. `entries` is caller-owned; kept sorted by key with
+ * no duplicates once canonicalized. */
+typedef struct BuildConfig {
+ BuildConfigEntry* entries;
+ size_t n;
+ size_t cap;
+} BuildConfig;
+
+/* A local argv vector. `args` is caller-owned; order is significant and
+ * preserved (argv is positional, not a map). */
+typedef struct BuildArgv {
+ char (*args)[BUILD_VAL_MAX];
+ size_t n;
+ size_t cap;
+} BuildArgv;
+
+/* Initialize a map/vector over caller storage. */
+void build_config_init(BuildConfig*, BuildConfigEntry* storage, size_t cap);
+void build_argv_init(BuildArgv*, char (*storage)[BUILD_VAL_MAX], size_t cap);
+
+/* Set key=value (replacing any existing key). Returns BUILD_ERR if full or the
+ * key/value overflows its cap. */
+int build_config_set(BuildConfig*, KitSlice key, KitSlice value);
+
+/* Populate `out` (over its own storage) from a request's positional argv slices
+ * — the effective local argv. `args == NULL` or `n == 0` yields the empty argv
+ * (its argv-id is the fixed hash of the canonical empty vector); there is no
+ * build-definition default. Returns BUILD_ERR if full or an arg overflows its
+ * cap. */
+int build_argv_set(BuildArgv* out, const KitSlice* args, size_t n);
+
+/* Look up `key`. On return *present is 0 when absent (== unset); *value (when
+ * present) borrows the stored value until the map is mutated. */
+int build_config_get(const BuildConfig*, KitSlice key, KitSlice* value,
+ int* present);
+
+/* Produce `out` = `base` with each k=v in `overrides` applied (the `need`
+ * overlay). `out` must be initialized over its own storage; it is filled and
+ * canonicalized. */
+int build_config_overlay(const BuildConfig* base, const KitBuildKV* overrides,
+ size_t noverrides, BuildConfig* out);
+
+/* Emit the canonical, byte-stable serialized form (BUILD_CONFIG_MAGIC version
+ * line, then sorted `key value` rows). Sorting/dedup happens here. */
+int build_config_emit(const BuildConfig*, KitWriter* out);
+int build_argv_emit(const BuildArgv*, KitWriter* out);
+
+/* Parse a serialized map/vector (e.g. a CAS blob recovered by id) back into a
+ * value. A non-canonical body is a parse error. */
+int build_config_parse(const uint8_t* data, size_t len, BuildConfig* out,
+ char* err, size_t errcap);
+int build_argv_parse(const uint8_t* data, size_t len, BuildArgv* out, char* err,
+ size_t errcap);
+
+/* Compute the content id of the canonical serialization:
+ * config-id = BLAKE2b(canonical map), argv-id = BLAKE2b(canonical argv).
+ * Equivalent to emitting then hashing; needs a heap for the scratch writer. */
+int build_config_id(KitHeap*, const BuildConfig*, uint8_t out[BUILD_HASH_LEN]);
+int build_argv_id(KitHeap*, const BuildArgv*, uint8_t out[BUILD_HASH_LEN]);
+
+#endif
diff --git a/src/build/coord.h b/src/build/coord.h
@@ -0,0 +1,229 @@
+#ifndef KIT_BUILD_COORD_H
+#define KIT_BUILD_COORD_H
+
+#include <kit/build_coord.h>
+#include <kit/cas.h>
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+#include "cfg.h"
+#include "defn.h"
+#include "store.h"
+#include "trace.h"
+
+/*
+ * The coordinator context and its process-lifetime in-memory state. All state
+ * hangs off one KitBuildCoordinator (no globals — the project rule). For the
+ * life of the process it memoizes every base-input probe and every target
+ * resolution, so a diamond in the graph is built once and a source file is
+ * hashed once. "Cached over the life of the coordinator" means exactly these
+ * memo tables: the coordinator assumes the workspace does not change under it
+ * mid-build.
+ *
+ * This header owns the coordinator struct, the resolution RESULT type the
+ * targets memo stores, the base-input memo accessors, and the concurrency
+ * primitives (jobs semaphore + per-target futures) built on the optional
+ * KitBuildSched. The resolution ALGORITHM is resolve.h; running a recipe is
+ * runner.h.
+ */
+
+/* The in-memory deepset node: one node of the transitive input-closure DAG that
+ * a resolution returns and the targets memo caches (trace.h BuildDeepSet is its
+ * on-disk/CAS-blob form). It carries this node's DIRECT source/glob leaves, its
+ * target + recipe-id, and pointers to its children's (interned) nodes. `id` is
+ * the deep-set-id — the CAS blob id of the emitted node — so an unchanged
+ * subtree is recognized by id equality without descending. Folded into a parent
+ * by build_leafset_union and refreshed (against the live workspace) by the deep
+ * fast path. Coordinator-owned: interned once per id and stable for the process
+ * (a node reached through many parents is one object — the Merkle DAG sharing). */
+typedef struct BuildLeafSet {
+ uint8_t id[BUILD_HASH_LEN]; /* deep-set-id == CAS blob id of this node */
+ char target[BUILD_TARGET_MAX];
+ uint8_t recipe[BUILD_HASH_LEN]; /* recompute via defn to detect a repoint */
+ BuildSourceLeaf* sources; /* DIRECT leaves of this node */
+ size_t n_sources;
+ BuildGlobLeaf* globs;
+ size_t n_globs;
+ const struct BuildLeafSet** children; /* direct deps' interned nodes */
+ size_t n_children;
+} BuildLeafSet;
+
+/* A completed resolution: what a `need` returns and what the targets memo
+ * caches. `path` is a materialized output directory; `leafset` is borrowed from
+ * the coordinator and lives for the process. */
+typedef struct BuildResolved {
+ uint8_t output_tree[BUILD_HASH_LEN];
+ char path[BUILD_PATH_MAX];
+ const BuildLeafSet* leafset;
+} BuildResolved;
+
+/* Opaque process-lifetime memo tables and the per-target future table. */
+typedef struct BuildSourceMemo BuildSourceMemo; /* path -> blob-id (+ stat) */
+typedef struct BuildGlobMemo BuildGlobMemo; /* pattern -> glob result */
+typedef struct BuildConfigMemo BuildConfigMemo; /* config-id -> map */
+typedef struct BuildArgvMemo BuildArgvMemo; /* argv-id -> vector */
+typedef struct BuildDeepSetMemo
+ BuildDeepSetMemo; /* deep-set-id -> {interned node, refresh-valid?} */
+typedef struct BuildPulledSet
+ BuildPulledSet; /* target-name -> already trace-remote-pulled */
+typedef struct BuildTargetTable
+ BuildTargetTable; /* (name,cfg-id,argv-id) -> future */
+typedef struct BuildTargetFuture BuildTargetFuture;
+
+struct KitBuildCoordinator {
+ KitContext ctx_storage; /* private copy so the handle outlives caller's ctx */
+ const KitContext* ctx;
+ KitBuildHost host; /* borrowed vtables */
+ KitBuildOptions opts; /* borrowed strings */
+ KitCas* cas; /* the shared content store */
+ BuildStore store; /* paths + record RMW + tree cache over build/ */
+ BuildDefn defn; /* parsed build definition (a tracked source) */
+ KitFileData defn_bytes; /* the definition file bytes (borrowed/owned by io) */
+ int jobs_limit; /* effective parallelism (1 when host->sched NULL) */
+ /* Process-lifetime memos. */
+ BuildSourceMemo* sources;
+ BuildGlobMemo* globs;
+ BuildConfigMemo* configs;
+ BuildArgvMemo* argvs;
+ BuildDeepSetMemo* deepsets; /* interned closure nodes + refresh-valid cache */
+ BuildPulledSet* pulled; /* targets a trace-remote pull was attempted for */
+ BuildTargetTable* targets;
+ /* Concurrency state (NULL/no-op when host->sched is NULL). */
+ KitBuildMutex* lock; /* guards the memos + targets table + stats */
+ void* jobs_sem; /* opaque counting semaphore over host->sched */
+ KitBuildStats stats; /* cumulative resolution counters; bumped under `lock` */
+};
+
+/* Bump one resolution counter (under `lock`); resolve/runner/remote/bundle call
+ * this so kit_build_stats can report per-phase work. */
+typedef enum BuildStatField {
+ BUILD_STAT_DEEP_HIT = 0,
+ BUILD_STAT_SHALLOW_HIT,
+ BUILD_STAT_RECIPE_RUN,
+ BUILD_STAT_MATERIALIZE_MISS,
+ BUILD_STAT_OBJECT_FETCH,
+ BUILD_STAT_TRACE_PULL,
+} BuildStatField;
+void build_coord_stat_bump(KitBuildCoordinator*, BuildStatField);
+
+/* Open/close. open loads + parses the build definition, opens the CAS and
+ * store, allocates the memos, and sizes the jobs semaphore (clamped to 1 when
+ * the host provides no sched). */
+KitStatus build_coord_open(const KitContext*, const KitBuildHost*,
+ KitSlice store_root, const KitBuildOptions*,
+ KitBuildCoordinator** out);
+void build_coord_close(KitBuildCoordinator*);
+
+/* ---- Base-input probes (memoized) ------------------------------------- */
+
+/* Hash a source file by workspace path. *present is 0 when the file is absent
+ * (absence is a legitimate, cacheable observation). Memoized per process. */
+int build_coord_source_hash(KitBuildCoordinator*, KitSlice path,
+ uint8_t out_blob[BUILD_HASH_LEN], int* present);
+
+/* Expand a glob and return its glob-result-hash; optionally enumerate the
+ * sorted matches via cb. Memoized per process. */
+typedef int (*BuildCoordGlobFn)(void* user, const char* path,
+ const uint8_t blob[BUILD_HASH_LEN]);
+int build_coord_glob(KitBuildCoordinator*, KitSlice pattern,
+ uint8_t out_result_hash[BUILD_HASH_LEN],
+ BuildCoordGlobFn cb, void* cb_user);
+
+/* Recover a propagated config map / an argv vector by its id (loads + parses
+ * the serialized CAS blob, memoized) — the replay path for a recorded `need`,
+ * which re-resolves the dep under its exact (config-id, argv-id). */
+int build_coord_config_by_id(KitBuildCoordinator*,
+ const uint8_t config_id[BUILD_HASH_LEN],
+ BuildConfig* out);
+int build_coord_argv_by_id(KitBuildCoordinator*,
+ const uint8_t argv_id[BUILD_HASH_LEN],
+ BuildArgv* out);
+
+/* recipe-id = BLAKE2b(recipe file bytes), from the target's definition stanza.
+ */
+int build_coord_recipe_id(KitBuildCoordinator*, KitSlice target,
+ uint8_t out[BUILD_HASH_LEN]);
+
+/* The effective argv is the request's argv (empty when none is supplied) — no
+ * definition lookup is involved; build it with build_argv_set and hash it with
+ * build_argv_id. */
+
+/* ---- Deepset nodes (process-lived closure DAG) ------------------------- */
+
+/* Intern a deepset node into coordinator-owned storage so it lives for the
+ * process and is shared by id: BuildResolved.leafset is borrowed-for-the-
+ * process, so a node built transiently (build_leafset_union folds children;
+ * a Phase-2 hit rebuilds one) is copied here, deduped by `id`, before it is
+ * handed back. *out is stable until build_coord_close. */
+int build_coord_leafset_intern(KitBuildCoordinator*, const BuildLeafSet* in,
+ const BuildLeafSet** out);
+
+/* Load the deepset node named by a deep-set-id: read its CAS blob
+ * (kit_cas_get_blob), parse (build_deepset_parse), and recursively load+intern
+ * its children, returning a fully-linked, interned node. Memoized by id (a node
+ * reached through many parents loads once). Returns BUILD_ERR if the blob — or
+ * any child blob — is absent, so Phase 1 treats the deep trace as absent and
+ * falls through (fail-safe). The replay path for the deep fast path. */
+int build_coord_deepset_load(KitBuildCoordinator*,
+ const uint8_t deepset_id[BUILD_HASH_LEN],
+ const BuildLeafSet** out);
+
+/* Refresh-validity memo over the deepset DAG (keyed by deep-set-id): whether a
+ * subtree's leaves all still match the live workspace. build_leafset_refresh
+ * (resolve.h) reads/writes this so a subtree checked once is reused by every
+ * parent that points at it, and an unchanged subtree is skipped by id equality. */
+int build_coord_deepset_valid_get(KitBuildCoordinator*,
+ const uint8_t deepset_id[BUILD_HASH_LEN],
+ int* known, int* valid);
+void build_coord_deepset_valid_set(KitBuildCoordinator*,
+ const uint8_t deepset_id[BUILD_HASH_LEN],
+ int valid);
+
+/* ---- Shared traces (lazy pull) ---------------------------------------- */
+
+/* On a local-record miss for `target`, attempt ONE trace-remote pull (delegates
+ * to bundle.h build_trace_remote_pull, which verifies signature/trust and
+ * installs trace bodies + referenced blobs). Idempotent per target via the
+ * `pulled` set: a second miss does not re-pull. *pulled_now reports whether new
+ * traces were installed (so resolve re-scans the record). No-op (BUILD_OK,
+ * *pulled_now=0) when no trace-remotes are configured. */
+int build_coord_trace_remote_pull_once(KitBuildCoordinator*, KitSlice target,
+ int* pulled_now);
+
+/* ---- Concurrency: jobs semaphore + per-target futures ----------------- */
+
+/* Bound concurrently *running* recipe processes to jobs_limit. The slot counts
+ * a recipe only while it is actively running: a recipe blocked servicing a
+ * `need` releases its slot (runner.h) and reacquires once the sub-build
+ * returns, so a dependency chain deeper than jobs_limit cannot deadlock. No-ops
+ * (always succeed immediately) in sequential mode. */
+void build_coord_jobs_acquire(KitBuildCoordinator*);
+void build_coord_jobs_release(KitBuildCoordinator*);
+
+/* Run `fn(arg)` on a fresh worker thread (host->sched->thread_spawn), tracked by
+ * the coordinator and joined at build completion — the substrate for concurrent
+ * `need` dispatch (build_dispatch, resolve.h). Returns BUILD_ERR when there is no
+ * sched (sequential mode), so the caller runs `fn` inline instead. The worker's
+ * sync point is the target future it completes, not a join. */
+int build_coord_spawn(KitBuildCoordinator*, void (*fn)(void*), void* arg);
+
+/* Intern a future for (target, config-id, argv-id): the cross-path memo AND
+ * in-flight dedup. On return *is_fresh == 1 means this caller owns the
+ * resolution and must complete or fail the future; *is_fresh == 0 means another
+ * resolution is (or was) in flight and the caller should await it. In
+ * sequential mode a fresh future resolves inline before any other caller can
+ * observe it. */
+int build_coord_target_intern(KitBuildCoordinator*, KitSlice target,
+ const uint8_t config_id[BUILD_HASH_LEN],
+ const uint8_t argv_id[BUILD_HASH_LEN],
+ BuildTargetFuture** out, int* is_fresh);
+/* Block until the future is completed or failed; copies the result out. */
+int build_coord_target_await(KitBuildCoordinator*, BuildTargetFuture*,
+ BuildResolved* out);
+void build_coord_target_complete(KitBuildCoordinator*, BuildTargetFuture*,
+ const BuildResolved*);
+void build_coord_target_fail(KitBuildCoordinator*, BuildTargetFuture*);
+
+#endif
diff --git a/src/build/defn.h b/src/build/defn.h
@@ -0,0 +1,62 @@
+#ifndef KIT_BUILD_DEFN_H
+#define KIT_BUILD_DEFN_H
+
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+
+/*
+ * The build definition: the file that maps each target name to one recipe. The
+ * coordinator reads it to resolve a target. The ONLY definition-derived input of
+ * a target is its resolved recipe-id (BLAKE2b of the recipe FILE bytes), carried
+ * as a per-node scalar in the deepset and refreshed by recomputing it through
+ * the LIVE definition (build_coord_recipe_id). That is content-only and correct:
+ * repointing a target to a different-content recipe changes its recipe-id (busts);
+ * repointing to a same-content recipe does not (same build); and — crucially —
+ * editing an UNRELATED stanza does not change defn_find(T)'s result, so it does
+ * NOT bust T. There is therefore NO whole-definition source leaf (an earlier
+ * design that would have busted every target on any stanza edit); the `bytes`
+ * below are retained only for the parse, not hashed as a dependency.
+ *
+ * The definition does NOT carry argv: local argv is supplied entirely by the
+ * build request (empty when none is given) and forms the argv-id dimension of
+ * the resolution key — see KitBuildRequest. Pure parse + lookup, no I/O.
+ *
+ * Format (v1, provisional) — BUILD_DEFN_MAGIC version line then one stanza per
+ * target:
+ *
+ * kit-build 1
+ * [target //app:server]
+ * recipe recipes/cc.sh
+ * [target //lib:core]
+ * recipe recipes/cc.sh
+ * ...
+ */
+
+/* One resolved target stanza: a name and the recipe it maps to. */
+typedef struct BuildTargetDefn {
+ char name[BUILD_TARGET_MAX];
+ char recipe_path[BUILD_PATH_MAX]; /* workspace-relative recipe file */
+} BuildTargetDefn;
+
+/* The parsed definition: an index of stanzas over caller-provided storage, plus
+ * the raw bytes (borrowed) so the coordinator can hash the file as a source. */
+typedef struct BuildDefn {
+ const uint8_t* bytes; /* the whole definition file, borrowed */
+ size_t len;
+ BuildTargetDefn* targets;
+ size_t n_targets;
+ size_t cap_targets;
+} BuildDefn;
+
+/* Parse a definition file into `out` over caller storage. Non-canonical input
+ * is a parse error -> BUILD_ERR with detail in err. */
+int build_defn_parse(const uint8_t* data, size_t len, BuildDefn* out, char* err,
+ size_t errcap);
+
+/* Find a target stanza by name; returns NULL if absent. */
+const BuildTargetDefn* build_defn_find(const BuildDefn*, KitSlice name);
+
+#endif
diff --git a/src/build/protocol.h b/src/build/protocol.h
@@ -0,0 +1,100 @@
+#ifndef KIT_BUILD_PROTOCOL_H
+#define KIT_BUILD_PROTOCOL_H
+
+#include <kit/build_coord.h> /* KitBuildKV */
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+
+/*
+ * The recipe protocol: the command set and length-prefixed framing both sides
+ * speak over a KitBuildTransport. Pure encode/decode — no I/O, no transport
+ * (the coordinator's runner drives the server side, the client drives the
+ * recipe side). Every request both returns a value and logs a dependency; the
+ * contract that makes caching correct is that an input the recipe reads but
+ * does not request through one of these commands is invisible to the cache.
+ *
+ * Wire framing: each frame is a single command/response, carried whole by the
+ * transport's read_frame/write_frame (the transport owns the length prefix).
+ * The body is a compact, byte-stable encoding of the structs below.
+ *
+ * One response exception — GLOB. A glob match set can exceed one frame, so a
+ * glob response is a SEQUENCE of frames: zero or more match-batch frames, each a
+ * BUILD_RESP_OK frame carrying a run of paths, terminated by a final
+ * BUILD_RESP_GLOB_END frame. The server (runner) writes batches until the
+ * matches are exhausted then the END frame; the client/runner-decoder loops
+ * read_frame, invoking the per-match callback on each batch, until END. Every
+ * OTHER command is exactly one request frame and one response frame.
+ */
+
+/* Largest single frame the protocol will encode or accept. Bounds a config
+ * value, an overlay set, or ONE glob match-batch (large match sets span
+ * multiple batch frames — see the framing note above). */
+#define BUILD_FRAME_MAX 65536u
+
+typedef enum BuildCmd {
+ BUILD_CMD_CONFIG_GET = 1, /* arg: key -> value | unset */
+ BUILD_CMD_SOURCE = 2, /* arg: path -> blob-id + realpath */
+ BUILD_CMD_GLOB = 3, /* arg: pattern -> sorted match paths */
+ BUILD_CMD_NEED = 4, /* arg: target + k=v[] + argv[] -> tree-id + path (blocking) */
+ BUILD_CMD_NEED_SUBMIT = 5, /* arg: target + k=v[] + argv[] -> token (non-blocking) */
+ BUILD_CMD_NEED_AWAIT = 6, /* token -> tree-id + path */
+} BuildCmd;
+
+typedef enum BuildRespStatus {
+ BUILD_RESP_OK = 0,
+ BUILD_RESP_UNSET = 1, /* config-get on an unset key (not an error) */
+ BUILD_RESP_ERROR = 2, /* the coordinator failed/refused the request */
+ BUILD_RESP_GLOB_END = 3, /* terminates a multi-frame glob match stream */
+} BuildRespStatus;
+
+/* A decoded request. `overrides` and `argv` (need / need-submit only) point into
+ * the frame buffer. `argv` is the dep's local argv; argc==0 (argv==NULL) means
+ * the empty argv. `token` carries the need-token for NEED_AWAIT. */
+typedef struct BuildReq {
+ uint8_t cmd; /* BuildCmd */
+ KitSlice arg; /* key | path | pattern | target (unused for AWAIT) */
+ const KitBuildKV* overrides; /* need/submit: propagated-config overlay */
+ size_t noverrides;
+ const KitSlice* argv; /* need/submit: local argv (NULL or argc==0 => empty argv) */
+ size_t argc;
+ uint64_t token; /* need-await: which submitted need to collect */
+} BuildReq;
+
+/* A decoded response. The value union is selected by `cmd`. Slices/ids borrow
+ * the frame buffer until the next read. */
+typedef struct BuildResp {
+ uint8_t status; /* BuildRespStatus */
+ uint8_t id[BUILD_HASH_LEN]; /* source: blob-id; need/await: output tree-id */
+ KitSlice text; /* config value; source/need/await realpath */
+ uint64_t token; /* need-submit: the coordinator-assigned token */
+ /* glob results are streamed: the decoder invokes a callback per match rather
+ * than materializing a list in the struct. */
+} BuildResp;
+
+/* Encode a request/response into `buf` (cap >= BUILD_FRAME_MAX); *n gets the
+ * byte count. The transport adds the length prefix. */
+int build_proto_encode_req(const BuildReq*, uint8_t* buf, size_t cap,
+ size_t* n);
+int build_proto_encode_resp(const BuildReq* for_cmd, const BuildResp*,
+ uint8_t* buf, size_t cap, size_t* n);
+
+/* Decode a request/response from a received frame. `cmd` tells the response
+ * decoder which value shape to expect. A NEED request's overlay pairs fill
+ * `ovr_storage` and its argv fills `argv_storage`. Malformed input returns
+ * BUILD_ERR. */
+int build_proto_decode_req(const uint8_t* buf, size_t len, BuildReq* out,
+ KitBuildKV* ovr_storage, size_t ovr_cap,
+ KitSlice* argv_storage, size_t argv_cap);
+
+/* Reports one glob match while decoding a glob response. Return non-zero to
+ * stop. */
+typedef int (*BuildProtoGlobFn)(void* user, const char* path);
+
+int build_proto_decode_resp(const uint8_t* buf, size_t len, uint8_t cmd,
+ BuildResp* out, BuildProtoGlobFn glob_cb,
+ void* glob_user);
+
+#endif
diff --git a/src/build/remote.h b/src/build/remote.h
@@ -0,0 +1,56 @@
+#ifndef KIT_BUILD_REMOTE_H
+#define KIT_BUILD_REMOTE_H
+
+#include <kit/build_coord.h> /* KitBuildExec, KitBuildObjectRemote */
+#include <kit/cas.h>
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+
+/*
+ * Untrusted remote object fetch — the extra rung in the materialization ladder
+ * (tree cache -> local CAS -> REMOTE FETCH -> run recipe). The coordinator
+ * performs no network I/O itself: when a needed object is absent locally and a
+ * remote is configured, it renders the user's fetch-recipe argv template and
+ * runs it via the exec host to pull one object into a temp file. The fetched
+ * bytes are VERIFIED against the requested content id before being installed
+ * into the local CAS, so a corrupt or malicious mirror fails the hash check and
+ * is discarded — the remotes and the fetch recipe are untrusted, and the
+ * self-verifying CAS makes that safe. Several mirrors may be configured; they
+ * are tried in order until one yields verified bytes.
+ *
+ * Trees vs blobs. A BLOB fetch verifies the bytes hash to `id` and installs
+ * them. A TREE fetch installs the id-checked tree MANIFEST only; the blobs that
+ * tree references are fetched on demand (as further BUILD_REMOTE_BLOB fetches)
+ * when build_store_cache_materialize verifies and restores the tree. So
+ * obtaining a tree is a two-step affair — manifest first, then its blobs — and
+ * the whole tree is verified before any cache hit is served from it.
+ *
+ * There is no separate deepset kind: a deepset closure node is a self-verifying
+ * CAS blob (deep-set-id == blob-id), so it — like config-map and argv blobs — is
+ * fetched as an ordinary BLOB ({kind}=blob) on demand when build_coord_deepset_-
+ * load needs one the local CAS lacks.
+ *
+ * (Trust, where it exists, is for signed trace bundles — bundle.h. Content
+ * fetched here is trustless.)
+ */
+
+typedef enum BuildRemoteKind {
+ BUILD_REMOTE_BLOB = 0,
+ BUILD_REMOTE_TREE = 1,
+} BuildRemoteKind;
+
+/* Fetch one object (`kind`, `id`) by trying each remote in `remotes` in order:
+ * render its template (tokens {kind} {pp} {id} {out}), spawn it via the exec
+ * host, verify the fetched bytes hash to `id`, and install them into `cas`.
+ * Returns BUILD_OK on the first verified install, BUILD_ERR if no remote is
+ * configured or all fail/verify-fail (bad bytes are discarded). Stages into
+ * `tmp_dir`. */
+int build_remote_fetch(const KitContext*, const KitBuildExec*,
+ const KitBuildObjectRemote* remotes, size_t nremotes,
+ KitCas* cas, KitSlice tmp_dir, BuildRemoteKind kind,
+ const uint8_t id[BUILD_HASH_LEN]);
+
+#endif
diff --git a/src/build/resolve.h b/src/build/resolve.h
@@ -0,0 +1,141 @@
+#ifndef KIT_BUILD_RESOLVE_H
+#define KIT_BUILD_RESOLVE_H
+
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+#include "cfg.h"
+#include "coord.h"
+
+/*
+ * The resolution algorithm: resolve(T, cfg, chain) -> {output-tree, path,
+ * leafset}, doing the least work necessary. Three phases (see doc/plan/BUILD.md
+ * §Resolution algorithm):
+ *
+ * Phase 1 deep fast path - same config-id AND argv-id, and an all-clear
+ * refresh of the deepset closure (every source/glob
+ * leaf unchanged AND every node's recipe-id still
+ * recomputes the same) => reuse, no graph walk.
+ * Phase 2 shallow path - config, argv, and/or sources moved: guard on the
+ * recipe-id, argv-id, consumed keys + direct leaves,
+ * then re-resolve each recorded dep under its
+ * recorded (config-id, argv-id) and compare OUTPUT
+ * tree-ids; a match still skips the recipe (and
+ * writes a fresh deep trace + deepset).
+ * Phase 3 run the recipe - no trace holds (runner.h).
+ *
+ * On a local-record miss, resolution first attempts one lazy trace-remote pull
+ * (build_coord_trace_remote_pull_once) and re-scans before Phase 3. The config-
+ * dependence is recorded in and verified against the traces; the deep path
+ * checks whole-config-id + argv-id equality, the shallow path checks consumed
+ * keys + argv-id. Recipe identity rides as a per-node scalar in the deepset
+ * (Phase 1) and as the recipe-id guard (Phase 2); there is NO whole-definition
+ * dependency. Cross-path dedup is the coordinator's targets memo; per-path cycle
+ * detection is the build chain below.
+ */
+
+/* One frame of the per-path build chain: the (target, config-id, argv-id) at one
+ * level, linked to its parent (NULL at the root). The chain is a parent-linked
+ * "cactus stack", NOT a mutable array — each dispatched `need` extends the
+ * shared, read-only parent prefix with its own frame, so concurrent siblings
+ * never copy or mutate one another's chain (the property the future-based `need`
+ * fan-out requires). A `need` whose triple already appears on the walk from a
+ * frame up to the root is a dependency cycle; walking the links renders the error
+ * path (//a -> //b -> //a). Per-path, distinct from the cross-path futures memo:
+ * a shared dep appears in many chains, never twice in one. Frames are allocated
+ * off the coordinator's per-build arena and live until the build completes
+ * (parents outlive their concurrent children by construction — no pop). */
+typedef struct BuildChainFrame {
+ const struct BuildChainFrame* parent; /* NULL at the root request */
+ char target[BUILD_TARGET_MAX];
+ uint8_t config_id[BUILD_HASH_LEN];
+ uint8_t argv_id[BUILD_HASH_LEN];
+} BuildChainFrame;
+
+/* Extend `parent` (NULL at the root) with (target, config-id, argv-id), returning
+ * the new tip in *out. Walks parent links to detect a cycle (the triple already
+ * present) -> BUILD_ERR with the rendered path in err. The frame is allocated off
+ * the coordinator's per-build arena; there is no pop (frames are immutable). */
+int build_chain_extend(KitBuildCoordinator*, const BuildChainFrame* parent,
+ KitSlice target, const uint8_t config_id[BUILD_HASH_LEN],
+ const uint8_t argv_id[BUILD_HASH_LEN],
+ const BuildChainFrame** out, char* err, size_t errcap);
+
+/* Resolve a target under the effective propagated config `cfg` and effective
+ * local `argv` (cfg carries its config-id; argv hashes to its argv-id) — the
+ * full (target, config-id, argv-id) identity. `chain` is the parent frame tip
+ * (NULL at the root); this call extends it. Equivalent to dispatch+await on the
+ * same thread (used by the top-level request and the fused one-shot `need`). On
+ * success *out holds the output tree-id, a materialized path, and the deepset
+ * node (its closure DAG root). Errors (cycle, recipe failure, missing inputs)
+ * surface through the coordinator's diag. */
+int build_resolve(KitBuildCoordinator*, KitSlice target, const BuildConfig* cfg,
+ const BuildArgv* argv, const BuildChainFrame* chain,
+ BuildResolved* out);
+
+/* The async half of resolution — the engine behind recipe-side `need_submit`.
+ * Extends `chain` (cycle-checked here, so submit fails fast on a cycle), interns
+ * the (target, config-id, argv-id) future, and — if fresh — drives the 3 phases
+ * on a worker thread (build_coord_spawn) that completes the future; a duplicate
+ * attaches to the in-flight future with no new thread. Returns the future in
+ * *out_future WITHOUT blocking, so a recipe can submit many and let them resolve
+ * concurrently. The result is collected later via build_coord_target_await.
+ * `build_resolve` is the inline sibling (dispatch's phases run on the caller's
+ * thread); in sequential mode (no sched) dispatch resolves inline and returns an
+ * already-completed future, so submit/await stays semantically transparent. */
+int build_dispatch(KitBuildCoordinator*, KitSlice target, const BuildConfig* cfg,
+ const BuildArgv* argv, const BuildChainFrame* chain,
+ BuildTargetFuture** out_future, char* err, size_t errcap);
+
+/* Phase 3 of resolution: run the recipe — reached when no trace holds, or a
+ * hit's bytes were evicted everywhere (build_materialize returned BUILD_ERR).
+ * resolve and runner form one strongly-connected component (resolve calls
+ * build_run_recipe for Phase 3; runner's service loop calls build_resolve for
+ * each `need`): NOT a DAG edge. To keep that recursion compiling without a
+ * resolve.h -> runner.h include, build_run_recipe is DECLARED here (the Phase-3
+ * continuation of build_resolve) and IMPLEMENTED in runner.c (which owns the
+ * spawn + service-loop machinery). Same *out contract as build_resolve. */
+int build_run_recipe(KitBuildCoordinator*, KitSlice target,
+ const BuildConfig* cfg, const BuildArgv* argv,
+ const BuildChainFrame* chain, BuildResolved* out);
+
+/* The materialization ladder shared by every cache hit:
+ * tree cache/ -> local CAS -> remote object fetch.
+ * A PURE locator: writes the materialized directory path and returns BUILD_OK,
+ * or BUILD_ERR when the bytes are gone everywhere (not in the cache, not in the
+ * CAS, no remote supplied them). It NEVER runs a recipe — on BUILD_ERR the
+ * caller (resolve) falls through to Phase 3, which rebuilds and records the
+ * correct output. Keeping it recipe-free removes the resolve->runner edge and
+ * the stale-tree-id hazard a rebuild-inside-materialize would create (the cache
+ * hit's recorded tree-id always describes the bytes returned). */
+int build_materialize(KitBuildCoordinator*,
+ const uint8_t tree_id[BUILD_HASH_LEN], char* path_out,
+ size_t cap);
+
+/* Build a parent's deepset node from its DIRECT leaves and its children's nodes.
+ * `direct` carries this node's target, recipe-id, and direct source/glob leaves
+ * (its children fields empty); `children` are the deps' resolved nodes. Computes
+ * the deep-set-id, stores the canonical node as a CAS blob (kit_cas_add_blob —
+ * the children's blobs were stored when they were built, so this is incremental
+ * and deduped), and returns the interned node. This IS the closure union —
+ * "concat direct leaves + child ids, hash" — never a flat N-way merge. */
+int build_leafset_union(KitBuildCoordinator*, const BuildLeafSet* direct,
+ const BuildLeafSet* const* children, size_t nchildren,
+ const BuildLeafSet** out);
+
+/* Refresh a deepset node against the live workspace: per node, rehash its direct
+ * source leaves (absent stays absent), reglob its glob leaves, recompute its
+ * recipe-id through the LIVE definition (catching a dep repointed to a
+ * different-content recipe, treating same-content as unchanged), and recurse
+ * into children — all through the per-process memos and the deep-set-id validity
+ * cache (build_coord_deepset_valid_*), so a shared subtree is checked once and
+ * an unchanged subtree is skipped by id equality without descending. BUILD_OK +
+ * *all_match==1 when the whole closure still matches; *all_match==0 on the first
+ * divergence (Phase 1 then gives up). Phase 1 calls build_coord_deepset_load to
+ * turn the trace's deep-set-id into the node passed here. */
+int build_leafset_refresh(KitBuildCoordinator*, const BuildLeafSet*,
+ int* all_match);
+
+#endif
diff --git a/src/build/runner.h b/src/build/runner.h
@@ -0,0 +1,115 @@
+#ifndef KIT_BUILD_RUNNER_H
+#define KIT_BUILD_RUNNER_H
+
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+#include "cfg.h"
+#include "coord.h"
+#include "resolve.h"
+#include "trace.h"
+
+/*
+ * Phase 3: running a recipe (the server side of the recipe protocol). Acquires
+ * a job slot, stages a sandbox, spawns the recipe via the exec host, services
+ * its protocol requests over the transport — logging each as a dependency —
+ * then on success ingests the output directory into the CAS, writes BOTH trace
+ * kinds, prepends them to the target record, installs the output in the tree
+ * cache, and returns the result. A nonzero exit propagates failure and writes
+ * NO trace.
+ *
+ * Writing both traces on every real build is what lets a later request take
+ * whichever path fits the change it sees.
+ */
+
+/* One submitted-but-not-yet-awaited need: the coordinator-assigned token, the
+ * resolved (config-id, argv-id) it was dispatched under, and the in-flight
+ * future. Becomes a recorded dep edge only when AWAITED; un-awaited entries are
+ * cancelled when the recipe exits. */
+typedef struct BuildPendingNeed {
+ uint64_t token;
+ char dep[BUILD_TARGET_MAX];
+ uint8_t config_id[BUILD_HASH_LEN];
+ uint8_t argv_id[BUILD_HASH_LEN];
+ BuildTargetFuture* future; /* from build_coord_dispatch */
+} BuildPendingNeed;
+
+/* The dependency log accumulated while servicing one recipe: the direct base
+ * inputs and direct target-dep edges, plus each dep's resolved deepset node, plus
+ * the live pending-needs table (submitted, not yet awaited). Converted on success
+ * into a shallow trace (direct inputs + dep edges) and a deep trace (a single
+ * pointer to the root deepset node, built by build_leafset_union over these direct
+ * leaves + the child nodes). Sections hang off caller/arena storage (no VLAs). */
+typedef struct BuildDepLog {
+ BuildConfigKey* config_keys; /* consumed propagated key names */
+ size_t n_config_keys, cap_config_keys;
+ BuildSourceLeaf* sources; /* direct source reads (absent recorded) */
+ size_t n_sources, cap_sources;
+ BuildGlobLeaf* globs; /* direct globs */
+ size_t n_globs, cap_globs;
+ BuildDepEdge* deps; /* AWAITED `need` edges (recorded) */
+ size_t n_deps, cap_deps;
+ const BuildLeafSet** child_leafsets; /* one deepset node per awaited dep */
+ size_t n_children, cap_children;
+ BuildPendingNeed* pending; /* submitted, not yet awaited */
+ size_t n_pending, cap_pending;
+ uint64_t next_token; /* monotonic per-recipe token allocator */
+} BuildDepLog;
+
+/* build_run_recipe — the Phase 3 entry, the continuation of build_resolve — is
+ * declared in resolve.h (so resolve.c can call it without including runner.h)
+ * and IMPLEMENTED in runner.c: it acquires a job slot, stages the sandbox,
+ * spawns the recipe (process argv = `argv`, delivered as-is), services its
+ * protocol connection via build_runner_service, and on success ingests the
+ * output and records both trace kinds. The two helpers below are the pieces of
+ * that flow split out so the in-process transport and the trace writer are
+ * independently testable.
+ *
+ * Recipe environment is CLEAN: the recipe receives its COMPLETE environment from
+ * the host (KitBuildExec.spawn does not inherit the coordinator's ambient env) —
+ * the KIT_BUILD_* vars plus the build's declared pass-through, sourced from the
+ * effective config's `env.` namespace (KIT_BUILD_ENV_PREFIX). Those keys are
+ * ordinary tracked config, so a recipe's environment is hermetic and cache-
+ * visible and replays/bundles like any other config. */
+
+/* Service one recipe connection until it exits: read framed requests and
+ * dispatch them, appending each dependency to `log`. Drives the protocol.h
+ * codec over the transport. `chain` is this recipe's chain frame (its dispatched
+ * needs extend it). Commands:
+ * config-get/source/glob — synchronous; log the dep, reply.
+ * need (fused) — build_resolve inline on this thread; log the edge.
+ * need-submit — build_coord_dispatch(dep) (concurrent, no block),
+ * stash a BuildPendingNeed, reply a token. Cycles and
+ * unknown targets fail fast here.
+ * need-await <token> — release this recipe's job slot, await the pending
+ * need's future, reacquire; THEN log the dep edge +
+ * push its child node, and reply. The job slot is held
+ * only while actively running, so a chain deeper than
+ * `jobs` cannot deadlock (the same release/reacquire
+ * applies to the fused `need`).
+ * On EOF, any still-pending (submitted, never-awaited) needs are cancelled
+ * (exec->kill) and recorded as NO dependency. Separated so the in-process
+ * transport can reuse it. */
+int build_runner_service(KitBuildCoordinator*, KitBuildConn*, KitSlice target,
+ const BuildConfig* cfg, const BuildChainFrame* chain,
+ BuildDepLog* log);
+
+/* Convert a completed dep-log + output into the canonical persisted state:
+ * - build the root deepset node (build_leafset_union over the log's direct
+ * leaves + recipe-id + child nodes), which stores it as a CAS blob and yields
+ * its deep-set-id;
+ * - emit + store the SHALLOW trace (direct inputs + dep edges, by hash) and the
+ * DEEP trace (scalars + the root deep-set-id), and prepend both to the target
+ * record.
+ * Records the target's recipe-id, config-id and argv-id, and each dep edge's
+ * (config-id, argv-id). Returns the root deepset node via *out_leafset so the
+ * caller can hand it back in BuildResolved. */
+int build_runner_record_traces(KitBuildCoordinator*, KitSlice target,
+ const BuildConfig* cfg, const BuildArgv* argv,
+ const BuildDepLog*,
+ const uint8_t output[BUILD_HASH_LEN],
+ const BuildLeafSet** out_leafset);
+
+#endif
diff --git a/src/build/store.h b/src/build/store.h
@@ -0,0 +1,129 @@
+#ifndef KIT_BUILD_STORE_H
+#define KIT_BUILD_STORE_H
+
+#include <kit/build_coord.h> /* KitBuildStoreIo */
+#include <kit/cas.h>
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+#include "trace.h"
+
+/*
+ * The on-disk build store and its crash-safe storage state machine. Sits beside
+ * (or contains) a CAS:
+ *
+ * <store>/
+ * cas/ # shared content store (KitCas), per
+ * DISTRIBUTE.md build/ trace/<pp>/<trace-id> # IMMUTABLE canonical trace
+ * bodies target/<pp>/<target-key> # MUTABLE per-target trace set (the only
+ * mutable state) cache/<pp>/<tree-id>/ # materialized output directories
+ * tmp/ # staging for atomic writes + recipe
+ * sandboxes
+ *
+ * Invariant: anything reachable under a content key (cas blob/tree,
+ * build/trace) is complete and matches its key — content objects are written
+ * into tmp/, fsync'd, then atomically renamed into place (idempotent, so a
+ * racing producer is harmless). The target record is the only
+ * ordering-sensitive state, updated by read-modify-write into tmp/ then atomic
+ * rename; a torn or unparseable record is treated as empty. The store degrades
+ * toward "rebuild," never toward "wrong answer."
+ *
+ * Config maps, argv vectors, deepset closure nodes, and source bytes are all
+ * ordinary CAS blobs (written via kit_cas_add_blob, read back by id via
+ * kit_cas_get_blob — a deepset is self-verifying content, so it needs no
+ * build-specific path); only trace/ (claims), target/ (the mutable record), and
+ * cache/ are build-specific. This module owns those paths, atomicity, and the
+ * record RMW; it does not interpret trace bodies (that is trace.h).
+ */
+
+typedef struct BuildStore {
+ const KitContext* ctx;
+ KitCas* cas; /* borrowed: the shared content store */
+ const KitBuildStoreIo* io; /* borrowed: fs atomicity extras */
+ const KitCasHost* cas_host; /* borrowed: file_io for trace/record reads */
+ char root[BUILD_PATH_MAX]; /* the <store>/build root */
+} BuildStore;
+
+/* Bind a store view over an open CAS and the build root. Borrows all pointers
+ * for the store's lifetime; creates the build/ skeleton if absent. */
+int build_store_open(const KitContext*, KitCas* cas, const KitBuildStoreIo*,
+ const KitCasHost*, KitSlice store_root, BuildStore* out);
+
+/* Final content-keyed paths within the store. Each writes a NUL-terminated path
+ * (cap >= BUILD_PATH_MAX) and returns BUILD_OK/BUILD_ERR. */
+int build_store_trace_path(const BuildStore*,
+ const uint8_t trace_id[BUILD_HASH_LEN], char* out,
+ size_t cap);
+int build_store_target_path(const BuildStore*,
+ const uint8_t target_key[BUILD_HASH_LEN], char* out,
+ size_t cap);
+int build_store_cache_path(const BuildStore*,
+ const uint8_t tree_id[BUILD_HASH_LEN], char* out,
+ size_t cap);
+
+/* ---- Trace bodies (immutable, content-addressed) ---------------------- */
+
+/* Store a canonical trace body: write to tmp/, fsync, atomic-rename to
+ * trace/<pp>/<trace-id>. Idempotent. *out_trace_id = BLAKE2b(body). */
+int build_store_put_trace(BuildStore*, const uint8_t* body, size_t len,
+ uint8_t out_trace_id[BUILD_HASH_LEN]);
+
+/* Load a trace body's bytes by id (borrowed via file_io; release with
+ * build_store_release). Returns BUILD_ERR (absent) on a miss — never fatal. */
+int build_store_get_trace(BuildStore*, const uint8_t trace_id[BUILD_HASH_LEN],
+ KitFileData* out);
+void build_store_release(BuildStore*, KitFileData*);
+
+/* ---- Target record (mutable, the only ordering-sensitive object) ------ */
+
+/* Load a target's record by key. A missing or unparseable record yields an
+ * empty record (BUILD_OK with n_rows == 0) — fail safe. */
+int build_store_record_load(BuildStore*,
+ const uint8_t target_key[BUILD_HASH_LEN],
+ KitSlice target_name, BuildTargetRecord* out);
+
+/* Prepend (kind, trace_id) to the target's record and commit it: read-modify-
+ * write into tmp/ then atomic rename, dedup + truncate to KIT_BUILD_RECORD_CAP.
+ * Takes the optional advisory lock around the RMW when the host provides one.
+ */
+int build_store_record_update(BuildStore*,
+ const uint8_t target_key[BUILD_HASH_LEN],
+ KitSlice target_name, BuildTraceKind kind,
+ const uint8_t trace_id[BUILD_HASH_LEN]);
+
+/* ---- Tree cache (materialized output directories) --------------------- */
+
+/* If tree-id is already materialized under cache/, write its path and return
+ * BUILD_OK; otherwise return BUILD_ERR (caller climbs the materialization
+ * ladder). */
+int build_store_cache_lookup(const BuildStore*,
+ const uint8_t tree_id[BUILD_HASH_LEN],
+ char* path_out, size_t cap);
+
+/* Materialize a stored CAS tree into cache/<pp>/<tree-id>/ (verifying each
+ * blob), staged in tmp/ then atomically renamed; write its path. Returns
+ * BUILD_ERR if the tree's bytes are not in the CAS (caller may fetch/rebuild).
+ */
+int build_store_cache_materialize(BuildStore*,
+ const uint8_t tree_id[BUILD_HASH_LEN],
+ char* path_out, size_t cap);
+
+/* Ingest a recipe's output directory into the CAS as a tree, then install it
+ * into the cache; *out_tree_id gets the tree id and `path_out` its cache path.
+ * Wraps kit_cas_add_tree_from_dir + cache install (atomic rename). */
+int build_store_ingest_output(BuildStore*, const char* out_dir,
+ uint8_t out_tree_id[BUILD_HASH_LEN],
+ char* path_out, size_t cap);
+
+/* ---- Recipe sandboxes -------------------------------------------------- */
+
+/* Create a fresh sandbox under build/tmp/run-<n>/ with an empty out/ subdir;
+ * write the sandbox root and the out/ path. Removed by build_store_sandbox_done
+ * on success; orphaned (and swept later) on failure/crash. */
+int build_store_sandbox_new(BuildStore*, char* sandbox_out, size_t sandbox_cap,
+ char* out_dir_out, size_t out_cap);
+void build_store_sandbox_done(BuildStore*, const char* sandbox);
+
+#endif
diff --git a/src/build/trace.h b/src/build/trace.h
@@ -0,0 +1,185 @@
+#ifndef KIT_BUILD_TRACE_H
+#define KIT_BUILD_TRACE_H
+
+#include <kit/core.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "build.h"
+
+/*
+ * Trace bodies and the per-target record: the immutable, content-addressed
+ * heart of the cache. Pure byte-level logic (emit / parse / id), no I/O — the
+ * store persists and loads these; resolution matches against them.
+ *
+ * Both trace kinds are strict, byte-stable, INI-style text (a version line,
+ * ordered scalar fields, then sorted sections). Unknown keys/sections,
+ * duplicate rows, and non-canonical ordering are PARSE ERRORS, and a parse
+ * error is treated as ABSENT, never as a match — fail safe. All ids are
+ * lowercase hex; config values, source bytes, and glob expansions are stored by
+ * hash, never inline, so a body's size is independent of its inputs' sizes.
+ *
+ * The variable-length sections hang off caller-provided, growable buffers (the
+ * DistTree pattern): no VLAs, and the api layer owns sizing.
+ */
+
+/* One source leaf: a file read directly, kept BY HASH (verified, never
+ * replayed). Sorted by path in a body. `absent` records that the path did NOT
+ * exist when read (a legitimate, cacheable observation): `blob` is unused and
+ * creating the file later busts the trace, mirroring config-consumed-while-unset.
+ * On the wire an absent leaf is "<path> -" (the blob field a lone '-'). */
+typedef struct BuildSourceLeaf {
+ char path[BUILD_PATH_MAX];
+ uint8_t blob[BUILD_HASH_LEN];
+ int absent; /* 1 => path was absent; blob ignored */
+} BuildSourceLeaf;
+
+/* One glob leaf: a pattern and the BLAKE2b of its canonical (path, blob-id)
+ * listing. One row covers the existence AND content of the whole match set, so
+ * files read through a glob need no separate source rows. Sorted by pattern. */
+typedef struct BuildGlobLeaf {
+ char pattern[BUILD_PATTERN_MAX];
+ uint8_t result_hash[BUILD_HASH_LEN];
+} BuildGlobLeaf;
+
+/* One target-dep edge (shallow only): replays one `need`. Carries the dep's
+ * config-id AND argv-id (so it re-resolves under the exact (config, argv) it
+ * used — the full resolution identity) and the output tree-id it produced.
+ * Sorted by (name, config-id, argv-id). */
+typedef struct BuildDepEdge {
+ char name[BUILD_TARGET_MAX];
+ uint8_t config_id[BUILD_HASH_LEN];
+ uint8_t argv_id[BUILD_HASH_LEN];
+ uint8_t output_tree[BUILD_HASH_LEN];
+} BuildDepEdge;
+
+/* One consumed propagated-key NAME (shallow only). A key is "consumed" whether
+ * it was set or unset; its value lives in the config-id map this trace
+ * references (absent there == consumed-while-unset, so adding it later busts
+ * the trace). Sorted. */
+typedef struct BuildConfigKey {
+ char name[BUILD_KEY_MAX];
+} BuildConfigKey;
+
+/* The shallow trace: direct inputs + direct deps, for fine-grained rechecking.
+ * `config` is this target's effective propagated config-id (full map blob);
+ * `argv` is its serialized local-config blob id. */
+typedef struct BuildShallowTrace {
+ char target[BUILD_TARGET_MAX];
+ uint8_t recipe[BUILD_HASH_LEN];
+ uint8_t output[BUILD_HASH_LEN];
+ uint8_t config[BUILD_HASH_LEN];
+ uint8_t argv[BUILD_HASH_LEN];
+ BuildConfigKey* config_keys; /* [config] section */
+ size_t n_config_keys;
+ size_t cap_config_keys;
+ BuildSourceLeaf* sources; /* [source] section */
+ size_t n_sources;
+ size_t cap_sources;
+ BuildGlobLeaf* globs; /* [glob] section */
+ size_t n_globs;
+ size_t cap_globs;
+ BuildDepEdge* deps; /* [dep] section */
+ size_t n_deps;
+ size_t cap_deps;
+} BuildShallowTrace;
+
+/* The deep trace: the (root config-id, argv) binding to an output, plus a single
+ * pointer to the TRANSITIVE input closure, held structurally as a deepset DAG
+ * (below) rather than inlined. If the request's config-id equals `root_config`,
+ * its argv-id equals `argv`, and refreshing the `deepset` closure finds nothing
+ * moved (every source/glob leaf still matches AND every node's recipe-id still
+ * recomputes the same), determinism makes the output identical — no recipe runs.
+ * The deepset blob must be present to refresh; if absent (GC'd), the trace is
+ * treated as absent and resolution falls through (fail-safe). */
+typedef struct BuildDeepTrace {
+ char target[BUILD_TARGET_MAX];
+ uint8_t recipe[BUILD_HASH_LEN];
+ uint8_t output[BUILD_HASH_LEN];
+ uint8_t root_config[BUILD_HASH_LEN];
+ uint8_t argv[BUILD_HASH_LEN];
+ uint8_t deepset[BUILD_HASH_LEN]; /* deep-set-id of the closure (a CAS blob) */
+} BuildDeepTrace;
+
+/* One node of the transitive input-closure DAG — the structural form of the old
+ * flat leafset ("compact deep traces"). One node per target build: its DIRECT
+ * source/glob leaves, its recipe-id, and the deep-set-ids of its DIRECT deps.
+ * The fully-flattened closure is the recursive union, NEVER materialized: a
+ * subtree reached through many parents is one node, stored once — a Merkle DAG
+ * that structurally shares like the CAS. Building a parent is "concat my direct
+ * leaves + child ids, hash"; that IS the union. deep-set-id = BLAKE2b(canonical
+ * body), which is exactly a CAS blob id: a deepset is SELF-VERIFYING content,
+ * stored in cas/blob/ (trustless, fetchable, bundled as a plain blob), unlike a
+ * trace, which is a CLAIM in build/trace/. `target` is carried so refresh can
+ * recompute the node's recipe-id through the live definition — catching a dep
+ * repointed to a different-content recipe, while treating same-content as
+ * unchanged. Sorted: [source] by path, [glob] by pattern, [child] by id. */
+typedef struct BuildDeepSet {
+ char target[BUILD_TARGET_MAX];
+ uint8_t recipe[BUILD_HASH_LEN];
+ BuildSourceLeaf* sources; /* this node's DIRECT source leaves */
+ size_t n_sources;
+ size_t cap_sources;
+ BuildGlobLeaf* globs; /* this node's DIRECT glob leaves */
+ size_t n_globs;
+ size_t cap_globs;
+ uint8_t (*children)[BUILD_HASH_LEN]; /* direct deps' deep-set-ids, sorted */
+ size_t n_children;
+ size_t cap_children;
+} BuildDeepSet;
+
+/* One row of the mutable target record. */
+typedef struct BuildRecordRow {
+ uint8_t kind; /* BuildTraceKind */
+ uint8_t trace_id[BUILD_HASH_LEN];
+} BuildRecordRow;
+
+/* The mutable per-target trace set: candidates newest-first, each kind capped
+ * at KIT_BUILD_RECORD_CAP. Points at trace bodies in build/trace/; a row may
+ * dangle (point at a GC'd body) and readers treat that as absent. */
+typedef struct BuildTargetRecord {
+ char target[BUILD_TARGET_MAX];
+ BuildRecordRow* rows;
+ size_t n_rows;
+ size_t cap_rows;
+} BuildTargetRecord;
+
+/* Emit canonical bytes. Sorting/dedup/validation of the sections happens here;
+ * a malformed trace (e.g. a path that fails validation) returns BUILD_ERR with
+ * detail in err. */
+int build_shallow_emit(const BuildShallowTrace*, KitWriter* out, char* err,
+ size_t errcap);
+int build_deep_emit(const BuildDeepTrace*, KitWriter* out, char* err,
+ size_t errcap);
+int build_deepset_emit(const BuildDeepSet*, KitWriter* out, char* err,
+ size_t errcap);
+int build_record_emit(const BuildTargetRecord*, KitWriter* out, char* err,
+ size_t errcap);
+
+/* Parse canonical bytes into a value (sections fill the caller-provided
+ * buffers). Non-canonical input is a parse error -> BUILD_ERR (treated as
+ * absent by callers). */
+int build_shallow_parse(const uint8_t* data, size_t len, BuildShallowTrace* out,
+ char* err, size_t errcap);
+int build_deep_parse(const uint8_t* data, size_t len, BuildDeepTrace* out,
+ char* err, size_t errcap);
+int build_deepset_parse(const uint8_t* data, size_t len, BuildDeepSet* out,
+ char* err, size_t errcap);
+int build_record_parse(const uint8_t* data, size_t len, BuildTargetRecord* out,
+ char* err, size_t errcap);
+
+/* trace-id = BLAKE2b(canonical trace body). The deep-set-id is the same hash
+ * over a deepset body — and equals the CAS blob id of those bytes, since a
+ * deepset is stored as an ordinary CAS blob (build_deepset_id is provided for
+ * symmetry; kit_blob_info over the emitted body yields the identical id). */
+void build_trace_id(const uint8_t* body, size_t len,
+ uint8_t out[BUILD_HASH_LEN]);
+void build_deepset_id(const uint8_t* body, size_t len,
+ uint8_t out[BUILD_HASH_LEN]);
+
+/* Prepend `row` to `rec` (newest-first), drop a duplicate of the same
+ * (kind, trace_id), and truncate that kind to KIT_BUILD_RECORD_CAP. Used by the
+ * record read-modify-write. */
+int build_record_prepend(BuildTargetRecord* rec, const BuildRecordRow* row);
+
+#endif