coord.h (14796B)
1 #ifndef KIT_BUILD_COORD_H 2 #define KIT_BUILD_COORD_H 3 4 #include <kit/build_coord.h> 5 #include <kit/cas.h> 6 #include <kit/core.h> 7 #include <stddef.h> 8 #include <stdint.h> 9 10 #include "build.h" 11 #include "cfg.h" 12 #include "defn.h" 13 #include "store.h" 14 #include "trace.h" 15 #include "workspace.h" 16 17 /* 18 * The coordinator context and its process-lifetime in-memory state. All state 19 * hangs off one KitBuildCoordinator (no globals — the project rule). For the 20 * life of the process it memoizes every base-input probe and every target 21 * resolution, so a diamond in the graph is built once and a source file is 22 * hashed once. "Cached over the life of the coordinator" means exactly these 23 * memo tables: the coordinator assumes the workspace does not change under it 24 * mid-build. 25 * 26 * This header owns the coordinator struct, the resolution RESULT type the 27 * targets memo stores, the base-input memo accessors, and the concurrency 28 * primitives (jobs semaphore + per-target futures) built on the optional 29 * KitBuildSched. The resolution ALGORITHM is resolve.h; running a recipe is 30 * runner.h. 31 */ 32 33 /* The in-memory deepset node: one node of the transitive input-closure DAG that 34 * a resolution returns and the targets memo caches (trace.h BuildDeepSet is its 35 * on-disk/CAS-blob form). It carries this node's DIRECT source/glob/blob leaves, 36 * its target-scope projected config observations, its target + recipe-id, and 37 * pointers to its children's (interned) nodes. `id` is 38 * the deep-set-id — the CAS blob id of the emitted node — so an unchanged 39 * subtree is recognized by id equality without descending. Folded into a parent 40 * by build_leafset_union and refreshed (against the live workspace) by the deep 41 * fast path. Coordinator-owned: interned once per id and stable for the process 42 * (a node reached through many parents is one object — the Merkle DAG sharing). */ 43 typedef struct BuildLeafSet { 44 uint8_t id[BUILD_HASH_LEN]; /* deep-set-id == CAS blob id of this node */ 45 char target[BUILD_TARGET_MAX]; 46 uint8_t recipe[BUILD_HASH_LEN]; /* recompute via defn to detect a repoint */ 47 BuildConfigLeaf* configs; /* target-scope config observations */ 48 size_t n_configs; 49 BuildSourceLeaf* sources; /* DIRECT leaves of this node */ 50 size_t n_sources; 51 BuildGlobLeaf* globs; 52 size_t n_globs; 53 BuildBlobLeaf* blobs; 54 size_t n_blobs; 55 const struct BuildLeafSet** children; /* direct deps' interned nodes */ 56 size_t n_children; 57 } BuildLeafSet; 58 59 typedef enum BuildActionKind { 60 BUILD_ACTION_BUILD = 0, 61 BUILD_ACTION_TEST = 1, 62 } BuildActionKind; 63 64 /* A completed resolution: what a `need` returns and what the targets memo 65 * caches. `path` is a materialized output directory; `leafset` is borrowed from 66 * the coordinator and lives for the process. */ 67 typedef struct BuildResolved { 68 uint8_t output_tree[BUILD_HASH_LEN]; 69 char path[BUILD_PATH_MAX]; 70 const BuildLeafSet* leafset; 71 } BuildResolved; 72 73 typedef struct BuildTestResolved { 74 KitTestStatus status; 75 int exit_code; 76 uint8_t result_tree[BUILD_HASH_LEN]; 77 char path[BUILD_PATH_MAX]; 78 uint8_t stdout_blob[BUILD_HASH_LEN]; 79 uint8_t stderr_blob[BUILD_HASH_LEN]; 80 const BuildLeafSet* leafset; 81 } BuildTestResolved; 82 83 /* Opaque process-lifetime memo tables and the per-target future table. */ 84 typedef struct BuildSourceMemo BuildSourceMemo; /* path -> blob-id (+ stat) */ 85 typedef struct BuildGlobMemo BuildGlobMemo; /* pattern -> glob result */ 86 typedef struct BuildConfigMemo BuildConfigMemo; /* config-id -> map */ 87 typedef struct BuildArgvMemo BuildArgvMemo; /* argv-id -> vector */ 88 typedef struct BuildDeepSetMemo 89 BuildDeepSetMemo; /* deep-set-id -> {interned node, refresh-valid?} */ 90 typedef struct BuildPulledSet 91 BuildPulledSet; /* target-name -> already trace-remote-pulled */ 92 typedef struct BuildTargetTable 93 BuildTargetTable; /* (name,cfg-id,argv-id) -> future */ 94 typedef struct BuildTargetFuture BuildTargetFuture; 95 typedef struct BuildExternalWorkspace BuildExternalWorkspace; 96 97 struct KitBuildCoordinator { 98 KitContext ctx_storage; /* private copy so the handle outlives caller's ctx */ 99 const KitContext* ctx; 100 KitBuildHost host; /* borrowed vtables */ 101 KitBuildOptions opts; /* borrowed strings */ 102 char workspace_root[BUILD_PATH_MAX]; 103 char store_root[BUILD_PATH_MAX]; 104 char cas_root[BUILD_PATH_MAX]; 105 char build_def_name[BUILD_PATH_MAX]; 106 BuildWorkspace root_workspace; 107 BuildExternalWorkspace* externals; 108 KitCas* cas; /* the shared content store */ 109 BuildStore store; /* paths + record RMW + tree cache over build/ */ 110 int jobs_limit; /* effective parallelism (1 when host->sched NULL) */ 111 /* Process-lifetime memos. */ 112 BuildSourceMemo* sources; 113 BuildGlobMemo* globs; 114 BuildConfigMemo* configs; 115 BuildArgvMemo* argvs; 116 BuildDeepSetMemo* deepsets; /* interned closure nodes + refresh-valid cache */ 117 BuildPulledSet* pulled; /* targets a trace-remote pull was attempted for */ 118 BuildTargetTable* targets; 119 /* Concurrency state (NULL/no-op when host->sched is NULL). */ 120 KitBuildMutex* lock; /* guards the memos + targets table + stats */ 121 void* jobs_sem; /* opaque counting semaphore over host->sched */ 122 KitBuildStats stats; /* cumulative resolution counters; bumped under `lock` */ 123 }; 124 125 /* Bump one resolution counter (under `lock`); resolve/runner/remote/bundle call 126 * this so kit_build_stats can report per-phase work. */ 127 typedef enum BuildStatField { 128 BUILD_STAT_DEEP_HIT = 0, 129 BUILD_STAT_SHALLOW_HIT, 130 BUILD_STAT_RECIPE_RUN, 131 BUILD_STAT_MATERIALIZE_MISS, 132 BUILD_STAT_OBJECT_FETCH, 133 BUILD_STAT_TRACE_PULL, 134 BUILD_STAT_TEST_RUN, 135 BUILD_STAT_TEST_CACHE_HIT, 136 BUILD_STAT_TEST_FAILURE, 137 } BuildStatField; 138 void build_coord_stat_bump(KitBuildCoordinator*, BuildStatField); 139 void build_coord_tracef(KitBuildCoordinator*, const char* fmt, ...); 140 141 /* Open/close. open records the workspace/package build-file name, opens the CAS 142 * and store, allocates the memos, and sizes the jobs semaphore (clamped to 1 143 * when the host provides no sched). Package BUILD.kit files are loaded on 144 * demand by target resolution. */ 145 KitStatus build_coord_open(const KitContext*, const KitBuildHost*, 146 KitSlice store_root, const KitBuildOptions*, 147 KitBuildCoordinator** out); 148 void build_coord_close(KitBuildCoordinator*); 149 150 /* ---- Base-input probes (memoized) ------------------------------------- */ 151 152 /* Hash a source file by workspace path. *present is 0 when the file is absent 153 * (absence is a legitimate, cacheable observation). Memoized per process. */ 154 int build_coord_source_hash(KitBuildCoordinator*, KitSlice path, 155 uint8_t out_blob[BUILD_HASH_LEN], int* present); 156 int build_coord_source_hash_target(KitBuildCoordinator*, KitSlice target, 157 KitSlice path, 158 uint8_t out_blob[BUILD_HASH_LEN], 159 int* present); 160 161 /* Fetch a pinned blob into the CAS if absent, trying URL hints in order. Returns 162 * the verified local CAS blob path in path_out. The URL list is untrusted and 163 * not part of cache identity; only expected_blob is trusted after verification. 164 */ 165 int build_coord_fetch_blob(KitBuildCoordinator*, 166 const uint8_t expected_blob[BUILD_HASH_LEN], 167 const KitSlice* urls, size_t nurls, char* path_out, 168 size_t path_cap); 169 170 /* Expand a glob and return its glob-result-hash; optionally enumerate the 171 * sorted matches via cb. Memoized per process. */ 172 typedef int (*BuildCoordGlobFn)(void* user, const char* path, 173 const uint8_t blob[BUILD_HASH_LEN]); 174 int build_coord_glob(KitBuildCoordinator*, KitSlice pattern, 175 uint8_t out_result_hash[BUILD_HASH_LEN], 176 BuildCoordGlobFn cb, void* cb_user); 177 int build_coord_glob_target(KitBuildCoordinator*, KitSlice target, 178 KitSlice pattern, 179 uint8_t out_result_hash[BUILD_HASH_LEN], 180 BuildCoordGlobFn cb, void* cb_user); 181 182 /* Recover a need-overlay map / an argv vector by its id (loads + parses the 183 * serialized CAS blob, memoized) — the replay path that applies the overlay to 184 * the current parent config and re-resolves the dep with its recorded argv. */ 185 int build_coord_config_by_id(KitBuildCoordinator*, 186 const uint8_t config_id[BUILD_HASH_LEN], 187 BuildConfig* out); 188 int build_coord_argv_by_id(KitBuildCoordinator*, 189 const uint8_t argv_id[BUILD_HASH_LEN], 190 BuildArgv* out); 191 192 int build_coord_top_config(KitBuildCoordinator*, const KitBuildKV* overrides, 193 size_t noverrides, BuildConfig* out); 194 195 typedef struct BuildRecipeResolution { 196 char canonical_target[BUILD_TARGET_MAX]; 197 char repo[BUILD_KEY_MAX]; 198 char package[BUILD_PATH_MAX]; 199 char local_name[BUILD_TARGET_MAX]; 200 char workspace_root[BUILD_PATH_MAX]; 201 char recipe_relpath[BUILD_PATH_MAX]; 202 char recipe_abspath[BUILD_PATH_MAX]; 203 } BuildRecipeResolution; 204 205 /* Canonicalize a label against a package. */ 206 int build_coord_canonical_target(KitBuildCoordinator*, KitSlice label, 207 KitSlice current_repo, 208 KitSlice current_package, 209 char out[BUILD_TARGET_MAX]); 210 211 /* Resolve a canonical target through its live package BUILD.kit and redo 212 * defaults. The returned recipe path is workspace-relative and absolute. */ 213 int build_coord_resolve_recipe(KitBuildCoordinator*, KitSlice canonical_target, 214 BuildRecipeResolution* out); 215 216 /* recipe-id = BLAKE2b(recipe file bytes), from the target's definition stanza. 217 */ 218 int build_coord_recipe_id(KitBuildCoordinator*, KitSlice target, 219 uint8_t out[BUILD_HASH_LEN]); 220 221 /* The effective argv is the request's argv (empty when none is supplied) — no 222 * definition lookup is involved; build it with build_argv_set and hash it with 223 * build_argv_id. */ 224 225 /* ---- Deepset nodes (process-lived closure DAG) ------------------------- */ 226 227 /* Intern a deepset node into coordinator-owned storage so it lives for the 228 * process and is shared by id: BuildResolved.leafset is borrowed-for-the- 229 * process, so a node built transiently (build_leafset_union folds children; 230 * a Phase-2 hit rebuilds one) is copied here, deduped by `id`, before it is 231 * handed back. *out is stable until build_coord_close. */ 232 int build_coord_leafset_intern(KitBuildCoordinator*, const BuildLeafSet* in, 233 const BuildLeafSet** out); 234 235 /* Load the deepset node named by a deep-set-id: read its CAS blob 236 * (kit_cas_get_blob), parse (build_deepset_parse), and recursively load+intern 237 * its children, returning a fully-linked, interned node. Memoized by id (a node 238 * reached through many parents loads once). Returns BUILD_ERR if the blob — or 239 * any child blob — is absent, so Phase 1 treats the deep trace as absent and 240 * falls through (fail-safe). The replay path for the deep fast path. */ 241 int build_coord_deepset_load(KitBuildCoordinator*, 242 const uint8_t deepset_id[BUILD_HASH_LEN], 243 const BuildLeafSet** out); 244 245 /* Refresh-validity slot over the deepset DAG (keyed by deep-set-id). Kept for 246 * coordinator-local callers that can prove the same validation context; config- 247 * aware deep refresh does not use this process-wide slot because validity 248 * depends on the current propagated config. */ 249 int build_coord_deepset_valid_get(KitBuildCoordinator*, 250 const uint8_t deepset_id[BUILD_HASH_LEN], 251 int* known, int* valid); 252 void build_coord_deepset_valid_set(KitBuildCoordinator*, 253 const uint8_t deepset_id[BUILD_HASH_LEN], 254 int valid); 255 256 /* ---- Shared traces (lazy pull) ---------------------------------------- */ 257 258 /* On a local-record miss for `target`, attempt ONE trace-remote pull (delegates 259 * to bundle.h build_trace_remote_pull, which verifies signature/trust and 260 * installs trace bodies + referenced blobs). Idempotent per target via the 261 * `pulled` set: a second miss does not re-pull. *pulled_now reports whether new 262 * traces were installed (so resolve re-scans the record). No-op (BUILD_OK, 263 * *pulled_now=0) when no trace-remotes are configured. */ 264 int build_coord_trace_remote_pull_once(KitBuildCoordinator*, KitSlice target, 265 int* pulled_now); 266 267 /* ---- Concurrency: jobs semaphore + per-target futures ----------------- */ 268 269 /* Bound concurrently *running* recipe processes to jobs_limit. The slot counts 270 * a recipe only while it is actively running: a recipe blocked servicing a 271 * `need` releases its slot (runner.h) and reacquires once the sub-build 272 * returns, so a dependency chain deeper than jobs_limit cannot deadlock. No-ops 273 * (always succeed immediately) in sequential mode. */ 274 void build_coord_jobs_acquire(KitBuildCoordinator*); 275 void build_coord_jobs_release(KitBuildCoordinator*); 276 277 /* Run `fn(arg)` on a fresh worker thread (host->sched->thread_spawn), tracked by 278 * the coordinator and joined at build completion — the substrate for concurrent 279 * `need` dispatch (build_dispatch, resolve.h). Returns BUILD_ERR when there is no 280 * sched (sequential mode), so the caller runs `fn` inline instead. The worker's 281 * sync point is the target future it completes, not a join. */ 282 int build_coord_spawn(KitBuildCoordinator*, void (*fn)(void*), void* arg); 283 284 /* Intern a future for (target, config-id, argv-id): the cross-path memo AND 285 * in-flight dedup. On return *is_fresh == 1 means this caller owns the 286 * resolution and must complete or fail the future; *is_fresh == 0 means another 287 * resolution is (or was) in flight and the caller should await it. In 288 * sequential mode a fresh future resolves inline before any other caller can 289 * observe it. */ 290 int build_coord_target_intern(KitBuildCoordinator*, KitSlice target, 291 const uint8_t config_id[BUILD_HASH_LEN], 292 const uint8_t argv_id[BUILD_HASH_LEN], 293 BuildTargetFuture** out, int* is_fresh); 294 /* Block until the future is completed or failed; copies the result out. */ 295 int build_coord_target_await(KitBuildCoordinator*, BuildTargetFuture*, 296 BuildResolved* out); 297 void build_coord_target_complete(KitBuildCoordinator*, BuildTargetFuture*, 298 const BuildResolved*); 299 void build_coord_target_fail(KitBuildCoordinator*, BuildTargetFuture*); 300 301 #endif