store.c (16160B)
1 #include "store.h" 2 3 #include <stdio.h> 4 #include <string.h> 5 6 #undef KIT_TRACE_MODULE 7 #define KIT_TRACE_MODULE "build/store" 8 9 static KitSlice str_slice(const char* s) { return kit_slice_cstr(s); } 10 11 static int path_set(char* out, size_t cap, KitSlice s) { 12 if (!out || cap == 0u || !s.s || s.len + 1u > cap) return BUILD_ERR; 13 memcpy(out, s.s, s.len); 14 out[s.len] = '\0'; 15 return BUILD_OK; 16 } 17 18 static int path_join2(char* out, size_t cap, const char* a, const char* b) { 19 size_t na, nb; 20 int need_sep; 21 if (!out || cap == 0u || !a || !b) return BUILD_ERR; 22 na = strlen(a); 23 nb = strlen(b); 24 need_sep = na > 0u && a[na - 1u] != '/'; 25 if (na + (need_sep ? 1u : 0u) + nb + 1u > cap) return BUILD_ERR; 26 memcpy(out, a, na); 27 if (need_sep) out[na++] = '/'; 28 memcpy(out + na, b, nb); 29 out[na + nb] = '\0'; 30 return BUILD_OK; 31 } 32 33 static int path_parent(const char* path, char* out, size_t cap) { 34 size_t n, i; 35 if (!path || !out || cap == 0u) return BUILD_ERR; 36 n = strlen(path); 37 for (i = n; i > 0u; --i) { 38 if (path[i - 1u] == '/') { 39 size_t len = i - 1u; 40 if (len == 0u) len = 1u; 41 if (len + 1u > cap) return BUILD_ERR; 42 memcpy(out, path, len); 43 out[len] = '\0'; 44 return BUILD_OK; 45 } 46 } 47 if (cap < 2u) return BUILD_ERR; 48 out[0] = '.'; 49 out[1] = '\0'; 50 return BUILD_OK; 51 } 52 53 static int id_path(const BuildStore* s, const char* kind, 54 const uint8_t id[BUILD_HASH_LEN], char* out, size_t cap) { 55 char hex[BUILD_HEX_LEN]; 56 char pp[BUILD_PP_LEN + 1u]; 57 char rel[BUILD_PATH_MAX]; 58 kit_hex_encode(hex, id, BUILD_HASH_LEN); 59 pp[0] = hex[0]; 60 pp[1] = hex[1]; 61 pp[2] = '\0'; 62 if (snprintf(rel, sizeof rel, "%s/%s/%s", kind, pp, hex) >= 63 (int)sizeof rel) 64 return BUILD_ERR; 65 return path_join2(out, cap, s->root, rel); 66 } 67 68 static int ensure_dir(const BuildStore* s, const char* path) { 69 if (!s || !s->cas_host || !s->cas_host->mkdir_p || !path) return BUILD_ERR; 70 return s->cas_host->mkdir_p(s->cas_host->user, path) == 0 ? BUILD_OK 71 : BUILD_ERR; 72 } 73 74 static int ensure_parent(const BuildStore* s, const char* path) { 75 char parent[BUILD_PATH_MAX]; 76 if (path_parent(path, parent, sizeof parent) != BUILD_OK) return BUILD_ERR; 77 return ensure_dir(s, parent); 78 } 79 80 static int sync_path(const BuildStore* s, const char* path) { 81 if (!s || !s->io || !s->io->sync_path || !path) return BUILD_OK; 82 return s->io->sync_path(s->io->user, str_slice(path)) == 0 ? BUILD_OK 83 : BUILD_ERR; 84 } 85 86 static int remove_path(const BuildStore* s, const char* path, int recursive) { 87 if (!s || !s->io || !s->io->remove || !path) return BUILD_ERR; 88 return s->io->remove(s->io->user, str_slice(path), recursive) == 0 89 ? BUILD_OK 90 : BUILD_ERR; 91 } 92 93 static int make_tmp_dir(const BuildStore* s, char* out, size_t cap) { 94 char tmp_parent[BUILD_PATH_MAX]; 95 if (!s || !s->io || !s->io->make_temp_dir) return BUILD_ERR; 96 if (path_join2(tmp_parent, sizeof tmp_parent, s->root, "tmp") != BUILD_OK) 97 return BUILD_ERR; 98 if (ensure_dir(s, tmp_parent) != BUILD_OK) return BUILD_ERR; 99 if (s->io->make_temp_dir(s->io->user, str_slice(tmp_parent), out, cap) != 0) 100 return BUILD_ERR; 101 return BUILD_OK; 102 } 103 104 static int write_file(const BuildStore* s, const char* path, 105 const uint8_t* data, size_t len) { 106 const KitFileIO* fio; 107 KitWriter* w = NULL; 108 KitStatus st; 109 if (!s || !s->cas_host || !s->cas_host->file_io || !path) return BUILD_ERR; 110 fio = s->cas_host->file_io; 111 if (!fio->open_writer) return BUILD_ERR; 112 if (fio->open_writer(fio->user, path, &w) != KIT_OK || !w) return BUILD_ERR; 113 st = len ? kit_writer_write(w, data, len) : KIT_OK; 114 if (st == KIT_OK) st = kit_writer_status(w); 115 kit_writer_close(w); 116 return st == KIT_OK ? BUILD_OK : BUILD_ERR; 117 } 118 119 static int atomic_write_file(BuildStore* s, const char* final_path, 120 const uint8_t* data, size_t len) { 121 char tmp_dir[BUILD_PATH_MAX]; 122 char tmp_file[BUILD_PATH_MAX]; 123 char parent[BUILD_PATH_MAX]; 124 int ok = BUILD_ERR; 125 if (!s || !s->io || !s->io->rename || !final_path) return BUILD_ERR; 126 if (ensure_parent(s, final_path) != BUILD_OK) return BUILD_ERR; 127 if (make_tmp_dir(s, tmp_dir, sizeof tmp_dir) != BUILD_OK) return BUILD_ERR; 128 if (path_join2(tmp_file, sizeof tmp_file, tmp_dir, "body") != BUILD_OK) 129 goto out; 130 if (write_file(s, tmp_file, data, len) != BUILD_OK) goto out; 131 if (sync_path(s, tmp_file) != BUILD_OK) goto out; 132 if (sync_path(s, tmp_dir) != BUILD_OK) goto out; 133 if (s->io->rename(s->io->user, str_slice(tmp_file), str_slice(final_path)) != 134 0) 135 goto out; 136 if (path_parent(final_path, parent, sizeof parent) == BUILD_OK) 137 (void)sync_path(s, parent); 138 ok = BUILD_OK; 139 out: 140 (void)remove_path(s, tmp_dir, 1); 141 return ok; 142 } 143 144 static int read_file(BuildStore* s, const char* path, KitFileData* out) { 145 const KitFileIO* fio; 146 if (!s || !s->cas_host || !s->cas_host->file_io || !path || !out) 147 return BUILD_ERR; 148 fio = s->cas_host->file_io; 149 out->data = NULL; 150 out->size = 0; 151 out->token = NULL; 152 return fio->read_all && fio->read_all(fio->user, path, out) == KIT_OK 153 ? BUILD_OK 154 : BUILD_ERR; 155 } 156 157 static void release_file(BuildStore* s, KitFileData* fd) { 158 const KitFileIO* fio; 159 if (!s || !fd || !s->cas_host || !s->cas_host->file_io) return; 160 fio = s->cas_host->file_io; 161 if (fd->data && fio->release) fio->release(fio->user, fd); 162 fd->data = NULL; 163 fd->size = 0; 164 fd->token = NULL; 165 } 166 167 static int read_verified_trace(BuildStore* s, 168 const uint8_t trace_id[BUILD_HASH_LEN], 169 KitFileData* out) { 170 char path[BUILD_PATH_MAX]; 171 uint8_t actual[BUILD_HASH_LEN]; 172 if (build_store_trace_path(s, trace_id, path, sizeof path) != BUILD_OK) 173 return BUILD_ERR; 174 if (read_file(s, path, out) != BUILD_OK) return BUILD_ERR; 175 build_trace_id(out->data, out->size, actual); 176 if (!build_id_eq(actual, trace_id)) { 177 release_file(s, out); 178 return BUILD_ERR; 179 } 180 return BUILD_OK; 181 } 182 183 static int copy_target(KitSlice name, char out[BUILD_TARGET_MAX]) { 184 if (!name.s || name.len == 0u || name.len >= BUILD_TARGET_MAX) 185 return BUILD_ERR; 186 memcpy(out, name.s, name.len); 187 out[name.len] = '\0'; 188 return BUILD_OK; 189 } 190 191 static void record_empty(KitSlice target_name, BuildTargetRecord* out) { 192 if (!out) return; 193 out->n_rows = 0; 194 memset(out->target, 0, sizeof out->target); 195 if (target_name.s && target_name.len < BUILD_TARGET_MAX) { 196 memcpy(out->target, target_name.s, target_name.len); 197 out->target[target_name.len] = '\0'; 198 } 199 } 200 201 static int ignore_dir_entry(void* user, KitSlice name) { 202 (void)user; 203 (void)name; 204 return 0; 205 } 206 207 static int cache_dir_confirmed(const BuildStore* s, const char* path) { 208 if (!s || !s->io || !s->io->list_dir || !path) return BUILD_ERR; 209 return s->io->list_dir(s->io->user, str_slice(path), ignore_dir_entry, 210 NULL) == 0 211 ? BUILD_OK 212 : BUILD_ERR; 213 } 214 215 int build_store_open(const KitContext* ctx, KitCas* cas, 216 const KitBuildStoreIo* io, const KitCasHost* cas_host, 217 KitSlice store_root, BuildStore* out) { 218 char root[BUILD_PATH_MAX]; 219 char path[BUILD_PATH_MAX]; 220 if (!ctx || !cas || !io || !cas_host || !cas_host->file_io || !out) 221 return BUILD_ERR; 222 memset(out, 0, sizeof *out); 223 if (path_set(root, sizeof root, store_root) != BUILD_OK) return BUILD_ERR; 224 if (path_join2(out->root, sizeof out->root, root, "build") != BUILD_OK) 225 return BUILD_ERR; 226 out->ctx = ctx; 227 out->cas = cas; 228 out->io = io; 229 out->cas_host = cas_host; 230 if (ensure_dir(out, out->root) != BUILD_OK) return BUILD_ERR; 231 if (path_join2(path, sizeof path, out->root, "trace") != BUILD_OK || 232 ensure_dir(out, path) != BUILD_OK) 233 return BUILD_ERR; 234 if (path_join2(path, sizeof path, out->root, "target") != BUILD_OK || 235 ensure_dir(out, path) != BUILD_OK) 236 return BUILD_ERR; 237 if (path_join2(path, sizeof path, out->root, "cache") != BUILD_OK || 238 ensure_dir(out, path) != BUILD_OK) 239 return BUILD_ERR; 240 if (path_join2(path, sizeof path, out->root, "tmp") != BUILD_OK || 241 ensure_dir(out, path) != BUILD_OK) 242 return BUILD_ERR; 243 return BUILD_OK; 244 } 245 246 int build_store_trace_path(const BuildStore* s, 247 const uint8_t trace_id[BUILD_HASH_LEN], char* out, 248 size_t cap) { 249 if (!s || !trace_id) return BUILD_ERR; 250 return id_path(s, "trace", trace_id, out, cap); 251 } 252 253 int build_store_target_path(const BuildStore* s, 254 const uint8_t target_key[BUILD_HASH_LEN], 255 char* out, size_t cap) { 256 if (!s || !target_key) return BUILD_ERR; 257 return id_path(s, "target", target_key, out, cap); 258 } 259 260 int build_store_cache_path(const BuildStore* s, 261 const uint8_t tree_id[BUILD_HASH_LEN], char* out, 262 size_t cap) { 263 if (!s || !tree_id) return BUILD_ERR; 264 return id_path(s, "cache", tree_id, out, cap); 265 } 266 267 int build_store_put_trace(BuildStore* s, const uint8_t* body, size_t len, 268 uint8_t out_trace_id[BUILD_HASH_LEN]) { 269 char path[BUILD_PATH_MAX]; 270 KitFileData existing; 271 if (!s || (!body && len) || !out_trace_id) return BUILD_ERR; 272 build_trace_id(body, len, out_trace_id); 273 kit_hex_encode(path, out_trace_id, BUILD_HASH_LEN); 274 KIT_LOGD("put trace %s bytes=%llu", path, (unsigned long long)len); 275 if (read_verified_trace(s, out_trace_id, &existing) == BUILD_OK) { 276 release_file(s, &existing); 277 return BUILD_OK; 278 } 279 if (build_store_trace_path(s, out_trace_id, path, sizeof path) != BUILD_OK) 280 return BUILD_ERR; 281 if (atomic_write_file(s, path, body, len) == BUILD_OK) return BUILD_OK; 282 if (read_verified_trace(s, out_trace_id, &existing) == BUILD_OK) { 283 release_file(s, &existing); 284 return BUILD_OK; 285 } 286 return BUILD_ERR; 287 } 288 289 int build_store_get_trace(BuildStore* s, 290 const uint8_t trace_id[BUILD_HASH_LEN], 291 KitFileData* out) { 292 char hex[BUILD_HEX_LEN]; 293 int r; 294 if (!s || !trace_id || !out) return BUILD_ERR; 295 kit_hex_encode(hex, trace_id, BUILD_HASH_LEN); 296 KIT_LOGD("get trace %s", hex); 297 r = read_verified_trace(s, trace_id, out); 298 KIT_LOGT("get trace %s %s", hex, r == BUILD_OK ? "hit" : "miss"); 299 return r; 300 } 301 302 void build_store_release(BuildStore* s, KitFileData* fd) { release_file(s, fd); } 303 304 int build_store_record_load(BuildStore* s, 305 const uint8_t target_key[BUILD_HASH_LEN], 306 KitSlice target_name, BuildTargetRecord* out) { 307 char path[BUILD_PATH_MAX]; 308 KitFileData fd; 309 char err[128]; 310 char target[BUILD_TARGET_MAX]; 311 if (!s || !target_key || !out) return BUILD_ERR; 312 if (copy_target(target_name, target) != BUILD_OK) return BUILD_ERR; 313 record_empty(target_name, out); 314 if (build_store_target_path(s, target_key, path, sizeof path) != BUILD_OK) 315 return BUILD_ERR; 316 if (read_file(s, path, &fd) != BUILD_OK) return BUILD_OK; 317 if (build_record_parse(fd.data, fd.size, out, err, sizeof err) != BUILD_OK || 318 strcmp(out->target, target) != 0) { 319 record_empty(target_name, out); 320 } 321 release_file(s, &fd); 322 return BUILD_OK; 323 } 324 325 int build_store_record_update(BuildStore* s, 326 const uint8_t target_key[BUILD_HASH_LEN], 327 KitSlice target_name, BuildTraceKind kind, 328 const uint8_t trace_id[BUILD_HASH_LEN]) { 329 BuildRecordRow rows[2u * KIT_BUILD_RECORD_CAP]; 330 BuildTargetRecord rec; 331 BuildRecordRow row; 332 KitBuildLock* lock = NULL; 333 char lock_key[BUILD_HEX_LEN]; 334 char path[BUILD_PATH_MAX]; 335 KitWriter* w = NULL; 336 const uint8_t* bytes; 337 size_t len; 338 char err[128]; 339 int ok = BUILD_ERR; 340 if (!s || !target_key || !trace_id) return BUILD_ERR; 341 kit_hex_encode(lock_key, target_key, BUILD_HASH_LEN); 342 if (s->io && s->io->lock) { 343 if (s->io->lock(s->io->user, str_slice(lock_key), &lock) != 0 || !lock) 344 return BUILD_ERR; 345 } 346 rec.rows = rows; 347 rec.n_rows = 0; 348 rec.cap_rows = sizeof rows / sizeof rows[0]; 349 if (build_store_record_load(s, target_key, target_name, &rec) != BUILD_OK) 350 goto out; 351 row.kind = (uint8_t)kind; 352 memcpy(row.trace_id, trace_id, BUILD_HASH_LEN); 353 if (build_record_prepend(&rec, &row) != BUILD_OK) goto out; 354 if (kit_writer_mem(s->ctx->heap, &w) != KIT_OK) goto out; 355 if (build_record_emit(&rec, w, err, sizeof err) != BUILD_OK || 356 kit_writer_status(w) != KIT_OK) 357 goto out; 358 bytes = kit_writer_mem_bytes(w, &len); 359 if (build_store_target_path(s, target_key, path, sizeof path) != BUILD_OK) 360 goto out; 361 if (atomic_write_file(s, path, bytes, len) != BUILD_OK) goto out; 362 ok = BUILD_OK; 363 out: 364 if (w) kit_writer_close(w); 365 if (lock && s->io && s->io->unlock) s->io->unlock(s->io->user, lock); 366 return ok; 367 } 368 369 int build_store_cache_lookup(const BuildStore* s, 370 const uint8_t tree_id[BUILD_HASH_LEN], 371 char* path_out, size_t cap) { 372 char path[BUILD_PATH_MAX]; 373 if (!s || !tree_id || !path_out) return BUILD_ERR; 374 if (build_store_cache_path(s, tree_id, path, sizeof path) != BUILD_OK) 375 return BUILD_ERR; 376 if (cache_dir_confirmed(s, path) != BUILD_OK) return BUILD_ERR; 377 return path_set(path_out, cap, str_slice(path)); 378 } 379 380 int build_store_cache_materialize(BuildStore* s, 381 const uint8_t tree_id[BUILD_HASH_LEN], 382 char* path_out, size_t cap) { 383 char final_path[BUILD_PATH_MAX]; 384 char tmp_dir[BUILD_PATH_MAX]; 385 char parent[BUILD_PATH_MAX]; 386 int ok = BUILD_ERR; 387 if (!s || !tree_id || !path_out) return BUILD_ERR; 388 if (build_store_cache_lookup(s, tree_id, path_out, cap) == BUILD_OK) 389 return BUILD_OK; 390 if (build_store_cache_path(s, tree_id, final_path, sizeof final_path) != 391 BUILD_OK) 392 return BUILD_ERR; 393 if (ensure_parent(s, final_path) != BUILD_OK) return BUILD_ERR; 394 if (make_tmp_dir(s, tmp_dir, sizeof tmp_dir) != BUILD_OK) return BUILD_ERR; 395 if (kit_cas_materialize_tree(s->cas, tree_id, tmp_dir) != KIT_OK) goto out; 396 if (sync_path(s, tmp_dir) != BUILD_OK) goto out; 397 if (s->io->rename(s->io->user, str_slice(tmp_dir), str_slice(final_path)) != 398 0) { 399 if (build_store_cache_lookup(s, tree_id, path_out, cap) == BUILD_OK) { 400 ok = BUILD_OK; 401 goto out; 402 } 403 goto out; 404 } 405 if (path_parent(final_path, parent, sizeof parent) == BUILD_OK) 406 (void)sync_path(s, parent); 407 if (path_set(path_out, cap, str_slice(final_path)) != BUILD_OK) goto out_no_rm; 408 ok = BUILD_OK; 409 out_no_rm: 410 if (ok == BUILD_OK) return BUILD_OK; 411 out: 412 (void)remove_path(s, tmp_dir, 1); 413 return ok; 414 } 415 416 int build_store_ingest_output(BuildStore* s, const char* out_dir, 417 uint8_t out_tree_id[BUILD_HASH_LEN], 418 char* path_out, size_t cap) { 419 if (!s || !out_dir || !out_tree_id || !path_out) return BUILD_ERR; 420 if (kit_cas_add_tree_from_dir(s->cas, out_dir, out_tree_id) != KIT_OK) 421 return BUILD_ERR; 422 return build_store_cache_materialize(s, out_tree_id, path_out, cap); 423 } 424 425 int build_store_sandbox_new(BuildStore* s, char* sandbox_out, 426 size_t sandbox_cap, char* out_dir_out, 427 size_t out_cap) { 428 char out_dir[BUILD_PATH_MAX]; 429 if (!s || !sandbox_out || !out_dir_out) return BUILD_ERR; 430 if (make_tmp_dir(s, sandbox_out, sandbox_cap) != BUILD_OK) return BUILD_ERR; 431 if (path_join2(out_dir, sizeof out_dir, sandbox_out, "out") != BUILD_OK) { 432 (void)remove_path(s, sandbox_out, 1); 433 return BUILD_ERR; 434 } 435 if (ensure_dir(s, out_dir) != BUILD_OK) { 436 (void)remove_path(s, sandbox_out, 1); 437 return BUILD_ERR; 438 } 439 (void)sync_path(s, sandbox_out); 440 if (path_set(out_dir_out, out_cap, str_slice(out_dir)) != BUILD_OK) { 441 (void)remove_path(s, sandbox_out, 1); 442 return BUILD_ERR; 443 } 444 return BUILD_OK; 445 } 446 447 void build_store_sandbox_done(BuildStore* s, const char* sandbox) { 448 if (!s || !sandbox) return; 449 (void)remove_path(s, sandbox, 1); 450 }