kit

kit
git clone https://git.ryansepassi.com/git/kit.git
Log | Files | Refs | README

cas.c (15210B)

      1 /* Public content-addressed store API: a thin composition over the internal
      2  * dist content model (src/dist/{blob,tree,cas}.c). See <kit/cas.h>. */
      3 
      4 #include "dist/cas.h"
      5 
      6 #include <kit/cas.h>
      7 #include <stdio.h>
      8 #include <string.h>
      9 
     10 #include "core/diag.h"
     11 #include "dist/blob.h"
     12 #include "dist/dist.h"
     13 #include "dist/tree.h"
     14 
     15 struct KitCas {
     16   /* Own a copy of the context so the handle outlives the caller's (possibly
     17    * stack-local) KitContext; ctx points at the stored copy. The pointed-to
     18    * heap/file_io/diag must still outlive the handle. */
     19   KitContext ctx_storage;
     20   const KitContext* ctx;
     21   KitCasHost host;
     22   DistCas dist;
     23 };
     24 
     25 struct KitCasTreeBuilder {
     26   KitCas* cas;
     27   DistTree tree;
     28   DistTreeEntry* entries;
     29 };
     30 
     31 void kit_blob_info(KitBlobInfo* out, const uint8_t* data, size_t len) {
     32   DistBlobInfo bi;
     33   if (!out) return;
     34   memset(out, 0, sizeof *out);
     35   if (dist_blob_info(&bi, data, len, DIST_BLOB_CHUNK_SIZE_DEFAULT) != DIST_OK)
     36     return;
     37   memcpy(out->id, bi.id, DIST_BLAKE2B_LEN);
     38   memcpy(out->root, bi.root, DIST_BLAKE2B_LEN);
     39   out->size = bi.size;
     40   out->chunks = bi.chunks;
     41 }
     42 
     43 void kit_hex_encode(char* out, const uint8_t* in, size_t n) {
     44   dist_hex_encode(out, in, n);
     45 }
     46 
     47 KitStatus kit_hex_decode(uint8_t* out, const char* in, size_t n) {
     48   return dist_hex_decode(out, in, n) == DIST_OK ? KIT_OK : KIT_MALFORMED;
     49 }
     50 
     51 KitStatus kit_cas_open(const KitContext* ctx, const KitCasHost* host,
     52                        const char* root_path, KitCas** out) {
     53   KitCas* cas;
     54   if (!ctx || !ctx->heap || !host || !host->file_io || !root_path || !out)
     55     return KIT_INVALID;
     56   *out = NULL;
     57   cas = (KitCas*)ctx->heap->alloc(ctx->heap, sizeof *cas, _Alignof(KitCas));
     58   if (!cas) return KIT_NOMEM;
     59   cas->ctx_storage = *ctx;
     60   cas->ctx = &cas->ctx_storage;
     61   cas->host = *host;
     62   cas->dist.host.file_io = host->file_io;
     63   cas->dist.host.mkdir_p = host->mkdir_p;
     64   cas->dist.host.mark_executable = host->mark_executable;
     65   cas->dist.host.user = host->user;
     66   cas->dist.root = root_path;
     67   *out = cas;
     68   return KIT_OK;
     69 }
     70 
     71 void kit_cas_close(KitCas* cas) {
     72   if (!cas) return;
     73   cas->ctx->heap->free(cas->ctx->heap, cas, sizeof *cas);
     74 }
     75 
     76 KitStatus kit_cas_add_blob(KitCas* cas, const uint8_t* data, size_t len,
     77                            KitBlobInfo* out) {
     78   DistBlobInfo bi;
     79   if (!cas || !out) return KIT_INVALID;
     80   if (dist_blob_info(&bi, data, len, DIST_BLOB_CHUNK_SIZE_DEFAULT) != DIST_OK) {
     81     kit_ctx_diagf(cas->ctx, "failed to hash blob");
     82     return KIT_ERR;
     83   }
     84   if (dist_cas_put_blob(&cas->dist, bi.id, data, len) != DIST_OK) {
     85     kit_ctx_diagf(cas->ctx, "failed to store blob");
     86     return KIT_IO;
     87   }
     88   memcpy(out->id, bi.id, DIST_BLAKE2B_LEN);
     89   memcpy(out->root, bi.root, DIST_BLAKE2B_LEN);
     90   out->size = bi.size;
     91   out->chunks = bi.chunks;
     92   return KIT_OK;
     93 }
     94 
     95 KitStatus kit_cas_has_blob(KitCas* cas,
     96                            const uint8_t id[KIT_CAS_HASH_LEN]) {
     97   KitFileData fd;
     98   KitStatus st;
     99   if (!cas || !id) return KIT_INVALID;
    100   fd.data = NULL;
    101   fd.size = 0;
    102   fd.token = NULL;
    103   st = kit_cas_get_blob(cas, id, &fd);
    104   if (st == KIT_OK) kit_cas_release(cas, &fd);
    105   return st;
    106 }
    107 
    108 KitStatus kit_cas_blob_path(KitCas* cas, const uint8_t id[KIT_CAS_HASH_LEN],
    109                             char* out, size_t cap) {
    110   if (!cas || !id || !out || cap == 0u) return KIT_INVALID;
    111   return dist_cas_blob_path(out, cap, cas->dist.root, id) == DIST_OK ? KIT_OK
    112                                                                      : KIT_ERR;
    113 }
    114 
    115 KitStatus kit_cas_get_blob(KitCas* cas, const uint8_t id[KIT_CAS_HASH_LEN],
    116                            KitFileData* out) {
    117   if (!cas || !id || !out) return KIT_INVALID;
    118   out->data = NULL;
    119   out->size = 0;
    120   out->token = NULL;
    121   if (dist_cas_get_blob(&cas->dist, id, out) != DIST_OK) return KIT_NOT_FOUND;
    122   return KIT_OK;
    123 }
    124 
    125 void kit_cas_release(KitCas* cas, KitFileData* fd) {
    126   if (!cas || !fd) return;
    127   if (fd->data && cas->host.file_io && cas->host.file_io->release)
    128     cas->host.file_io->release(cas->host.file_io->user, fd);
    129   fd->data = NULL;
    130   fd->size = 0;
    131   fd->token = NULL;
    132 }
    133 
    134 KitStatus kit_cas_tree_builder_new(KitCas* cas, KitCasTreeBuilder** out) {
    135   KitCasTreeBuilder* b;
    136   KitHeap* h;
    137   if (!cas || !out) return KIT_INVALID;
    138   *out = NULL;
    139   h = cas->ctx->heap;
    140   b = (KitCasTreeBuilder*)h->alloc(h, sizeof *b, _Alignof(KitCasTreeBuilder));
    141   if (!b) return KIT_NOMEM;
    142   b->cas = cas;
    143   b->entries = (DistTreeEntry*)h->alloc(h, DIST_MAX_FILES * sizeof *b->entries,
    144                                         _Alignof(DistTreeEntry));
    145   if (!b->entries) {
    146     h->free(h, b, sizeof *b);
    147     return KIT_NOMEM;
    148   }
    149   b->tree.entries = b->entries;
    150   b->tree.n_entries = 0;
    151   b->tree.cap_entries = DIST_MAX_FILES;
    152   *out = b;
    153   return KIT_OK;
    154 }
    155 
    156 KitStatus kit_cas_tree_builder_add(KitCasTreeBuilder* b, const char* tree_path,
    157                                    KitTreeMode mode, const uint8_t* data,
    158                                    size_t len) {
    159   KitCas* cas;
    160   DistBlobInfo bi;
    161   DistTreeEntry* e;
    162   if (!b || !tree_path) return KIT_INVALID;
    163   cas = b->cas;
    164   if (b->tree.n_entries >= b->tree.cap_entries) {
    165     kit_ctx_diagf(cas->ctx, "too many tree entries");
    166     return KIT_ERR;
    167   }
    168   if (!dist_tree_path_valid(tree_path)) {
    169     kit_ctx_diagf(cas->ctx, "unsafe tree path: %s", tree_path);
    170     return KIT_INVALID;
    171   }
    172   if (!dist_tree_mode_name((uint8_t)mode)) {
    173     kit_ctx_diagf(cas->ctx, "bad tree mode for: %s", tree_path);
    174     return KIT_INVALID;
    175   }
    176   if (dist_blob_info(&bi, data, len, DIST_BLOB_CHUNK_SIZE_DEFAULT) != DIST_OK) {
    177     kit_ctx_diagf(cas->ctx, "failed to hash blob: %s", tree_path);
    178     return KIT_ERR;
    179   }
    180   if (dist_cas_put_blob(&cas->dist, bi.id, data, len) != DIST_OK) {
    181     kit_ctx_diagf(cas->ctx, "failed to store blob: %s", tree_path);
    182     return KIT_IO;
    183   }
    184   e = &b->tree.entries[b->tree.n_entries++];
    185   memset(e, 0, sizeof *e);
    186   snprintf(e->path, sizeof e->path, "%s", tree_path);
    187   e->mode = (uint8_t)mode;
    188   e->size = bi.size;
    189   memcpy(e->blob, bi.id, DIST_BLAKE2B_LEN);
    190   memcpy(e->root, bi.root, DIST_BLAKE2B_LEN);
    191   return KIT_OK;
    192 }
    193 
    194 KitStatus kit_cas_tree_builder_finish(KitCasTreeBuilder* b,
    195                                       uint8_t out_tree_id[KIT_CAS_HASH_LEN]) {
    196   KitCas* cas;
    197   KitWriter* w = NULL;
    198   const uint8_t* bytes;
    199   size_t len;
    200   char err[128];
    201   if (!b || !out_tree_id) return KIT_INVALID;
    202   cas = b->cas;
    203   if (dist_tree_sort_validate(&b->tree, err, sizeof err) != DIST_OK) {
    204     kit_ctx_diagf(cas->ctx, "%s", err);
    205     return KIT_MALFORMED;
    206   }
    207   if (kit_writer_mem(cas->ctx->heap, &w) != KIT_OK) {
    208     kit_ctx_diagf(cas->ctx, "failed to allocate tree writer");
    209     return KIT_NOMEM;
    210   }
    211   if (dist_tree_emit(&b->tree, w) != DIST_OK ||
    212       kit_writer_status(w) != KIT_OK) {
    213     kit_writer_close(w);
    214     kit_ctx_diagf(cas->ctx, "failed to emit tree manifest");
    215     return KIT_ERR;
    216   }
    217   bytes = kit_writer_mem_bytes(w, &len);
    218   dist_tree_id(out_tree_id, bytes, len);
    219   if (dist_cas_put_tree(&cas->dist, out_tree_id, bytes, len) != DIST_OK) {
    220     kit_writer_close(w);
    221     kit_ctx_diagf(cas->ctx, "failed to store tree manifest");
    222     return KIT_IO;
    223   }
    224   kit_writer_close(w);
    225   return KIT_OK;
    226 }
    227 
    228 KitStatus kit_cas_add_tree_manifest(KitCas* cas, const uint8_t* data,
    229                                     size_t len,
    230                                     uint8_t out_tree_id[KIT_CAS_HASH_LEN]) {
    231   DistTree tree;
    232   DistTreeEntry* entries = NULL;
    233   KitWriter* w = NULL;
    234   const uint8_t* canon;
    235   size_t canon_len;
    236   char err[128];
    237   KitStatus st = KIT_OK;
    238 
    239   if (!cas || (!data && len) || !out_tree_id) return KIT_INVALID;
    240   entries = (DistTreeEntry*)cas->ctx->heap->alloc(
    241       cas->ctx->heap, DIST_MAX_FILES * sizeof *entries, _Alignof(DistTreeEntry));
    242   if (!entries) return KIT_NOMEM;
    243   tree.entries = entries;
    244   tree.n_entries = 0;
    245   tree.cap_entries = DIST_MAX_FILES;
    246   if (dist_tree_parse(data, len, &tree, err, sizeof err) != DIST_OK) {
    247     kit_ctx_diagf(cas->ctx, "%s", err);
    248     st = KIT_MALFORMED;
    249     goto out;
    250   }
    251   if (kit_writer_mem(cas->ctx->heap, &w) != KIT_OK) {
    252     kit_ctx_diagf(cas->ctx, "failed to allocate tree writer");
    253     st = KIT_NOMEM;
    254     goto out;
    255   }
    256   if (dist_tree_emit(&tree, w) != DIST_OK || kit_writer_status(w) != KIT_OK) {
    257     kit_ctx_diagf(cas->ctx, "failed to emit tree manifest");
    258     st = KIT_ERR;
    259     goto out;
    260   }
    261   canon = kit_writer_mem_bytes(w, &canon_len);
    262   if (canon_len != len || (len && memcmp(canon, data, len) != 0)) {
    263     kit_ctx_diagf(cas->ctx, "non-canonical tree manifest");
    264     st = KIT_MALFORMED;
    265     goto out;
    266   }
    267   dist_tree_id(out_tree_id, data, len);
    268   if (dist_cas_put_tree(&cas->dist, out_tree_id, data, len) != DIST_OK) {
    269     kit_ctx_diagf(cas->ctx, "failed to store tree manifest");
    270     st = KIT_IO;
    271     goto out;
    272   }
    273 
    274 out:
    275   if (w) kit_writer_close(w);
    276   cas->ctx->heap->free(cas->ctx->heap, entries,
    277                        DIST_MAX_FILES * sizeof *entries);
    278   return st;
    279 }
    280 
    281 void kit_cas_tree_builder_free(KitCasTreeBuilder* b) {
    282   KitHeap* h;
    283   if (!b) return;
    284   h = b->cas->ctx->heap;
    285   h->free(h, b->entries, DIST_MAX_FILES * sizeof *b->entries);
    286   h->free(h, b, sizeof *b);
    287 }
    288 
    289 typedef struct CasDirWalk {
    290   KitCasTreeBuilder* b;
    291   KitStatus status;
    292 } CasDirWalk;
    293 
    294 static int cas_dir_walk_file(void* user, const char* source_path,
    295                              const char* tree_path, int executable) {
    296   CasDirWalk* w = (CasDirWalk*)user;
    297   KitCas* cas = w->b->cas;
    298   const KitFileIO* io = cas->host.file_io;
    299   KitFileData fd;
    300   KitStatus st;
    301   fd.data = NULL;
    302   fd.size = 0;
    303   fd.token = NULL;
    304   if (io->read_all(io->user, source_path, &fd) != KIT_OK) {
    305     kit_ctx_diagf(cas->ctx, "failed to read: %s", source_path);
    306     w->status = KIT_IO;
    307     return 1;
    308   }
    309   st = kit_cas_tree_builder_add(
    310       w->b, tree_path, executable ? KIT_TREE_MODE_EXEC : KIT_TREE_MODE_FILE,
    311       fd.data, fd.size);
    312   if (io->release) io->release(io->user, &fd);
    313   if (st != KIT_OK) {
    314     w->status = st;
    315     return 1;
    316   }
    317   return 0;
    318 }
    319 
    320 KitStatus kit_cas_add_tree_from_dir(KitCas* cas, const char* root,
    321                                     uint8_t out_tree_id[KIT_CAS_HASH_LEN]) {
    322   KitCasTreeBuilder* b;
    323   CasDirWalk w;
    324   KitStatus st;
    325   if (!cas || !root || !out_tree_id) return KIT_INVALID;
    326   if (!cas->host.walk_regular_files) return KIT_UNSUPPORTED;
    327   st = kit_cas_tree_builder_new(cas, &b);
    328   if (st != KIT_OK) return st;
    329   w.b = b;
    330   w.status = KIT_OK;
    331   if (cas->host.walk_regular_files(cas->host.user, root, cas_dir_walk_file,
    332                                    &w) != 0) {
    333     if (w.status == KIT_OK) {
    334       kit_ctx_diagf(cas->ctx, "failed to walk directory: %s", root);
    335       w.status = KIT_IO;
    336     }
    337     kit_cas_tree_builder_free(b);
    338     return w.status;
    339   }
    340   st = kit_cas_tree_builder_finish(b, out_tree_id);
    341   kit_cas_tree_builder_free(b);
    342   return st;
    343 }
    344 
    345 /* Load and parse a stored tree into a heap-allocated entries buffer. On
    346  * success, *raw holds the borrowed manifest bytes (release via file_io) and
    347  * *entries the allocation to free. */
    348 static KitStatus cas_load_tree(KitCas* cas,
    349                                const uint8_t tree_id[KIT_CAS_HASH_LEN],
    350                                DistTree* tree, DistTreeEntry** entries,
    351                                KitFileData* raw) {
    352   KitHeap* h = cas->ctx->heap;
    353   char err[128];
    354   *entries = (DistTreeEntry*)h->alloc(h, DIST_MAX_FILES * sizeof **entries,
    355                                       _Alignof(DistTreeEntry));
    356   if (!*entries) return KIT_NOMEM;
    357   tree->entries = *entries;
    358   tree->n_entries = 0;
    359   tree->cap_entries = DIST_MAX_FILES;
    360   raw->data = NULL;
    361   raw->size = 0;
    362   raw->token = NULL;
    363   if (dist_cas_get_tree(&cas->dist, tree_id, raw) != DIST_OK) {
    364     kit_ctx_diagf(cas->ctx, "failed to load tree");
    365     h->free(h, *entries, DIST_MAX_FILES * sizeof **entries);
    366     *entries = NULL;
    367     return KIT_NOT_FOUND;
    368   }
    369   if (dist_tree_parse(raw->data, raw->size, tree, err, sizeof err) != DIST_OK) {
    370     kit_ctx_diagf(cas->ctx, "%s", err);
    371     if (cas->host.file_io->release)
    372       cas->host.file_io->release(cas->host.file_io->user, raw);
    373     h->free(h, *entries, DIST_MAX_FILES * sizeof **entries);
    374     *entries = NULL;
    375     return KIT_MALFORMED;
    376   }
    377   return KIT_OK;
    378 }
    379 
    380 static void cas_free_tree(KitCas* cas, DistTreeEntry* entries,
    381                           KitFileData* raw) {
    382   KitHeap* h = cas->ctx->heap;
    383   if (cas->host.file_io->release)
    384     cas->host.file_io->release(cas->host.file_io->user, raw);
    385   if (entries) h->free(h, entries, DIST_MAX_FILES * sizeof *entries);
    386 }
    387 
    388 KitStatus kit_cas_inspect_tree(KitCas* cas,
    389                                const uint8_t tree_id[KIT_CAS_HASH_LEN],
    390                                KitWriter* out) {
    391   KitFileData raw;
    392   if (!cas || !tree_id || !out) return KIT_INVALID;
    393   raw.data = NULL;
    394   raw.size = 0;
    395   raw.token = NULL;
    396   if (dist_cas_get_tree(&cas->dist, tree_id, &raw) != DIST_OK) {
    397     kit_ctx_diagf(cas->ctx, "failed to load tree");
    398     return KIT_NOT_FOUND;
    399   }
    400   if (raw.size && kit_writer_write(out, raw.data, raw.size) != KIT_OK) {
    401     if (cas->host.file_io->release)
    402       cas->host.file_io->release(cas->host.file_io->user, &raw);
    403     kit_ctx_diagf(cas->ctx, "failed to write tree manifest");
    404     return KIT_IO;
    405   }
    406   if (cas->host.file_io->release)
    407     cas->host.file_io->release(cas->host.file_io->user, &raw);
    408   return kit_writer_status(out) == KIT_OK ? KIT_OK : KIT_IO;
    409 }
    410 
    411 KitStatus kit_cas_verify_tree(KitCas* cas,
    412                               const uint8_t tree_id[KIT_CAS_HASH_LEN]) {
    413   DistTree tree;
    414   DistTreeEntry* entries;
    415   KitFileData raw;
    416   KitStatus st;
    417   size_t i;
    418   if (!cas || !tree_id) return KIT_INVALID;
    419   st = cas_load_tree(cas, tree_id, &tree, &entries, &raw);
    420   if (st != KIT_OK) return st;
    421   st = KIT_OK;
    422   for (i = 0; i < tree.n_entries; ++i) {
    423     const DistTreeEntry* e = &tree.entries[i];
    424     KitFileData fd;
    425     DistBlobInfo bi;
    426     fd.data = NULL;
    427     fd.size = 0;
    428     fd.token = NULL;
    429     if (dist_cas_get_blob(&cas->dist, e->blob, &fd) != DIST_OK) {
    430       kit_ctx_diagf(cas->ctx, "missing or corrupt blob for: %s", e->path);
    431       st = KIT_NOT_FOUND;
    432       break;
    433     }
    434     if (dist_blob_info(&bi, fd.data, fd.size, DIST_BLOB_CHUNK_SIZE_DEFAULT) !=
    435             DIST_OK ||
    436         bi.size != e->size || memcmp(bi.root, e->root, DIST_BLAKE2B_LEN) != 0) {
    437       if (cas->host.file_io->release)
    438         cas->host.file_io->release(cas->host.file_io->user, &fd);
    439       kit_ctx_diagf(cas->ctx, "blob root mismatch for: %s", e->path);
    440       st = KIT_INVALID;
    441       break;
    442     }
    443     if (cas->host.file_io->release)
    444       cas->host.file_io->release(cas->host.file_io->user, &fd);
    445   }
    446   cas_free_tree(cas, entries, &raw);
    447   return st;
    448 }
    449 
    450 KitStatus kit_cas_materialize_tree(KitCas* cas,
    451                                    const uint8_t tree_id[KIT_CAS_HASH_LEN],
    452                                    const char* dst) {
    453   DistTree tree;
    454   DistTreeEntry* entries;
    455   KitFileData raw;
    456   KitStatus st;
    457   if (!cas || !tree_id || !dst) return KIT_INVALID;
    458   st = cas_load_tree(cas, tree_id, &tree, &entries, &raw);
    459   if (st != KIT_OK) return st;
    460   if (dist_cas_materialize_tree(&cas->dist, &tree, dst) != DIST_OK) {
    461     kit_ctx_diagf(cas->ctx, "failed to materialize tree");
    462     st = KIT_ERR;
    463   }
    464   cas_free_tree(cas, entries, &raw);
    465   return st;
    466 }