boot2

kernel.c (61992B)

---

 /* seed kernel — minimal OS satisfying docs/OS.md Tier 1.
  *
  * Boots through an arch backend with two virtio-blk-MMIO disks, parses
  * the DTB to find virtio_mmio nodes + memory, brings up
  * a small polling virtio-blk driver, reads the cpio newc archive from
  * blk0 (read-only) into the in-memory tmpfs, loads /init (a static
  * target ELF), and enters it through the arch trap-return path. Syscall
  * traps land in trap_sync() and dispatch Tier-1/Tier-2 syscalls. On exit,
  * the tmpfs is
  * serialised to blk1 in a small SEEDFS table for the host extractor.
  */
 
 typedef unsigned char  u8;
 typedef unsigned short u16;
 typedef unsigned int   u32;
 typedef unsigned long  u64;
 typedef long           i64;
 typedef int            i32;
 
 #include "arch.h"
 
 /* ─── Console ───────────────────────────────────────────────────────────── */
 
 static void uart_putc(char c) {
     arch_console_putc(c);
 }
 
 static void uart_puts(const char *s) {
     while (*s) {
         if (*s == '\n') uart_putc('\r');
         uart_putc(*s++);
     }
 }
 
 static void uart_putx(u64 v) {
     static const char hex[] = "0123456789abcdef";
     uart_puts("0x");
     for (int i = 60; i >= 0; i -= 4) uart_putc(hex[(v >> i) & 0xf]);
 }
 
 static void uart_putd(i64 v) {
     if (v < 0) { uart_putc('-'); v = -v; }
     char buf[24];
     int i = 0;
     if (v == 0) buf[i++] = '0';
     while (v) { buf[i++] = '0' + (v % 10); v /= 10; }
     while (i--) uart_putc(buf[i]);
 }
 
 __attribute__((noreturn)) static void hang(void) { for (;;) arch_pause(); }
 
 /* ─── Tiny libc-ish helpers ─────────────────────────────────────────────── */
 
 /* memcpy / memset / memmove come from tcc/cc/mem.c, linked alongside.
  * Both gcc and tcc emit calls to these for struct copies and bulk
  * zero-init past their inline thresholds; centralising them in
  * tcc/cc/mem.c keeps the tcc-built and gcc-built kernels in sync. */
 void *memcpy(void *dst, const void *src, u64 n);
 void *memset(void *dst, int c, u64 n);
 void *memmove(void *dst, const void *src, u64 n);
 
 static int str_eq(const char *a, const char *b) {
     while (*a && *a == *b) { a++; b++; }
     return *a == 0 && *b == 0;
 }
 static int str_n(const char *s) { int n = 0; while (s[n]) n++; return n; }
 static void mem_cpy(void *d, const void *s, u64 n) {
     /* 8-byte fast path when both pointers are 8-aligned and n is a multiple
      * of 8. Under TCG this is roughly 8× faster than the byte loop. */
     u8 *dd = d; const u8 *ss = s;
     if ((((u64)dd | (u64)ss | n) & 7) == 0) {
         u64 *dq = (u64 *)dd;
         const u64 *sq = (const u64 *)ss;
         u64 m = n >> 3;
         for (u64 i = 0; i < m; i++) dq[i] = sq[i];
         return;
     }
     for (u64 i = 0; i < n; i++) dd[i] = ss[i];
 }
 static void mem_set(void *d, int c, u64 n) {
     u8 *dd = d;
     for (u64 i = 0; i < n; i++) dd[i] = (u8)c;
 }
 
 /* User address-space constants are supplied by the arch backend. */
 #define USER_POOL_A_PA ARCH_USER_POOL_A_PA
 #define USER_POOL_B_PA ARCH_USER_POOL_B_PA
 #define USER_POOL_SIZE ARCH_USER_POOL_SIZE
 #define USER_VA_LO     ARCH_USER_VA_LO
 #define USER_VA_HI     ARCH_USER_VA_HI
 #define USER_POOL_FIRST_SLOT ARCH_USER_POOL_FIRST_SLOT
 #define USER_POOL_LAST_SLOT  ARCH_USER_POOL_LAST_SLOT
 
 /* 0 = pool A is currently mapped at user VAs; 1 = pool B. */
 static int current_pool = 0;
 
 /* ─── Kernel heap (bump allocator) ──────────────────────────────────────── */
 
 extern char _end[];
 static u8 *kheap_ptr;
 static u8 *kheap_end;
 
 static void *kalloc(u64 n) {
     n = (n + 15) & ~15UL;
     if (kheap_ptr + n > kheap_end) {
         uart_puts("kalloc: out of memory\n");
         hang();
     }
     void *r = kheap_ptr;
     kheap_ptr += n;
     return r;
 }
 
 /* ─── Big-endian readers (DTB is BE) ────────────────────────────────────── */
 
 static u32 be32(const u8 *p) { return (u32)p[0]<<24 | (u32)p[1]<<16 | (u32)p[2]<<8 | (u32)p[3]; }
 static u64 be64(const u8 *p) { return ((u64)be32(p) << 32) | (u64)be32(p + 4); }
 
 /* ─── Flattened Device Tree walker ──────────────────────────────────────── */
 
 /* DTB ("flattened device tree blob") header + token codes — devicetree
  * spec §5.1 (header) + §5.4.1 (struct block tokens). */
 #define FDT_MAGIC      0xd00dfeedu
 #define FDT_BEGIN_NODE 1
 #define FDT_END_NODE   2
 #define FDT_PROP       3
 #define FDT_NOP        4
 #define FDT_END        9
 
 /* Xen PVH ABI handoff (see Xen's docs/misc/pvh.pandoc). On PVH boot QEMU
  * points EBX at a `struct hvm_start_info`; the amd64 long-mode init in
  * arch/amd64/kernel.S preserves EBX through to kmain's `dtb_phys` arg.
  * First word is the magic; cmdline_paddr sits at offset 0x18. microvm
  * has no DTB, so this is the only path for `qemu -append "..."`. */
 #define PVH_HVM_START_MAGIC       0x336ec578u
 #define PVH_HVM_START_OFF_CMDLINE 0x18
 
 /* cpio "newc" (SVR4 portable) format — first 6 bytes of every header
  * record are the magic; header is fixed-size (110 bytes) followed by
  * the NUL-terminated name and the file data, both 4-byte padded. The
  * 13 numeric fields are 8-char ASCII hex starting at offset 6. Spec:
  * cpio(5) under "New ASCII Format". */
 #define CPIO_NEWC_MAGIC "070701"
 #define CPIO_NEWC_HDR_SIZE 110
 #define CPIO_NEWC_FIELD(p, n) ((p) + 6 + (n) * 8)
 #define CPIO_MODE_TYPE_MASK 0xf000          /* st_mode file-type bits */
 #define CPIO_MODE_TYPE_DIR  0x4000
 #define CPIO_MODE_TYPE_REG  0x8000
 
 /* QEMU virt has 32 virtio-mmio slots (0x0a000000..0x0a004000, 0x200 each).
  * Most are unpopulated and report MagicValue=0/DeviceID=0 — we capture all
  * slots advertised by the DTB and the driver init filters at probe time. */
 #define MAX_VIRTIO_MMIO 32
 
 struct dtb_info {
     u64 mem_start;
     u64 mem_size;
     u64 virtio_mmio_pa[MAX_VIRTIO_MMIO];
     int virtio_mmio_n;
     char bootargs[256];
 };
 
 /* str_starts: returns 1 iff `s` begins with `prefix`. */
 static int str_starts(const char *s, const char *prefix) {
     while (*prefix) { if (*s++ != *prefix++) return 0; }
     return 1;
 }
 
 static void parse_dtb(const void *dtb, struct dtb_info *out) {
 #ifdef ARCH_STATIC_VIRTIO_MMIO_BASE
     /* No DTB on this arch (amd64 microvm). mem + virtio-mmio come from
      * arch.h compile-time constants; the kernel cmdline arrives via the
      * PVH hvm_start_info struct (see PVH_HVM_START_* above). */
     out->mem_start = ARCH_STATIC_MEM_START;
     out->mem_size = ARCH_STATIC_MEM_SIZE;
     out->virtio_mmio_n = ARCH_STATIC_VIRTIO_MMIO_COUNT;
     for (int i = 0; i < out->virtio_mmio_n && i < MAX_VIRTIO_MMIO; i++)
         out->virtio_mmio_pa[i] = ARCH_STATIC_VIRTIO_MMIO_BASE +
                                  (u64)i * ARCH_STATIC_VIRTIO_MMIO_STRIDE;
     if ((u64)dtb != 0) {
         const u8 *p = dtb;
         u32 magic = (u32)p[0] | ((u32)p[1] << 8) | ((u32)p[2] << 16) | ((u32)p[3] << 24);
         if (magic == PVH_HVM_START_MAGIC) {
             const u8 *cp = p + PVH_HVM_START_OFF_CMDLINE;
             u64 cmdline_paddr = 0;
             for (int i = 0; i < 8; i++) cmdline_paddr |= (u64)cp[i] << (i * 8);
             if (cmdline_paddr) {
                 const char *s = (const char *)cmdline_paddr;
                 int i = 0;
                 while (s[i] && i < 255) { out->bootargs[i] = s[i]; i++; }
                 out->bootargs[i] = 0;
             }
         }
     }
     return;
 #endif
     const u8 *base = dtb;
     if (be32(base) != FDT_MAGIC) {
         uart_puts("DTB: bad magic\n"); return;
     }
     u32 off_struct = be32(base + 8);
     u32 off_strings = be32(base + 12);
     const u8 *strings = base + off_strings;
     const u8 *p = base + off_struct;
 
     char path[4][64] = {{0}};
     int depth = -1;
 
     for (;;) {
         u32 tok = be32(p); p += 4;
         if (tok == FDT_BEGIN_NODE) {
             depth++;
             if (depth < 4) {
                 int i = 0;
                 while (p[i] && i < 63) { path[depth][i] = (char)p[i]; i++; }
                 path[depth][i] = 0;
             }
             while (*p) p++;
             p++;
             p = (const u8 *)(((u64)p + 3) & ~3UL);
         } else if (tok == FDT_END_NODE) {
             depth--;
         } else if (tok == FDT_PROP) {
             u32 len = be32(p); p += 4;
             u32 nameoff = be32(p); p += 4;
             const char *pn = (const char *)(strings + nameoff);
 
             if (depth == 1 && str_eq(path[1], "chosen")) {
                 if (str_eq(pn, "bootargs")) {
                     u32 i = 0;
                     while (i < len && i < 255) { out->bootargs[i] = (char)p[i]; i++; }
                     out->bootargs[i] = 0;
                 }
             }
             if (depth == 1) {
                 /* memory node is named "memory@<addr>" */
                 if (str_starts(path[1], "memory") &&
                     str_eq(pn, "reg") && len >= 16 && out->mem_size == 0) {
                     out->mem_start = be64(p);
                     out->mem_size  = be64(p + 8);
                 }
             }
             /* virtio-mmio nodes: capture each slot's PA. Root/soc
              * #address-cells/#size-cells are both 2 on QEMU virt, so reg
              * is 16 bytes (PA u64, size u64); we only need the PA. */
             if (depth >= 1 && str_starts(path[depth], "virtio_mmio@") &&
                 str_eq(pn, "reg") && len >= 16 &&
                 out->virtio_mmio_n < MAX_VIRTIO_MMIO) {
                 out->virtio_mmio_pa[out->virtio_mmio_n++] = be64(p);
             }
             p += len;
             p = (const u8 *)(((u64)p + 3) & ~3UL);
         } else if (tok == FDT_NOP) {
             /* skip */
         } else if (tok == FDT_END) {
             break;
         } else {
             uart_puts("DTB: bad token "); uart_putx(tok); uart_puts("\n");
             break;
         }
     }
 }
 
 /* ─── virtio-blk-MMIO driver (polling, single-VQ) ───────────────────────── */
 /*
  * Two block devices: blk0 = read-only cpio input, blk1 = read-write output.
  * Identification is content-based (sector 0 cpio newc magic "070701" ⇒ blk0)
  * so we don't depend on -drive ordering on the qemu command line.
  *
  * The driver is intentionally small: 8-entry split virtqueue, one in-flight
  * request at a time, polling used.idx. No interrupts (DAIF stays masked).
  *
  * MMIO transport regs are reached via DEVICE_ALIAS_BASE + PA. virtqueue
  * memory comes from kernel BSS (identity-mapped Normal, VA == PA).
  */
 
 #define VIRTIO_MMIO_MAGIC          0x000
 #define VIRTIO_MMIO_VERSION        0x004
 #define VIRTIO_MMIO_DEVICE_ID      0x008
 #define VIRTIO_MMIO_DEV_FEATURES   0x010
 #define VIRTIO_MMIO_DEV_FEAT_SEL   0x014
 #define VIRTIO_MMIO_DRV_FEATURES   0x020
 #define VIRTIO_MMIO_DRV_FEAT_SEL   0x024
 #define VIRTIO_MMIO_QUEUE_SEL      0x030
 #define VIRTIO_MMIO_QUEUE_NUM_MAX  0x034
 #define VIRTIO_MMIO_QUEUE_NUM      0x038
 #define VIRTIO_MMIO_QUEUE_READY    0x044
 #define VIRTIO_MMIO_QUEUE_NOTIFY   0x050
 #define VIRTIO_MMIO_INT_STATUS     0x060
 #define VIRTIO_MMIO_INT_ACK        0x064
 #define VIRTIO_MMIO_STATUS         0x070
 #define VIRTIO_MMIO_QUEUE_DESC_LO  0x080
 #define VIRTIO_MMIO_QUEUE_DESC_HI  0x084
 #define VIRTIO_MMIO_QUEUE_DRIVER_LO 0x090
 #define VIRTIO_MMIO_QUEUE_DRIVER_HI 0x094
 #define VIRTIO_MMIO_QUEUE_DEVICE_LO 0x0a0
 #define VIRTIO_MMIO_QUEUE_DEVICE_HI 0x0a4
 #define VIRTIO_MMIO_CONFIG         0x100
 
 #define VIRTIO_STATUS_ACKNOWLEDGE  1
 #define VIRTIO_STATUS_DRIVER       2
 #define VIRTIO_STATUS_DRIVER_OK    4
 #define VIRTIO_STATUS_FEATURES_OK  8
 #define VIRTIO_STATUS_FAILED       128
 
 #define VIRTIO_F_VERSION_1_BIT     32  /* bit 32 in feature space */
 
 #define VIRTIO_BLK_T_IN  0
 #define VIRTIO_BLK_T_OUT 1
 
 #define VRING_DESC_F_NEXT  1
 #define VRING_DESC_F_WRITE 2
 
 #define VQ_SIZE 8
 
 struct vring_desc {
     u64 addr;
     u32 len;
     u16 flags;
     u16 next;
 };
 
 struct vring_avail {
     u16 flags;
     u16 idx;
     u16 ring[VQ_SIZE];
     u16 used_event;
 };
 
 struct vring_used_elem {
     u32 id;
     u32 len;
 };
 
 struct vring_used {
     u16 flags;
     u16 idx;
     struct vring_used_elem ring[VQ_SIZE];
     u16 avail_event;
 };
 
 struct virtio_blk_req_hdr {
     u32 type;
     u32 reserved;
     u64 sector;
 };
 
 #define BLK_DEV_MAX 2
 
 struct blk_dev {
     volatile u8 *regs;             /* alias VA pointing at the MMIO region */
     u64 capacity_sectors;
     int present;
 };
 
 static struct blk_dev blk_devs[BLK_DEV_MAX];
 static int blk_n_devs = 0;
 
 /* One vring per device in BSS, 4 KB-aligned. Layout within the page:
  *   desc:  offset 0   (128 B for VQ_SIZE=8)
  *   avail: offset 128 (24 B with VQ_SIZE=8 + used_event)
  *   used:  offset 256 (72 B with VQ_SIZE=8 + avail_event)
  * Plenty of slack inside one 4 KB page. */
 __attribute__((aligned(4096))) static u8 vq_pages[BLK_DEV_MAX][4096];
 
 #define VQ_DESC_OFF  0
 #define VQ_AVAIL_OFF 128
 #define VQ_USED_OFF  256
 
 static struct vring_desc *vq_desc(int i)
     { return (struct vring_desc *)(vq_pages[i] + VQ_DESC_OFF); }
 static struct vring_avail *vq_avail(int i)
     { return (struct vring_avail *)(vq_pages[i] + VQ_AVAIL_OFF); }
 static struct vring_used *vq_used(int i)
     { return (struct vring_used *)(vq_pages[i] + VQ_USED_OFF); }
 
 /* MMIO accessors. Reg offsets are byte offsets per the spec. */
 static u32 mmio_r32(struct blk_dev *d, u32 off) {
     return *(volatile u32 *)(d->regs + off);
 }
 static void mmio_w32(struct blk_dev *d, u32 off, u32 val) {
     *(volatile u32 *)(d->regs + off) = val;
 }
 
 /* Initialise one device per spec §3.1.1 / §4.2 / §5.2. Returns 1 on
  * success (device is virtio-blk and ready), 0 if slot is empty / not
  * a block device, -1 on error. */
 static int blk_init_one(struct blk_dev *d) {
     u32 magic = mmio_r32(d, VIRTIO_MMIO_MAGIC);
     if (magic != 0x74726976) return 0;                  /* not a virtio slot */
     u32 devid = mmio_r32(d, VIRTIO_MMIO_DEVICE_ID);
     if (devid == 0) return 0;                           /* unpopulated slot */
     if (devid != 2) return 0;                           /* not a block device */
     u32 ver   = mmio_r32(d, VIRTIO_MMIO_VERSION);
     if (ver != 2) {
         /* QEMU virt defaults to legacy (version 1) virtio-mmio transports
          * unless the host passes -global virtio-mmio.force-legacy=false.
          * The harness scripts set that flag — reaching here means it was
          * forgotten. */
         uart_puts("[seed] virtio-mmio version != 2 (legacy): ");
         uart_putd((i64)ver);
         uart_puts(" — pass -global virtio-mmio.force-legacy=false\n");
         return -1;
     }
 
     /* Reset, ack, driver. */
     mmio_w32(d, VIRTIO_MMIO_STATUS, 0);
     mmio_w32(d, VIRTIO_MMIO_STATUS, VIRTIO_STATUS_ACKNOWLEDGE);
     mmio_w32(d, VIRTIO_MMIO_STATUS,
              VIRTIO_STATUS_ACKNOWLEDGE | VIRTIO_STATUS_DRIVER);
 
     /* Negotiate VIRTIO_F_VERSION_1 only (bit 32 → feature word 1). */
     mmio_w32(d, VIRTIO_MMIO_DEV_FEAT_SEL, 1);
     u32 dev_feat_hi = mmio_r32(d, VIRTIO_MMIO_DEV_FEATURES);
     if (!(dev_feat_hi & (1u << (VIRTIO_F_VERSION_1_BIT - 32)))) {
         uart_puts("[seed] virtio: device lacks VERSION_1\n");
         return -1;
     }
     mmio_w32(d, VIRTIO_MMIO_DRV_FEAT_SEL, 0);
     mmio_w32(d, VIRTIO_MMIO_DRV_FEATURES, 0);
     mmio_w32(d, VIRTIO_MMIO_DRV_FEAT_SEL, 1);
     mmio_w32(d, VIRTIO_MMIO_DRV_FEATURES, 1u << (VIRTIO_F_VERSION_1_BIT - 32));
 
     mmio_w32(d, VIRTIO_MMIO_STATUS,
              VIRTIO_STATUS_ACKNOWLEDGE | VIRTIO_STATUS_DRIVER |
              VIRTIO_STATUS_FEATURES_OK);
     u32 st = mmio_r32(d, VIRTIO_MMIO_STATUS);
     if (!(st & VIRTIO_STATUS_FEATURES_OK)) {
         uart_puts("[seed] virtio: FEATURES_OK rejected\n");
         return -1;
     }
 
     /* Queue 0. */
     mmio_w32(d, VIRTIO_MMIO_QUEUE_SEL, 0);
     u32 qmax = mmio_r32(d, VIRTIO_MMIO_QUEUE_NUM_MAX);
     if (qmax < VQ_SIZE) {
         uart_puts("[seed] virtio: QueueNumMax < VQ_SIZE\n");
         return -1;
     }
     mmio_w32(d, VIRTIO_MMIO_QUEUE_NUM, VQ_SIZE);
 
     int i = (int)(d - blk_devs);
     /* Zero the queue page so all idx/flags start at 0. */
     for (int k = 0; k < 4096; k++) vq_pages[i][k] = 0;
 
     u64 desc_pa  = (u64)vq_desc(i);
     u64 avail_pa = (u64)vq_avail(i);
     u64 used_pa  = (u64)vq_used(i);
     mmio_w32(d, VIRTIO_MMIO_QUEUE_DESC_LO,    (u32)desc_pa);
     mmio_w32(d, VIRTIO_MMIO_QUEUE_DESC_HI,    (u32)(desc_pa >> 32));
     mmio_w32(d, VIRTIO_MMIO_QUEUE_DRIVER_LO,  (u32)avail_pa);
     mmio_w32(d, VIRTIO_MMIO_QUEUE_DRIVER_HI,  (u32)(avail_pa >> 32));
     mmio_w32(d, VIRTIO_MMIO_QUEUE_DEVICE_LO,  (u32)used_pa);
     mmio_w32(d, VIRTIO_MMIO_QUEUE_DEVICE_HI,  (u32)(used_pa >> 32));
 
     mmio_w32(d, VIRTIO_MMIO_QUEUE_READY, 1);
     mmio_w32(d, VIRTIO_MMIO_STATUS,
              VIRTIO_STATUS_ACKNOWLEDGE | VIRTIO_STATUS_DRIVER |
              VIRTIO_STATUS_FEATURES_OK | VIRTIO_STATUS_DRIVER_OK);
 
     /* virtio-blk config @+0 = capacity (sectors, little-endian u64). */
     u32 cap_lo = *(volatile u32 *)(d->regs + VIRTIO_MMIO_CONFIG + 0);
     u32 cap_hi = *(volatile u32 *)(d->regs + VIRTIO_MMIO_CONFIG + 4);
     d->capacity_sectors = ((u64)cap_hi << 32) | cap_lo;
     d->present = 1;
     return 1;
 }
 
 /* Issue one request (VIRTIO_BLK_T_IN or _OUT) and poll for completion.
  * `buf` PA == VA (kheap or kernel BSS / stack). nsec ≤ 2048 (1 MB/req). */
 static int blk_request_one(int devi, u32 type, u64 sector, void *buf, u64 nsec) {
     struct blk_dev *d = &blk_devs[devi];
     /* Per-call hdr/status — both reachable via VA==PA on the kernel stack. */
     struct virtio_blk_req_hdr hdr = { .type = type, .reserved = 0, .sector = sector };
     volatile u8 status = 0xff;
 
     struct vring_desc *desc = vq_desc(devi);
     struct vring_avail *avail = vq_avail(devi);
     struct vring_used *used = vq_used(devi);
 
     desc[0].addr  = (u64)&hdr;
     desc[0].len   = (u32)sizeof(hdr);
     desc[0].flags = VRING_DESC_F_NEXT;
     desc[0].next  = 1;
 
     desc[1].addr  = (u64)buf;
     desc[1].len   = (u32)(nsec * 512);
     /* For READ (T_IN), device writes into our buffer (F_WRITE).
      * For WRITE (T_OUT), device reads our buffer (no F_WRITE). */
     desc[1].flags = VRING_DESC_F_NEXT | (type == VIRTIO_BLK_T_IN ? VRING_DESC_F_WRITE : 0);
     desc[1].next  = 2;
 
     desc[2].addr  = (u64)&status;
     desc[2].len   = 1;
     desc[2].flags = VRING_DESC_F_WRITE;
     desc[2].next  = 0;
 
     u16 head = 0;
     u16 ai = avail->idx;
     avail->ring[ai % VQ_SIZE] = head;
     arch_wmb();
     avail->idx = ai + 1;
     arch_wmb();
 
     mmio_w32(d, VIRTIO_MMIO_QUEUE_NOTIFY, 0);
 
     /* Poll used.idx — single in-flight, advances by exactly one. */
     while (used->idx == ai) {
         arch_pause();
     }
     arch_rmb();
 
     /* Acknowledge any pending interrupt status (we don't service IRQs but
      * the device sets these bits anyway). */
     u32 is = mmio_r32(d, VIRTIO_MMIO_INT_STATUS);
     if (is) mmio_w32(d, VIRTIO_MMIO_INT_ACK, is);
 
     if (status != 0) {
         uart_puts("[seed] virtio-blk req failed status="); uart_putd((i64)status);
         uart_puts("\n");
         return -1;
     }
     return 0;
 }
 
 /* Multi-chunk read/write. Chunks at 1 MB (2048 sectors) per request. */
 #define BLK_CHUNK_SECTORS 2048
 
 static int blk_io(int devi, u32 type, u64 sector, u8 *buf, u64 nsec) {
     while (nsec) {
         u64 n = nsec;
         if (n > BLK_CHUNK_SECTORS) n = BLK_CHUNK_SECTORS;
         if (blk_request_one(devi, type, sector, buf, n) < 0) return -1;
         sector += n;
         buf    += n * 512;
         nsec   -= n;
     }
     return 0;
 }
 
 static int blk_read(int devi, u64 sector, void *buf, u64 nsec) {
     return blk_io(devi, VIRTIO_BLK_T_IN, sector, buf, nsec);
 }
 static int blk_write(int devi, u64 sector, const void *buf, u64 nsec) {
     return blk_io(devi, VIRTIO_BLK_T_OUT, sector, (u8 *)buf, nsec);
 }
 
 /* Probe every populated MMIO slot the DTB advertised; bring up block
  * devices; identify blk0 vs blk1 by reading sector 0 — the cpio newc
  * magic ("070701") is on blk0, the other is blk1. Panics if exactly one
  * of each isn't found. */
 static int g_blk_input  = -1;     /* index in blk_devs[] for cpio input */
 static int g_blk_output = -1;     /* index for output dump              */
 
 static void blk_init(struct dtb_info *dt) {
     int n_blocks = 0;
     for (int i = 0; i < dt->virtio_mmio_n; i++) {
         u64 pa = dt->virtio_mmio_pa[i];
         if (n_blocks >= BLK_DEV_MAX) break;
         /* Stage into blk_devs[n_blocks] so blk_init_one's index-derived
          * vq page assignment is correct from the start. */
         struct blk_dev *d = &blk_devs[n_blocks];
         d->regs = arch_mmio_ptr(pa);
         d->capacity_sectors = 0;
         d->present = 0;
         int r = blk_init_one(d);
         if (r > 0) {
             n_blocks++;
         } else if (r < 0) {
             uart_puts("[seed] virtio: init failed at PA="); uart_putx(pa);
             uart_puts("\n");
             hang();
         }
     }
     blk_n_devs = n_blocks;
     if (n_blocks != 2) {
         uart_puts("[seed] virtio-blk: expected 2 block devices, got ");
         uart_putd((i64)n_blocks); uart_puts("\n");
         hang();
     }
 
     /* Identify blk0 (cpio) vs blk1 (output) by reading sector 0. */
     __attribute__((aligned(16))) static u8 probe[512];
     for (int i = 0; i < n_blocks; i++) {
         if (blk_read(i, 0, probe, 1) < 0) {
             uart_puts("[seed] virtio-blk: probe read failed dev=");
             uart_putd((i64)i); uart_puts("\n");
             hang();
         }
         int is_cpio = str_starts((const char *)probe, CPIO_NEWC_MAGIC);
         if (is_cpio) {
             if (g_blk_input >= 0) {
                 uart_puts("[seed] virtio-blk: multiple cpio disks\n");
                 hang();
             }
             g_blk_input = i;
         } else {
             if (g_blk_output >= 0) {
                 uart_puts("[seed] virtio-blk: multiple non-cpio disks\n");
                 hang();
             }
             g_blk_output = i;
         }
     }
     if (g_blk_input < 0 || g_blk_output < 0) {
         uart_puts("[seed] virtio-blk: failed to identify in/out\n");
         hang();
     }
     uart_puts("[seed] virtio-blk: in=dev"); uart_putd((i64)g_blk_input);
     uart_puts(" cap=");                     uart_putd((i64)blk_devs[g_blk_input].capacity_sectors);
     uart_puts(" sec  out=dev");             uart_putd((i64)g_blk_output);
     uart_puts(" cap=");                     uart_putd((i64)blk_devs[g_blk_output].capacity_sectors);
     uart_puts(" sec\n");
 }
 
 /* ─── In-memory tmpfs from cpio newc ────────────────────────────────────── */
 
 /* boot5 stages a full musl tree in the cpio (~1300 .c sources + ~1200
  * headers/aux) plus per-TU .o outputs (~1300) — observed cpio entry
  * count is ~2600 inputs + ~1300 outputs ≈ 3900. Round up to 4096 to
  * leave headroom; the struct is ~120 bytes so this costs ~480 KB of
  * kernel BSS — comfortably within the 192 MB kheap. Path length is 96
  * to fit "tmp/musl-1.2.5/obj/src/<sub>/<name>.o" (paths stored with
  * leading slashes stripped, so these are user-visible as
  * /tmp/musl-1.2.5/obj/...). */
 #define MAX_FILES 4096
 struct file {
     int   used;
     char  path[96];
     u8   *data;
     u64   len;
     u64   cap;
 };
 static struct file files[MAX_FILES];
 
 /* Resolve "." and ".." segments in `in` into `out`. Required because real
  * filesystems normalize `foo/../bar` → `bar` at lookup time, but our
  * tmpfs is a flat path → blob map; without this, tcc's pstrcat-style
  * include resolution (e.g. "/tmp/musl-1.2.5/src/include/" +
  * "../../include/features.h") produces a literal path that misses the
  * real entry "tmp/musl-1.2.5/include/features.h". The buffers are sized
  * for our worst case (~96-char paths) plus headroom for the unresolved
  * form. Trailing slashes are dropped. */
 static void normalize_path(const char *in, char *out, int outsz) {
     char buf[256];
     int n = 0;
     while (in[n] && n < (int)sizeof(buf) - 1) { buf[n] = in[n]; n++; }
     buf[n] = 0;
 
     const char *segs[64];
     int seg_lens[64];
     int nsegs = 0;
     int i = 0;
     while (buf[i]) {
         int start = i;
         while (buf[i] && buf[i] != '/') i++;
         int len = i - start;
         if (buf[i]) { buf[i] = 0; i++; }
         if (len == 0) continue;
         if (len == 1 && buf[start] == '.') continue;
         if (len == 2 && buf[start] == '.' && buf[start + 1] == '.') {
             if (nsegs > 0) nsegs--;
             continue;
         }
         if (nsegs < 64) {
             segs[nsegs] = &buf[start];
             seg_lens[nsegs] = len;
             nsegs++;
         }
     }
 
     int o = 0;
     for (int k = 0; k < nsegs; k++) {
         for (int j = 0; j < seg_lens[k] && o < outsz - 1; j++)
             out[o++] = segs[k][j];
         if (k < nsegs - 1 && o < outsz - 1) out[o++] = '/';
     }
     out[o] = 0;
 }
 
 static int find_file(const char *path) {
     while (*path == '/') path++;
     char norm[128];
     normalize_path(path, norm, sizeof(norm));
     for (int i = 0; i < MAX_FILES; i++) {
         if (files[i].used && str_eq(files[i].path, norm)) return i;
     }
     return -1;
 }
 
 static int new_file(const char *path) {
     while (*path == '/') path++;
     /* Normalize at store time so all later lookups match regardless of
      * how the caller spelled `..` / `.`. */
     char norm[128];
     normalize_path(path, norm, sizeof(norm));
     for (int i = 0; i < MAX_FILES; i++) {
         if (!files[i].used) {
             files[i].used = 1;
             int j = 0;
             while (norm[j] && j < (int)sizeof(files[i].path) - 1) {
                 files[i].path[j] = norm[j]; j++;
             }
             files[i].path[j] = 0;
             files[i].data = 0;
             files[i].len = 0;
             files[i].cap = 0;
             return i;
         }
     }
     return -1;
 }
 
 static u64 hex_n(const char *s, int n) {
     u64 v = 0;
     for (int i = 0; i < n; i++) {
         char c = s[i];
         v <<= 4;
         if (c >= '0' && c <= '9') v |= (u64)(c - '0');
         else if (c >= 'a' && c <= 'f') v |= (u64)(c - 'a' + 10);
         else if (c >= 'A' && c <= 'F') v |= (u64)(c - 'A' + 10);
     }
     return v;
 }
 
 static void parse_cpio(const void *cpio, u64 total) {
     const u8 *p = cpio;
     const u8 *end = p + total;
     while (p + CPIO_NEWC_HDR_SIZE <= end) {
         if (!str_starts((const char *)p, CPIO_NEWC_MAGIC)) break;
         u64 mode    = hex_n((const char *)CPIO_NEWC_FIELD(p, 1),  8);
         u64 fsz     = hex_n((const char *)CPIO_NEWC_FIELD(p, 6),  8);
         u64 nsz     = hex_n((const char *)CPIO_NEWC_FIELD(p, 11), 8);
         const char *name = (const char *)(p + CPIO_NEWC_HDR_SIZE);
         if (str_eq(name, "TRAILER!!!")) break;
 
         u64 hstride = (CPIO_NEWC_HDR_SIZE + nsz + 3) & ~3UL;
         u64 fstride = (fsz + 3) & ~3UL;
         const u8 *fdata = p + hstride;
 
         int is_dir = ((mode & CPIO_MODE_TYPE_MASK) == CPIO_MODE_TYPE_DIR);
         int is_reg = ((mode & CPIO_MODE_TYPE_MASK) == CPIO_MODE_TYPE_REG);
         if (is_reg && !str_eq(name, ".")) {
             int idx = new_file(name);
             if (idx >= 0) {
                 /* Copy out — we'll let the user write back later if needed. */
                 files[idx].data = kalloc(fsz ? fsz : 1);
                 files[idx].cap  = fsz ? fsz : 1;
                 files[idx].len  = fsz;
                 if (fsz) mem_cpy(files[idx].data, fdata, fsz);
             } else {
                 /* Silent drops here are how MAX_FILES being too low
                  * masquerades as random "file not found" errors during
                  * the build — surface it loudly. */
                 uart_puts("[seed] WARN: cpio entry dropped (MAX_FILES "
                           "exhausted): "); uart_puts(name); uart_puts("\n");
             }
         }
         (void)is_dir;
         p += hstride + fstride;
     }
 }
 
 /* ─── ELF64 static loader ───────────────────────────────────────────────── */
 
 struct ehdr { u8 e_ident[16]; u16 e_type, e_machine; u32 e_version; u64 e_entry, e_phoff, e_shoff; u32 e_flags; u16 e_ehsize, e_phentsize, e_phnum, e_shentsize, e_shnum, e_shstrndx; };
 struct phdr { u32 p_type, p_flags; u64 p_offset, p_vaddr, p_paddr, p_filesz, p_memsz, p_align; };
 
 #define PT_LOAD 1
 
 /* Highest VA touched by the most recently loaded image's PT_LOAD segments
  * (after USER_VA_HI clipping). load_elf updates this; kmain / sys_spawn
  * use it to seed brk_base above the user image's BSS. */
 static u64 g_user_image_end;
 
 static u64 load_elf(const u8 *elf) {
     const struct ehdr *eh = (const struct ehdr *)elf;
     if (!(eh->e_ident[0] == 0x7f && eh->e_ident[1] == 'E' &&
           eh->e_ident[2] == 'L'  && eh->e_ident[3] == 'F')) {
         uart_puts("ELF: bad magic\n"); return 0;
     }
     if (eh->e_machine != ARCH_ELF_MACHINE) {
         uart_puts("ELF: not "); uart_puts(ARCH_ELF_MACHINE_NAME); uart_puts("\n"); return 0;
     }
     /* p_flags (R/W/X) are deliberately ignored: the user mapping is one
      * giant Normal-memory RWX region (see arch_setup_mmu). OS.md
      * §"Memory model" permits this — there's no W^X enforcement in the
      * contract, and tcc-boot2 never JITs.
      *
      * Segments are clipped at USER_VA_HI: a binary may declare a BSS that
      * extends past the mapped user window (scheme1 reserves ~256 MB), and
      * a naive mem_set would walk into the device-block region above and
      * trigger an external abort. The user image gets only the portion of
      * its memsz that fits in the user pool; if user code later touches
      * the unmapped tail, that's a user-space fault, not a kernel panic. */
     u64 hi = 0;
     for (int i = 0; i < eh->e_phnum; i++) {
         const struct phdr *ph = (const struct phdr *)(elf + eh->e_phoff + (u64)i * eh->e_phentsize);
         if (ph->p_type != PT_LOAD) continue;
         u64 vaddr = ph->p_vaddr;
         u64 filesz = ph->p_filesz;
         u64 memsz  = ph->p_memsz;
         if (vaddr >= USER_VA_HI) continue;          /* segment fully out of window */
         u64 reach = USER_VA_HI - vaddr;
         if (filesz > reach) filesz = reach;
         if (memsz > reach)  memsz = reach;
         u8 *dst = (u8 *)vaddr;
         const u8 *src = elf + ph->p_offset;
         mem_cpy(dst, src, filesz);
         if (memsz > filesz)
             mem_set(dst + filesz, 0, memsz - filesz);
         u64 end = vaddr + memsz;
         if (end > hi) hi = end;
     }
     /* Round up to 16 bytes so callers can use it directly as brk_base. */
     g_user_image_end = (hi + 15) & ~15UL;
     /* I-cache sync (cheap insurance even with caches off). */
     arch_icache_sync();
     return eh->e_entry;
 }
 
 /* ─── Syscall layer (Tier 1) ────────────────────────────────────────────── */
 
 #define MAX_FD 32
 struct fdent { int used; int fidx; u64 pos; int wflag; int append; };
 static struct fdent fdtab[MAX_FD];
 
 /* User program break (single-process). */
 static u64 brk_base;
 static u64 brk_cur;
 static u64 brk_max;
 
 #define EBADF   9
 #define ENOENT  2
 #define EINVAL  22
 #define EMFILE  24
 #define EFAULT  14
 #define ENOSPC  28
 
 #define O_RDONLY  0
 #define O_WRONLY  1
 #define O_RDWR    2
 #define O_CREAT   0100
 #define O_TRUNC   01000
 #define O_APPEND  02000
 
 #define AT_FDCWD (-100)
 
 #define SYS_unlinkat   ARCH_SYS_unlinkat
 #define SYS_openat     ARCH_SYS_openat
 #ifdef ARCH_SYS_open
 #define SYS_open       ARCH_SYS_open
 #endif
 #define SYS_close      ARCH_SYS_close
 #define SYS_lseek      ARCH_SYS_lseek
 #define SYS_read       ARCH_SYS_read
 #define SYS_write      ARCH_SYS_write
 #define SYS_exit_group ARCH_SYS_exit_group
 #define SYS_waitid     ARCH_SYS_waitid
 #define SYS_brk        ARCH_SYS_brk
 /* Private syscall number, deliberately outside the normal Linux range.
  * The scheme1 prelude probes (sys-spawn) once at init: on Linux this
  * number is unmapped so the probe gets -ENOSYS and the prelude falls
  * back to the classic clone+execve path; on the seed kernel the probe
  * succeeds (or returns -ENOENT for a missing file) and the prelude uses
  * sys-spawn for every (run …) thereafter. */
 #define SYS_spawn      ARCH_SYS_spawn
 
 #define ECHILD     10
 #define EAGAIN     11
 #define ENOEXEC     8
 
 static i64 sys_write(int fd, const void *buf, u64 len) {
     if (fd == 1 || fd == 2) {
         const u8 *s = buf;
         for (u64 i = 0; i < len; i++) uart_putc((char)s[i]);
         return (i64)len;
     }
     if (fd < 3 || fd >= MAX_FD || !fdtab[fd].used || !fdtab[fd].wflag) return -EBADF;
     struct file *f = &files[fdtab[fd].fidx];
     u64 pos = fdtab[fd].append ? f->len : fdtab[fd].pos;
     u64 need = pos + len;
     if (need > f->cap) {
         u64 ncap = f->cap ? f->cap : 64;
         while (ncap < need) ncap *= 2;
         u8 *nd = kalloc(ncap);
         if (f->len) mem_cpy(nd, f->data, f->len);
         f->data = nd;
         f->cap = ncap;
     }
     mem_cpy(f->data + pos, buf, len);
     if (need > f->len) f->len = need;
     fdtab[fd].pos = pos + len;
     return (i64)len;
 }
 
 static i64 sys_read(int fd, void *buf, u64 len) {
     if (fd == 0) return 0;
     if (fd < 3 || fd >= MAX_FD || !fdtab[fd].used) return -EBADF;
     struct file *f = &files[fdtab[fd].fidx];
     u64 pos = fdtab[fd].pos;
     if (pos >= f->len) return 0;
     u64 n = len;
     if (pos + n > f->len) n = f->len - pos;
     mem_cpy(buf, f->data + pos, n);
     fdtab[fd].pos = pos + n;
     return (i64)n;
 }
 
 static i64 sys_openat(int dirfd, const char *path, int flags, int mode) {
     (void)dirfd; (void)mode;
     int fidx = find_file(path);
     int wflag = (flags & 3) != 0;
     if (fidx < 0) {
         if (!(flags & O_CREAT)) return -ENOENT;
         fidx = new_file(path);
         if (fidx < 0) return -ENOSPC;
     } else if (flags & O_TRUNC) {
         files[fidx].len = 0;
     }
     int fd = -1;
     for (int i = 3; i < MAX_FD; i++) {
         if (!fdtab[i].used) { fd = i; break; }
     }
     if (fd < 0) return -EMFILE;
     fdtab[fd].used = 1;
     fdtab[fd].fidx = fidx;
     fdtab[fd].pos = 0;
     fdtab[fd].wflag = wflag;
     fdtab[fd].append = (flags & O_APPEND) ? 1 : 0;
     return fd;
 }
 
 static i64 sys_close(int fd) {
     if (fd < 3 || fd >= MAX_FD || !fdtab[fd].used) return -EBADF;
     fdtab[fd].used = 0;
     return 0;
 }
 
 static i64 sys_lseek(int fd, i64 off, int whence) {
     if (fd < 3 || fd >= MAX_FD || !fdtab[fd].used) return -EBADF;
     struct file *f = &files[fdtab[fd].fidx];
     i64 base;
     if (whence == 0) base = 0;
     else if (whence == 1) base = (i64)fdtab[fd].pos;
     else if (whence == 2) base = (i64)f->len;
     else return -EINVAL;
     i64 np = base + off;
     if (np < 0) return -EINVAL;
     fdtab[fd].pos = (u64)np;
     return np;
 }
 
 static i64 sys_brk(u64 addr) {
     if (addr == 0) return (i64)brk_cur;
     if (addr < brk_base || addr > brk_max) return (i64)brk_cur;
     brk_cur = addr;
     return (i64)brk_cur;
 }
 
 static i64 sys_unlinkat(int dirfd, const char *path, int flags) {
     /* dirfd is ignored: tmpfs is a flat path map, all callers pass
      * AT_FDCWD. flags (AT_REMOVEDIR) is ignored too — there are no
      * directories in our tmpfs, so the dir-vs-file distinction the
      * flag selects has no observable effect. */
     (void)dirfd; (void)flags;
     int fidx = find_file(path);
     if (fidx < 0) return -ENOENT;
     files[fidx].used = 0;
     return 0;
 }
 
 /* ─── Tier 2: atomic spawn (spawn / waitid / exit_group) ────────────────── */
 /*
  * The boot2 chain's process-creation shape (scheme1/prelude.scm `(spawn …)`)
  * is rigidly synchronous: parent creates a child to run a single program,
  * waits for it, reads the exit code. Nothing else runs in the child between
  * creation and the new program's entry, and nothing else runs in the
  * parent between creation and wait.
  *
  * We implement that as a single atomic syscall on a single-threaded kernel:
  *
  *   sys_spawn   → capture path+argv into kernel buffers (still reading from
  *                 the parent's pool); push parent state (regs, brk, fd
  *                 table, current pool) onto proc_stack; remap user VAs to
  *                 the alternate pool with NO COPY (the child won't read any
  *                 byte of the parent's pool — load_elf overwrites just the
  *                 PT_LOAD ranges and build_user_stack writes the top of the
  *                 user VA window); load_elf into the alternate pool, reset
  *                 brk, build the user stack, rewrite tf so trap return
  *                 enters the new program at its entry with the new stack.
  *   sys_exit    → if proc_stack non-empty: stash exit code in last_child,
  *                 swap user VAs back to the parent's pool (no copy — the
  *                 parent's pool was never written by the child), restore
  *                 regs/brk/fds, cache-sync (the user VAs now resolve to
  *                 different physical pages), set tf so trap return resumes
  *                 the parent's spawn() call with child pid. If proc_stack
  *                 empty: real exit (dump tmpfs, arch shutdown).
  *   sys_waitid  → return last_child's exit code via the siginfo struct.
  *
  * No actual concurrency. The "parent" is suspended at the moment of spawn
  * and resumed only when the child calls exit_group.
  *
  * Memory cost per spawn: zero copy. User-map rewrite + TLB/cache sync
  * + load_elf (which copies just the new image's PT_LOAD bytes, typically
  * ~1 MB for tcc). This replaces the previous clone/execve design which
  * paid one 768 MB mem_cpy per fork to seed the child's pool with parent
  * state — needed only because scheme1 ran a few interpreter-bytecode
  * cells of user code between clone and execve, which would otherwise
  * mutate the parent's BSS heap-allocator globals (heap_next,
  * current_heap_next_ptr, scratch_next). Folding clone+execve into one
  * syscall closes that window entirely.
  */
 
 /* Forward decls for state defined further down. boot5's per-source
  * compile passes ~25 argv entries / ~750 bytes, but the final
  * `tcc -ar rcs libc.a obj1 … obj1263` call passes ~1300 entries totalling
  * ~65 KB of strings. MAX_ARGV / spawn_argv_pool size for that worst
  * case; both are kept generous so a future taller chain doesn't hit a
  * silent-truncation cliff. */
 #define MAX_ARGV 2048
 static u64 build_user_stack(u64 stack_top, int argc, char **argv);
 static int tokenise(char *src, char **argv, int cap);
 
 #define MAX_PROC_DEPTH 1
 
 struct proc_save {
     int active;
     u64 child_pid;
     /* Saved trap-frame state — enough to resume the parent at the trap
      * instruction following its sys_spawn. The return register is
      * overwritten with child_pid at restore time. */
     struct trapframe tf;
     u64 user_sp;
     /* Per-process state at the moment of spawn. brk_base is saved alongside
      * brk_cur because sys_spawn resets it above the new image's end-of-bss;
      * the parent's value comes back with the parent's pool. */
     u64 brk_base_save;
     u64 brk_cur_save;
     struct fdent fdtab_save[MAX_FD];
     int pool_save;     /* parent's user pool (0=A, 1=B) */
 };
 
 /* Rewrite the user-VA mapping to point at pool `which`, then flush TLB. */
 static void swap_user_pool(int which) {
     arch_swap_user_pool(which);
     current_pool = which;
 }
 
 static struct proc_save proc_stack[MAX_PROC_DEPTH];
 static int proc_depth = 0;
 static u64 g_next_pid = 2;
 
 /* The most recently exited child, for sys_waitid to consume. */
 static int last_child_valid = 0;
 static u64 last_child_pid = 0;
 static int last_child_code = 0;
 
 /* sys_spawn captures path+argv from the parent's pool into kernel buffers
  * BEFORE swapping pools — load_elf will only ever read from the cpio-
  * staged file (kernel state) and write to the alternate pool, but the
  * argv strings the caller passed live in the parent's pool, which we're
  * about to stop mapping. The pool + argv pointer table sit in BSS
  * (rather than on the kernel stack) because MAX_ARGV * 8 = 16 KB is
  * too large to put on the syscall stack. */
 static char  spawn_argv_pool[131072];     /* 128 KB; boot5 ar peaks ~65 KB */
 static char *spawn_argv_ptrs[MAX_ARGV];
 static i64 sys_spawn(struct trapframe *tf, const char *path, char **argv) {
     if (!path) return -EFAULT;
     if (proc_depth >= MAX_PROC_DEPTH) return -EAGAIN;
 
     /* Copy path out of the parent's pool first (find_file uses kernel
      * state, but the caller's `path` pointer is into user memory). */
     char path_buf[128];
     int pn = 0;
     while (path[pn] && pn < 127) { path_buf[pn] = path[pn]; pn++; }
     path_buf[pn] = 0;
     int fidx = find_file(path_buf);
     if (fidx < 0) return -ENOENT;
 
     /* Capture argv strings into spawn_argv_pool (kernel BSS, not the
      * user pool — survives the pool swap below). Truncation here is
      * silent but loud: we panic-warn on the UART so a too-low MAX_ARGV
      * surfaces as a kernel message, not a downstream link failure. */
     int argc = 0;
     int pool_off = 0;
     if (argv) {
         while (argc < MAX_ARGV - 1 && argv[argc]) {
             const char *s = argv[argc];
             int n = 0;
             while (s[n] && pool_off + n < (int)sizeof(spawn_argv_pool) - 1) n++;
             for (int j = 0; j < n; j++) spawn_argv_pool[pool_off + j] = s[j];
             spawn_argv_pool[pool_off + n] = 0;
             spawn_argv_ptrs[argc] = &spawn_argv_pool[pool_off];
             pool_off += n + 1;
             argc++;
         }
         if (argv[argc]) {
             uart_puts("[seed] WARN: sys_spawn argv truncated at MAX_ARGV="
                       ); uart_putd(MAX_ARGV); uart_puts(" for path=");
             uart_puts(path_buf); uart_puts("\n");
         }
     }
     if (argc == 0) {
         /* Synthesise argv[0] from the path so user code that reads argv[0]
          * doesn't crash. */
         int n = 0;
         while (path_buf[n] && pool_off + n < (int)sizeof(spawn_argv_pool) - 1) n++;
         for (int j = 0; j < n; j++) spawn_argv_pool[pool_off + j] = path_buf[j];
         spawn_argv_pool[pool_off + n] = 0;
         spawn_argv_ptrs[0] = &spawn_argv_pool[pool_off];
         pool_off += n + 1;
         argc = 1;
     }
 
     /* Save parent state — regs, brk, fd table, which pool the parent ran
      * in. After sys_exit_or_resume_parent restores from this frame, the
      * parent's spawn() call returns with child_pid. */
     struct proc_save *p = &proc_stack[proc_depth];
     p->active = 1;
     p->child_pid = g_next_pid++;
     p->tf = *tf;
     p->user_sp = arch_read_user_sp();
     p->brk_base_save = brk_base;
     p->brk_cur_save  = brk_cur;
     for (int i = 0; i < MAX_FD; i++) p->fdtab_save[i] = fdtab[i];
     p->pool_save = current_pool;
 
     /* Swap to the alternate pool. NO COPY: the child will only read
      * memory that load_elf writes (its own PT_LOAD segments) and what
      * build_user_stack writes (top of user VA). Stale bytes elsewhere in
      * the alt pool are user-invisible — sbrk pages aren't zeroed but
      * neither were they under the old execve path. */
     int new_pool = current_pool ^ 1;
     swap_user_pool(new_pool);
     proc_depth++;
 
     /* Load new ELF into the (just-swapped) alt pool. files[fidx].data is
      * in kernel heap, not the user pool, so this read is unaffected. */
     u64 entry = load_elf(files[fidx].data);
     if (!entry) {
         /* Roll back: alt pool is in undefined state but parent pool is
          * still pristine. Swap back and pop proc_stack. */
         proc_depth--;
         swap_user_pool(p->pool_save);
         return -ENOEXEC;
     }
 
     /* Reset brk above the new image's end-of-bss, page-aligned up.
      * Some seed binaries (e.g. riscv64 hex2) embed PC-relative scratch
      * buffers in the bytes just past their loaded image and assume brk
      * lives a full page beyond — Linux rounds brk to PAGE_SIZE. If we
      * placed the heap immediately after the image (16-byte aligned), a
      * write through the in-binary scratch overlaps the first heap node
      * and silently corrupts the user's data structures. */
     brk_base = g_user_image_end ? g_user_image_end : USER_VA_LO;
     brk_base = (brk_base + 0xfffUL) & ~0xfffUL;
     brk_cur  = brk_base;
 
     /* Build new user stack at top of user VA window. */
     u64 new_sp = build_user_stack(USER_VA_HI, argc, spawn_argv_ptrs);
 
     /* Rewrite trap frame so eret enters the child at the new image's
      * entry with a clean register state and the new stack. The parent's
      * regs sit on proc_stack until sys_exit_or_resume_parent restores
      * them on child exit. */
     arch_clear_to_user_entry(tf, entry);
     /* Some backends keep the user stack pointer outside the saved
      * trapframe, so set it through the arch hook. */
     arch_write_user_sp(new_sp);
     /* Returning 0; dispatcher writes the arch return register. The child's
      * _start reads argc/argv from the stack, so the return register is
      * don't-care. */
     return 0;
 }
 
 static i64 sys_waitid(struct trapframe *tf, int idtype, u64 id,
                       void *info, int options) {
     (void)tf; (void)idtype; (void)id; (void)options;
     if (!last_child_valid) return -ECHILD;
     /* scheme1/prelude.scm:497-506 reads info[8]=si_code (CLD_EXITED=1) and
      * info[24]=si_status. siginfo_t is sparsely written — zero the rest so
      * the prelude's view is deterministic. */
     if (info) {
         u8 *p = info;
         for (int i = 0; i < 128; i++) p[i] = 0;
         u32 *si_code   = (u32 *)(p + 8);
         u32 *si_status = (u32 *)(p + 24);
         *si_code   = 1;                       /* CLD_EXITED */
         *si_status = (u32)last_child_code;
     }
     last_child_valid = 0;
     return 0;
 }
 
 static int g_exit_code = 0;
 static int g_exited = 0;
 
 /* On-disk dump format on blk1 (SEEDFS, sector-aligned, little-endian):
  *
  *   sector 0:        struct seedfs_hdr {
  *                       char magic[8]    = "SEEDFS\0\0";
  *                       u32  nfiles;
  *                       u32  reserved;
  *                    };  (16 B; rest of sector zero-padded)
  *   sector 1..T:     nfiles directory entries, 4 entries/sector:
  *                    struct seedfs_ent {
  *                       char path[96];
  *                       u32  data_offset_sectors;
  *                       u32  _pad;
  *                       u64  size_bytes;
  *                    };  (112 B; T = ceil(nfiles/4))
  *   sector T+1..:    file data, each file padded up to a 512-byte boundary.
  *
  * The host-side extractor (extract-blk.sh) walks the table and writes
  * each file out by data_offset_sectors / size_bytes.
  *
  * Runs unconditionally on user exit. If the user code never reached exit
  * (kernel panic, hang, etc.) the host extractor sees no SEEDFS magic at
  * sector 0 and reports the missing-exit failure mode. */
 
 #define SEEDFS_ENT_SZ 112
 
 struct seedfs_hdr {
     char magic[8];
     u32  nfiles;
     u32  reserved;
 };
 
 struct seedfs_ent {
     char path[96];
     u32  data_offset_sectors;
     u32  _pad;
     u64  size_bytes;
 };
 
 /* Scratch sector for trailing-byte padding of files whose size isn't a
  * multiple of 512. Single 512-byte buffer is enough — we serialise file
  * writes and rezero before each use. */
 __attribute__((aligned(16))) static u8 dump_tail_sector[512];
 
 static void dump_tmpfs_blk(void) {
     /* Count active files. */
     u32 nfiles = 0;
     for (int i = 0; i < MAX_FILES; i++) if (files[i].used) nfiles++;
 
     u32 table_sectors = (nfiles + 3) / 4;
     u32 hdr_sectors   = 1 + table_sectors;
     u64 hdr_bytes     = (u64)hdr_sectors * 512;
 
     u8 *hdr_buf = kalloc(hdr_bytes);
     for (u64 i = 0; i < hdr_bytes; i++) hdr_buf[i] = 0;
 
     struct seedfs_hdr *hdr = (struct seedfs_hdr *)hdr_buf;
     hdr->magic[0]='S'; hdr->magic[1]='E'; hdr->magic[2]='E';
     hdr->magic[3]='D'; hdr->magic[4]='F'; hdr->magic[5]='S';
     hdr->magic[6]=0;   hdr->magic[7]=0;
     hdr->nfiles   = nfiles;
     hdr->reserved = 0;
 
     /* Walk files: fill table entries, write data sectors, advance cursor. */
     u32 ent_idx = 0;
     u64 cursor  = (u64)hdr_sectors;
     /* Output device capacity guard — we don't grow blk1, the host pre-
      * sized it (256 MB by default). Refuse the dump if it would exceed. */
     u64 out_cap = blk_devs[g_blk_output].capacity_sectors;
 
     for (int i = 0; i < MAX_FILES; i++) {
         if (!files[i].used) continue;
         struct seedfs_ent *e = (struct seedfs_ent *)(hdr_buf + 512 +
                                   (u64)ent_idx * SEEDFS_ENT_SZ);
         int j = 0;
         while (files[i].path[j] && j < (int)sizeof(e->path) - 1) {
             e->path[j] = files[i].path[j]; j++;
         }
         e->path[j] = 0;
         e->data_offset_sectors = (u32)cursor;
         e->size_bytes          = files[i].len;
 
         u64 nsec_full = files[i].len / 512;
         u64 rem       = files[i].len - nsec_full * 512;
         u64 need      = nsec_full + (rem ? 1 : 0);
         if (cursor + need > out_cap) {
             uart_puts("[seed] dump: out.img too small for tmpfs\n");
             return;
         }
         if (nsec_full)
             blk_write(g_blk_output, cursor, files[i].data, nsec_full);
         cursor += nsec_full;
         if (rem) {
             for (int k = 0; k < 512; k++) dump_tail_sector[k] = 0;
             for (u64 k = 0; k < rem; k++)
                 dump_tail_sector[k] = files[i].data[nsec_full * 512 + k];
             blk_write(g_blk_output, cursor, dump_tail_sector, 1);
             cursor++;
         }
         ent_idx++;
     }
 
     blk_write(g_blk_output, 0, hdr_buf, hdr_sectors);
     uart_puts("[seed] dump: nfiles="); uart_putd((i64)nfiles);
     uart_puts(" cursor=");              uart_putd((i64)cursor);
     uart_puts(" sectors\n");
 }
 
 static void sys_exit_final(int code) {
     g_exit_code = code;
     g_exited = 1;
     dump_tmpfs_blk();
     uart_puts("\n[seed] user exit_group("); uart_putd(code); uart_puts(")\n");
     arch_system_off();
     arch_idle_forever();
 }
 
 /* Dispatcher-side exit_group: pops proc_stack and resumes the parent's
  * sys_spawn if there's a saved frame, otherwise falls through to the
  * real shutdown path. Returns 1 if the trap frame was rewritten (resume
  * parent), 0 if the caller should treat it as a normal trap-return path
  * (which will never happen, since sys_exit_final does not return). */
 static int sys_exit_or_resume_parent(struct trapframe *tf, int code) {
     code &= 0xff;
     if (proc_depth > 0) {
         struct proc_save *p = &proc_stack[--proc_depth];
         last_child_pid   = p->child_pid;
         last_child_code  = code;
         last_child_valid = 1;
         /* Swap the user-VA mapping back to the parent's pool. The parent's
          * physical pool was never overwritten — only the child's pool was
          * — so no mem_cpy is needed. */
         if (current_pool != p->pool_save) swap_user_pool(p->pool_save);
         brk_base = p->brk_base_save;
         brk_cur  = p->brk_cur_save;
         for (int i = 0; i < MAX_FD; i++) fdtab[i] = p->fdtab_save[i];
         /* Restore registers; the dispatcher writes the child pid into the
          * arch return register below. */
         *tf = p->tf;
         arch_write_user_sp(p->user_sp);
         /* I-cache invalidation. The parent's pool was never written, so
          * its instruction bytes (in DRAM) are byte-identical to what was
          * originally fetched. But the same user VAs were just used to
          * fetch the child's instructions from the other physical pool;
          * I-caches may hold lines tagged by VA whose translation just
          * changed, so the arch backend invalidates whatever is needed. */
         arch_icache_context_sync();
         return (int)p->child_pid;     /* >0: tells dispatcher to write this as r */
     }
     sys_exit_final(code);
     return 0;                        /* unreachable */
 }
 
 /* ─── Trap dispatch (called from start.S vector handlers) ───────────────── */
 
 i64 trap_sync(u64 esr, struct trapframe *tf);
 void trap_kernel(u64 esr, struct trapframe *tf);
 void trap_unhandled(u64 esr, struct trapframe *tf);
 
 i64 trap_sync(u64 esr, struct trapframe *tf) {
     if (ARCH_IS_SYSCALL(esr)) {
         u64 nr = ARCH_SYSCALL_NR(tf);
         u64 a0 = ARCH_SYSCALL_ARG(tf, 0), a1 = ARCH_SYSCALL_ARG(tf, 1);
         u64 a2 = ARCH_SYSCALL_ARG(tf, 2), a3 = ARCH_SYSCALL_ARG(tf, 3);
         u64 a4 = ARCH_SYSCALL_ARG(tf, 4), a5 = ARCH_SYSCALL_ARG(tf, 5);
         i64 r;
         switch (nr) {
         case SYS_read:       r = sys_read((int)a0, (void *)a1, a2); break;
         case SYS_write:      r = sys_write((int)a0, (const void *)a1, a2); break;
         case SYS_openat:     r = sys_openat((int)a0, (const char *)a1, (int)a2, (int)a3); break;
 #ifdef SYS_open
         /* amd64 hex0/hex1/hex2/M0 seed binaries call legacy `open(path,
          * flags, mode)` directly; alias to openat(AT_FDCWD, ...). */
         case SYS_open:       r = sys_openat(AT_FDCWD, (const char *)a0, (int)a1, (int)a2); break;
 #endif
         case SYS_close:      r = sys_close((int)a0); break;
         case SYS_lseek:      r = sys_lseek((int)a0, (i64)a1, (int)a2); break;
         case SYS_brk:        r = sys_brk(a0); break;
         case SYS_unlinkat:   r = sys_unlinkat((int)a0, (const char *)a1, (int)a2); break;
         case SYS_spawn:      r = sys_spawn(tf, (const char *)a0, (char **)a1); break;
         case SYS_waitid:     r = sys_waitid(tf, (int)a0, a1, (void *)a2, (int)a3); break;
         case SYS_exit_group:
             r = sys_exit_or_resume_parent(tf, (int)a0);
             /* If we resumed the parent, sys_exit_or_resume_parent has
              * rewritten the trapframe; set only the arch return register. */
             if (proc_depth >= 0 && r != 0) {
                 ARCH_SET_RET(tf, r);
                 return 0;
             }
             break;
         default:
             uart_puts("[seed] ENOSYS "); uart_putd((i64)nr); uart_puts("\n");
             r = -38; /* ENOSYS */
         }
         ARCH_SET_RET(tf, r);
         (void)a4; (void)a5;
         return 0;
     }
     uart_puts("[seed] PANIC: user sync, ESR="); uart_putx(esr);
     uart_puts(" ELR=");                 uart_putx(ARCH_TF_PC(tf));
     uart_puts(" FAR=");
     u64 far = arch_fault_addr(); uart_putx(far);
     uart_puts("\n");
     hang();
 }
 
 void trap_kernel(u64 esr, struct trapframe *tf) {
     u64 far = arch_fault_addr();
     uart_puts("[seed] PANIC: kernel sync, ESR="); uart_putx(esr);
     uart_puts(" ELR=");                          uart_putx(ARCH_TF_PC(tf));
     uart_puts(" FAR=");                          uart_putx(far);
     uart_puts("\n");
     hang();
 }
 
 void trap_unhandled(u64 esr, struct trapframe *tf) {
     uart_puts("[seed] PANIC: unhandled exception, ESR="); uart_putx(esr);
     uart_puts(" ELR=");                                   uart_putx(ARCH_TF_PC(tf));
     uart_puts("\n");
     hang();
 }
 
 /* ─── User stack setup + entry ──────────────────────────────────────────── */
 
 /* Tokenise `src` in place (whitespace separators) into argv slots.
  * Writes pointers into argv[0..argc-1] and returns argc. Stops at cap. */
 static int tokenise(char *src, char **argv, int cap) {
     int argc = 0;
     char *p = src;
     while (*p && argc < cap) {
         while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++;
         if (!*p) break;
         argv[argc++] = p;
         while (*p && *p != ' ' && *p != '\t' && *p != '\n' && *p != '\r') p++;
         if (*p) *p++ = 0;
     }
     return argc;
 }
 
 /* Out-of-stack scratch for the per-call user-VA pointer table. With
  * MAX_ARGV=2048, sizeof(strs)=16 KB — too large to put on the syscall
  * stack. */
 static u64 build_user_stack_strs[MAX_ARGV];
 
 static u64 build_user_stack(u64 stack_top, int argc, char **argv) {
     /* SysV layout, low to high at the returned sp:
      *   argc, argv[0..argc-1], NULL (argv term), NULL (envp term).
      * Strings live above the vectors, in a string pool placed just below
      * stack_top so the user image's high-water mark is stable. */
     if (argc < 1) argc = 1;
     if (argc > MAX_ARGV) argc = MAX_ARGV;
 
     /* Lay strings down from stack_top - 16 (16-byte alignment slack). */
     u64 strs_top = stack_top - 16;
     u64 *strs = build_user_stack_strs;
     char *cursor = (char *)strs_top;
     for (int i = argc - 1; i >= 0; i--) {
         int n = str_n(argv[i]) + 1;
         cursor -= n;
         for (int j = 0; j < n; j++) cursor[j] = argv[i][j];
         strs[i] = (u64)cursor;
     }
 
     /* sp must hold: argc + (argc+1)*8 (argv + NULL) + 8 (envp NULL) */
     u64 sp = (u64)cursor - (u64)((argc + 3) * 8);
     sp &= ~15UL;
     u64 *p = (u64 *)sp;
     p[0] = (u64)argc;
     for (int i = 0; i < argc; i++) p[1 + i] = strs[i];
     p[1 + argc] = 0;                /* argv terminator */
     p[2 + argc] = 0;                /* envp terminator */
     return sp;
 }
 
 /* ─── kmain ─────────────────────────────────────────────────────────────── */
 
 void kmain(u64 dtb_phys) {
     arch_setup_mmu();
 
     /* Bring up heap immediately — placed at a 16MB-aligned offset above
      * our image, well clear of BSS/stack. Without -initrd reserving the
      * 0x44000000–0x4b000000 region, the full 176 MB is ours from boot. */
     u64 image_end = (u64)_end;
     kheap_ptr = (u8 *)((image_end + 0xfffful) & ~0xfffful);
     kheap_end = (u8 *)ARCH_KERNEL_HEAP_END;
 
     uart_puts("\n[seed] "); uart_puts(ARCH_NAME); uart_puts(" boot, dtb=");
     uart_putx(dtb_phys); uart_puts("\n");
 
     struct dtb_info dt = {0};
     parse_dtb((const void *)dtb_phys, &dt);
     uart_puts("[seed] mem ");      uart_putx(dt.mem_start);
     uart_puts(" + ");              uart_putx(dt.mem_size); uart_puts("\n");
     uart_puts("[seed] virtio-mmio slots="); uart_putd((i64)dt.virtio_mmio_n);
     uart_puts("\n");
     if (dt.bootargs[0]) { uart_puts("[seed] bootargs: "); uart_puts(dt.bootargs); uart_puts("\n"); }
 
     /* Bring up virtio-blk: identifies blk0 (cpio) and blk1 (output). */
     blk_init(&dt);
 
     /* Reserve the cpio buffer at the top of kheap so we can release it
      * after parse_cpio. parse_cpio kallocs each file's data below cpio_buf;
      * once it returns, the cpio buffer's bytes are dead and we can let
      * subsequent kallocs use that space. */
     u64 in_cap_sec   = blk_devs[g_blk_input].capacity_sectors;
     u64 in_cap_bytes = in_cap_sec * 512;
     u64 in_cap_aln   = (in_cap_bytes + 0xfffUL) & ~0xfffUL;
     u8 *cpio_buf     = (u8 *)((u64)kheap_end - in_cap_aln);
     u8 *kheap_end_full = kheap_end;
     kheap_end = cpio_buf;
 
     if (blk_read(g_blk_input, 0, cpio_buf, in_cap_sec) < 0) {
         uart_puts("[seed] cpio read failed\n");
         hang();
     }
     parse_cpio(cpio_buf, in_cap_bytes);
     uart_puts("[seed] tmpfs:\n");
     for (int i = 0; i < MAX_FILES; i++) {
         if (!files[i].used) continue;
         uart_puts("  /"); uart_puts(files[i].path);
         uart_puts(" ("); uart_putd((i64)files[i].len); uart_puts(" bytes)\n");
     }
 
     int init_idx = find_file("init");
     if (init_idx < 0) { uart_puts("[seed] no /init in initrd, halting\n"); hang(); }
 
     u64 entry = load_elf(files[init_idx].data);
     if (!entry) { uart_puts("[seed] load_elf failed\n"); hang(); }
     uart_puts("[seed] /init e_entry="); uart_putx(entry); uart_puts("\n");
 
     /* parse_cpio + load_elf are done — cpio buffer's bytes are dead.
      * Release the reserved tail of kheap for tmpfs file growth. */
     kheap_end = kheap_end_full;
 
     /* User runs in the L2-mapped low-VA window (USER_VA_LO..USER_VA_HI,
      * physically backed by pool A initially). Stack grows down from the top
      * of the window; brk grows up from above the loaded image's
      * end-of-bss (g_user_image_end, set by load_elf). 16 MB reserved at
      * the top for the user stack. */
     u64 ustack_top = USER_VA_HI;
     /* See sys_spawn for why brk_base is page-rounded above end-of-image. */
     brk_base = g_user_image_end ? g_user_image_end : USER_VA_LO;
     brk_base = (brk_base + 0xfffUL) & ~0xfffUL;
     brk_cur  = brk_base;
     brk_max  = USER_VA_HI - 0x01000000UL;
 
     /* Build argv. Priority:
      *   1. DTB /chosen/bootargs (whitespace-tokenised — qemu -append "...").
      *   2. /init.argv from the initramfs (one arg per line).
      *   3. Fallback: argc=1, argv[0]="init".
      * In all three cases, argv passed to user is exactly what the source
      * provided — no implicit argv[0]="init" prefix. */
     static char argv_pool[512];
     char *uargv[MAX_ARGV];
     int uargc = 0;
 
     if (dt.bootargs[0]) {
         int n = 0;
         while (dt.bootargs[n] && n < (int)sizeof(argv_pool) - 1) {
             argv_pool[n] = dt.bootargs[n]; n++;
         }
         argv_pool[n] = 0;
         uargc = tokenise(argv_pool, uargv, MAX_ARGV);
     }
     if (uargc == 0) {
         int aidx = find_file("init.argv");
         if (aidx >= 0) {
             u64 n = files[aidx].len;
             if (n >= sizeof(argv_pool)) n = sizeof(argv_pool) - 1;
             for (u64 i = 0; i < n; i++) argv_pool[i] = (char)files[aidx].data[i];
             argv_pool[n] = 0;
             uargc = tokenise(argv_pool, uargv, MAX_ARGV);
         }
     }
     if (uargc == 0) {
         argv_pool[0] = 'i'; argv_pool[1] = 'n'; argv_pool[2] = 'i';
         argv_pool[3] = 't'; argv_pool[4] = 0;
         uargv[0] = argv_pool;
         uargc = 1;
     }
     uart_puts("[seed] argv:");
     for (int i = 0; i < uargc; i++) { uart_puts(" "); uart_puts(uargv[i]); }
     uart_puts("\n");
 
     u64 user_sp = build_user_stack(ustack_top, uargc, uargv);
 
     uart_puts("[seed] eret to user, sp="); uart_putx(user_sp); uart_puts("\n");
     eret_to_user(entry, user_sp);
     /* unreachable */
 }