commit a35b5d50f67122dfadc832705faea4bead9f617c
parent e9bbba2b0ac78b989cca2d0bf6ffd66d12a5221a
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 9 May 2026 11:43:15 -0700
execmem: dual-mapping JIT regions for strict W^X
Code regions now come back as a write alias (RW, never X) and a runtime
alias (R, flipped to RX by protect, never W) backing the same physical
pages — mach_vm_remap on Apple, memfd_create+dual-mmap on Linux. No VA
ever holds W and X simultaneously. link_jit reserves per-segment so data
and rodata stay single-aliased and JITed code can still write to globals
and BSS through their own RW pages.
Diffstat:
6 files changed, 525 insertions(+), 95 deletions(-)
diff --git a/driver/env.c b/driver/env.c
@@ -14,6 +14,38 @@
#include <time.h>
#include <unistd.h>
+/* Dual-mapping back-ends for strict W^X. Picks per-platform:
+ *
+ * - Apple (any arch): mach_vm_remap creates a second VA pointing at the
+ * same physical memory. The write alias is RW; the runtime alias is
+ * left at PROT_READ until protect() flips it to PROT_READ|PROT_EXEC.
+ * No MAP_JIT, no per-thread mode bit, no entitlement coupling for
+ * unsigned dev binaries (signed/hardened-runtime binaries need the
+ * com.apple.security.cs.allow-unsigned-executable-memory entitlement).
+ *
+ * - Linux: memfd_create + two mmaps of the same fd backs both aliases
+ * with the same physical pages.
+ *
+ * - Other POSIX: fall back to a single mapping (no dual aliasing
+ * available). Still W^X — reserve returns RW, protect transitions
+ * to RX with no overlap, no RWX state.
+ */
+#if defined(__APPLE__)
+# include <mach/mach.h>
+# include <mach/mach_vm.h>
+# include <mach/vm_map.h>
+# define DRIVER_DUAL_APPLE 1
+#else
+# define DRIVER_DUAL_APPLE 0
+#endif
+
+#if defined(__linux__)
+# include <sys/syscall.h>
+# define DRIVER_DUAL_LINUX 1
+#else
+# define DRIVER_DUAL_LINUX 0
+#endif
+
/* Host-side implementations of the three vtable interfaces libcfree needs:
*
* - DriverHeapLibc — malloc/realloc/free
@@ -91,13 +123,129 @@ static int cfree_to_posix_prot(int prot)
return p;
}
-static void* execmem_reserve(void* user, size_t size, int prot)
+/* Per-region bookkeeping used by reserve/release. Only EXEC regions
+ * actually carry a token (single-mapping reservations leave it NULL).
+ * Apple stores nothing extra; Linux stores the memfd-backed write alias
+ * size so munmap can release the runtime alias separately. */
+typedef struct ExecMemToken {
+ void* write_addr;
+ void* runtime_addr;
+ size_t size;
+} ExecMemToken;
+
+static int execmem_reserve_single(size_t size, CfreeExecMemRegion* out)
+{
+ void* p = mmap(NULL, size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANON, -1, 0);
+ if (p == MAP_FAILED) return 1;
+ out->write = p;
+ out->runtime = p;
+ out->size = size;
+ out->token = NULL; /* munmap suffices on release */
+ return 0;
+}
+
+#if DRIVER_DUAL_APPLE
+static int execmem_reserve_dual_apple(size_t size, CfreeExecMemRegion* out)
+{
+ /* 1) Allocate the writable backing via mmap. */
+ void* w = mmap(NULL, size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANON, -1, 0);
+ mach_vm_address_t r_addr = 0;
+ vm_prot_t cur = 0, max = 0;
+ kern_return_t kr;
+ ExecMemToken* tok;
+ if (w == MAP_FAILED) return 1;
+
+ /* 2) Create an alias mapping. copy=FALSE makes the new VA share the
+ * same physical pages. We request VM_INHERIT_NONE so child
+ * processes don't observe the executable alias accidentally. */
+ kr = mach_vm_remap(mach_task_self(), &r_addr, (mach_vm_size_t)size,
+ /*mask=*/0,
+ VM_FLAGS_ANYWHERE,
+ mach_task_self(), (mach_vm_address_t)(uintptr_t)w,
+ /*copy=*/FALSE,
+ &cur, &max,
+ VM_INHERIT_NONE);
+ if (kr != KERN_SUCCESS) { munmap(w, size); return 1; }
+
+ /* 3) Drop the runtime alias to PROT_READ until protect() flips it.
+ * No window of writable+executable: the write alias has W (no X),
+ * the runtime alias has R only (no W, no X yet). */
+ if (mprotect((void*)(uintptr_t)r_addr, size, PROT_READ) != 0) {
+ munmap((void*)(uintptr_t)r_addr, size);
+ munmap(w, size);
+ return 1;
+ }
+
+ tok = (ExecMemToken*)malloc(sizeof(*tok));
+ if (!tok) {
+ munmap((void*)(uintptr_t)r_addr, size);
+ munmap(w, size);
+ return 1;
+ }
+ tok->write_addr = w;
+ tok->runtime_addr = (void*)(uintptr_t)r_addr;
+ tok->size = size;
+
+ out->write = w;
+ out->runtime = (void*)(uintptr_t)r_addr;
+ out->size = size;
+ out->token = tok;
+ return 0;
+}
+#endif
+
+#if DRIVER_DUAL_LINUX
+static int execmem_reserve_dual_linux(size_t size, CfreeExecMemRegion* out)
+{
+ /* memfd_create gives us an anonymous fd; two mmaps of that fd alias
+ * the same physical pages at distinct VAs. */
+ int fd = (int)syscall(SYS_memfd_create, "cfree-jit", 0u);
+ void* w;
+ void* r;
+ ExecMemToken* tok;
+ if (fd < 0) return 1;
+ if (ftruncate(fd, (off_t)size) != 0) { close(fd); return 1; }
+
+ w = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (w == MAP_FAILED) { close(fd); return 1; }
+ r = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
+ if (r == MAP_FAILED) { munmap(w, size); close(fd); return 1; }
+ /* The fd is no longer needed once both aliases are mapped. */
+ close(fd);
+
+ tok = (ExecMemToken*)malloc(sizeof(*tok));
+ if (!tok) { munmap(r, size); munmap(w, size); return 1; }
+ tok->write_addr = w;
+ tok->runtime_addr = r;
+ tok->size = size;
+
+ out->write = w;
+ out->runtime = r;
+ out->size = size;
+ out->token = tok;
+ return 0;
+}
+#endif
+
+static int execmem_reserve(void* user, size_t size, int prot,
+ CfreeExecMemRegion* out)
{
- void* p;
(void)user;
- p = mmap(NULL, size, cfree_to_posix_prot(prot),
- MAP_PRIVATE | MAP_ANON, -1, 0);
- return (p == MAP_FAILED) ? NULL : p;
+ if (!out || !size) return 1;
+ if (prot & CFREE_PROT_EXEC) {
+#if DRIVER_DUAL_APPLE
+ return execmem_reserve_dual_apple(size, out);
+#elif DRIVER_DUAL_LINUX
+ return execmem_reserve_dual_linux(size, out);
+#else
+ /* No dual-mapping primitive available: fall back to single
+ * mapping. Still W^X by mprotect transition (RW → RX). */
+ return execmem_reserve_single(size, out);
+#endif
+ }
+ return execmem_reserve_single(size, out);
}
static int execmem_protect(void* user, void* addr, size_t size, int prot)
@@ -106,10 +254,24 @@ static int execmem_protect(void* user, void* addr, size_t size, int prot)
return mprotect(addr, size, cfree_to_posix_prot(prot));
}
-static void execmem_release(void* user, void* addr, size_t size)
+static void execmem_release(void* user, CfreeExecMemRegion* region)
{
(void)user;
- munmap(addr, size);
+ if (!region || !region->size) return;
+ if (region->token) {
+ ExecMemToken* tok = (ExecMemToken*)region->token;
+ if (tok->runtime_addr && tok->runtime_addr != tok->write_addr)
+ munmap(tok->runtime_addr, tok->size);
+ if (tok->write_addr)
+ munmap(tok->write_addr, tok->size);
+ free(tok);
+ } else if (region->write) {
+ munmap(region->write, region->size);
+ }
+ region->write = NULL;
+ region->runtime = NULL;
+ region->size = 0;
+ region->token = NULL;
}
static void execmem_flush_icache(void* user, void* addr, size_t size)
@@ -122,7 +284,7 @@ static void execmem_flush_icache(void* user, void* addr, size_t size)
#endif
}
-static size_t host_page_size(void)
+static size_t driver_host_page_size(void)
{
long p = sysconf(_SC_PAGESIZE);
return (p > 0) ? (size_t)p : (size_t)0x4000;
@@ -301,7 +463,7 @@ void driver_env_init(DriverEnv* e)
e->file_io.open_writer = posix_open_writer;
e->file_io.user = e;
- g_execmem_posix.page_size = host_page_size();
+ g_execmem_posix.page_size = driver_host_page_size();
g_execmem_posix.reserve = execmem_reserve;
g_execmem_posix.protect = execmem_protect;
g_execmem_posix.release = execmem_release;
diff --git a/include/cfree.h b/include/cfree.h
@@ -221,18 +221,32 @@ typedef struct CfreeFileIO {
/* Executable-memory vtable. Required by the JIT mapper (cfree_jit_from_image)
* and the emu runtime; consulted by the linker for page-aligned segment
* layout. May be NULL for hosts that never JIT and never run the emu — link
- * layout falls back to a 16 KiB page in that case. Semantics mirror
- * mmap/mprotect/munmap closely so a posix host needs only a thin shim.
+ * layout falls back to a 16 KiB page in that case.
*
- * reserve — allocate `size` bytes (page_size-aligned) with initial
- * perms `prot`. Returns NULL on failure.
- * protect — change perms of [addr, addr+size) (page_size-aligned)
- * to `prot`. Returns 0 on success, nonzero on failure.
- * release — free a prior reservation; addr/size match the reserve
- * call.
- * flush_icache — make freshly written instructions in [addr, addr+size)
- * visible to the CPU. May be a no-op on x86; required on
- * aarch64 before transferring control to JITed code.
+ * The vtable enforces strict W^X: no virtual page is ever simultaneously
+ * writable and executable. For regions that will eventually hold code
+ * (CFREE_PROT_EXEC in the requested perms) the host returns a dual mapping —
+ * two virtual addresses that alias the same physical memory, where the
+ * `write` alias has WRITE but never EXEC, and the `runtime` alias has
+ * EXEC after a corresponding protect() call but never WRITE. Callers
+ * populate code via the `write` alias and execute / take addresses against
+ * the `runtime` alias. For non-EXEC regions a single mapping suffices and
+ * write == runtime.
+ *
+ * reserve — allocate `size` bytes (page_size-aligned) whose final
+ * perms will be `prot`. On success returns 0 and fills
+ * *out (write/runtime/size/token); returns non-zero on
+ * failure. The returned `write` alias is always RW;
+ * `runtime` starts read-only and is flipped to final
+ * perms by protect().
+ * protect — apply final perms via the runtime alias for [addr,
+ * addr+size) (page_size-aligned, lying inside the
+ * reservation's runtime alias). Returns 0 on success.
+ * release — free a prior reservation, including both aliases.
+ * flush_icache — make freshly written instructions visible to the CPU
+ * at [addr, addr+size) on the runtime alias. May be a
+ * no-op on x86; required on aarch64 before transferring
+ * control to JITed code.
*
* `prot` is a bitmask of CFREE_PROT_*. */
enum {
@@ -242,11 +256,20 @@ enum {
CFREE_PROT_EXEC = 1 << 2,
};
+typedef struct CfreeExecMemRegion {
+ void* write; /* RW alias for population; never has EXEC */
+ void* runtime; /* runtime/execution alias; never has WRITE.
+ For non-EXEC reservations equals `write`. */
+ size_t size; /* page-aligned bytes */
+ void* token; /* opaque host handle for release() */
+} CfreeExecMemRegion;
+
typedef struct CfreeExecMem {
size_t page_size;
- void* (*reserve) (void* user, size_t size, int prot);
+ int (*reserve) (void* user, size_t size, int prot,
+ CfreeExecMemRegion* out);
int (*protect) (void* user, void* addr, size_t size, int prot);
- void (*release) (void* user, void* addr, size_t size);
+ void (*release) (void* user, CfreeExecMemRegion* region);
void (*flush_icache)(void* user, void* addr, size_t size);
void* user;
} CfreeExecMem;
diff --git a/src/emu/runtime.c b/src/emu/runtime.c
@@ -49,10 +49,10 @@ static u64 align_up_u64(u64 v, u64 a)
}
struct EmuCodeRegion {
- Compiler* c;
- void* base;
- size_t size;
- uintptr_t rx_end; /* high-water of pages currently RX */
+ Compiler* c;
+ CfreeExecMemRegion region; /* dual-aliased on hosts that support it */
+ uintptr_t rx_end; /* high-water of runtime-alias pages
+ currently flipped to RX */
};
EmuCodeRegion* emu_code_region_new(Compiler* c, size_t reserve_size)
@@ -60,23 +60,28 @@ EmuCodeRegion* emu_code_region_new(Compiler* c, size_t reserve_size)
Heap* h;
const CfreeExecMem* mem;
EmuCodeRegion* r;
- void* p;
size_t aligned;
+ CfreeExecMemRegion region;
if (!c) return NULL;
h = (Heap*)c->env->heap;
mem = require_execmem(c);
aligned = (size_t)align_up_u64((u64)reserve_size, page_size_bytes(mem));
- p = mem->reserve(mem->user, aligned, CFREE_PROT_NONE);
- if (!p) return NULL;
+ /* Reserve as a code region. The host returns dual-mapped memory
+ * (writable alias / runtime alias) so the linker can write through
+ * the write alias while the runtime alias starts read-only and is
+ * flipped to RX page-by-page as cold blocks are committed. */
+ if (mem->reserve(mem->user, aligned,
+ CFREE_PROT_READ | CFREE_PROT_EXEC, ®ion) != 0) {
+ return NULL;
+ }
r = (EmuCodeRegion*)h->alloc(h, sizeof(*r), _Alignof(EmuCodeRegion));
- if (!r) { mem->release(mem->user, p, aligned); return NULL; }
+ if (!r) { mem->release(mem->user, ®ion); return NULL; }
r->c = c;
- r->base = p;
- r->size = aligned;
- r->rx_end = (uintptr_t)p;
+ r->region = region;
+ r->rx_end = (uintptr_t)region.runtime;
return r;
}
@@ -87,20 +92,21 @@ void emu_code_region_free(EmuCodeRegion* r)
if (!r) return;
h = (Heap*)r->c->env->heap;
mem = r->c->env->execmem;
- if (r->base && r->size && mem && mem->release) {
- mem->release(mem->user, r->base, r->size);
+ if (r->region.size && mem && mem->release) {
+ mem->release(mem->user, &r->region);
}
h->free(h, r, sizeof(*r));
}
uintptr_t emu_code_region_base(const EmuCodeRegion* r)
{
- return r ? (uintptr_t)r->base : 0;
+ /* Runtime alias — what JIT'd block addresses must use. */
+ return r ? (uintptr_t)r->region.runtime : 0;
}
size_t emu_code_region_size(const EmuCodeRegion* r)
{
- return r ? r->size : 0;
+ return r ? r->region.size : 0;
}
void emu_code_region_commit_rx_to(EmuCodeRegion* r, uintptr_t end)
@@ -110,20 +116,20 @@ void emu_code_region_commit_rx_to(EmuCodeRegion* r, uintptr_t end)
size_t len;
if (!r) return;
mem = require_execmem(r->c);
- base = (uintptr_t)r->base;
+ base = (uintptr_t)r->region.runtime;
page_end = (uintptr_t)align_up_u64((u64)end, page_size_bytes(mem));
/* Monotonic: never lower the high-water; chaining patches
* already-committed code and depends on it staying RX. */
if (page_end <= r->rx_end) return;
- if (page_end > base + r->size) page_end = base + r->size;
+ if (page_end > base + r->region.size) page_end = base + r->region.size;
if (page_end <= r->rx_end) return;
len = (size_t)(page_end - r->rx_end);
- /* Linker has already written + relocated the section bytes via
- * the original PROT_NONE mapping (which is technically a fault
- * unless the mapping was promoted to RW). The actual write path
- * is owned by link_resolve_extend; in v1 we expect the linker
- * to flip to RW prior to writing. RX flip happens here. */
+ /* Bytes are written through the WRITE alias by link_resolve_extend
+ * (a stub today). The runtime alias starts at PROT_READ and we flip
+ * it to PROT_READ|PROT_EXEC here; W^X is preserved because the two
+ * aliases are distinct VAs and neither holds W and X at the same
+ * time. */
if (mem->protect(mem->user, (void*)r->rx_end, len,
CFREE_PROT_READ | CFREE_PROT_EXEC) == 0) {
if (mem->flush_icache) {
diff --git a/src/link/link_jit.c b/src/link/link_jit.c
@@ -38,11 +38,18 @@ static u64 jit_page_size(Compiler* c)
return m->page_size ? (u64)m->page_size : 0x4000u;
}
+/* Per-segment runtime placement. Each segment gets its own host
+ * reservation. Code segments come back as a dual mapping (write alias /
+ * runtime alias backing the same physical pages); data and rodata are
+ * single-aliased. We populate via the write alias, encode runtime
+ * addresses against the runtime alias, then protect the runtime alias to
+ * its final perms. No virtual page is ever simultaneously writable and
+ * executable. */
struct CfreeJit {
- Compiler* c;
- LinkImage* image;
- void* base;
- size_t map_size;
+ Compiler* c;
+ LinkImage* image;
+ CfreeExecMemRegion* segs; /* one per image->nsegments */
+ u32 nsegs;
};
static int perms_for(u32 secflags)
@@ -53,14 +60,50 @@ static int perms_for(u32 secflags)
return p;
}
+/* Find the segment that contains image-relative `vaddr` and return its
+ * runtime address (the runtime alias, not the write alias). Up to 3
+ * segments after layout, so a linear scan is fine. */
+static uintptr_t vaddr_to_runtime(const LinkImage* img,
+ const CfreeExecMemRegion* segs,
+ u64 vaddr)
+{
+ u32 i;
+ for (i = 0; i < img->nsegments; ++i) {
+ const LinkSegment* s = &img->segments[i];
+ u64 lo = s->vaddr;
+ u64 hi = lo + s->mem_size;
+ if (vaddr >= lo && vaddr < hi)
+ return (uintptr_t)segs[i].runtime + (uintptr_t)(vaddr - lo);
+ }
+ return 0;
+}
+
+/* Same as above but returns the WRITE-alias address — used to determine
+ * the byte position at which to apply a relocation. The PC-relative
+ * arithmetic in link_reloc_apply uses the runtime address; the bytes
+ * themselves get patched at the matching offset of the write alias. */
+static uintptr_t vaddr_to_write(const LinkImage* img,
+ const CfreeExecMemRegion* segs,
+ u64 vaddr)
+{
+ u32 i;
+ for (i = 0; i < img->nsegments; ++i) {
+ const LinkSegment* s = &img->segments[i];
+ u64 lo = s->vaddr;
+ u64 hi = lo + s->mem_size;
+ if (vaddr >= lo && vaddr < hi)
+ return (uintptr_t)segs[i].write + (uintptr_t)(vaddr - lo);
+ }
+ return 0;
+}
+
CfreeJit* cfree_jit_from_image(LinkImage* img)
{
Compiler* c;
Heap* heap;
const CfreeExecMem* mem;
CfreeJit* jit;
- void* base;
- size_t map_size = 0;
+ CfreeExecMemRegion* segs;
u64 page;
u32 i;
@@ -70,37 +113,48 @@ CfreeJit* cfree_jit_from_image(LinkImage* img)
mem = require_execmem(c);
page = jit_page_size(c);
- /* Total mapping size = top of last segment, page-aligned. */
if (img->nsegments == 0) {
compiler_panic(c, no_loc(),
"cfree_jit_from_image: image has no segments");
}
- for (i = 0; i < img->nsegments; ++i) {
- const LinkSegment* seg = &img->segments[i];
- u64 end = seg->vaddr + align_up_u64(seg->mem_size, page);
- if (end > map_size) map_size = (size_t)end;
- }
- map_size = (size_t)align_up_u64((u64)map_size, page);
- base = mem->reserve(mem->user, map_size,
- CFREE_PROT_READ | CFREE_PROT_WRITE);
- if (!base) {
+ segs = (CfreeExecMemRegion*)heap->alloc(
+ heap, sizeof(*segs) * img->nsegments,
+ _Alignof(CfreeExecMemRegion));
+ if (!segs) {
compiler_panic(c, no_loc(),
- "cfree_jit_from_image: execmem.reserve failed");
+ "cfree_jit_from_image: oom on segment table");
}
- /* Reservation is required to be zeroed — matches mmap MAP_ANON
- * semantics; BSS is naturally zero. */
+ memset(segs, 0, sizeof(*segs) * img->nsegments);
- /* Copy each segment's file bytes to (base + vaddr). */
+ /* Reserve each segment with its FINAL perms. For EXEC segments the
+ * host returns a dual mapping (write alias / runtime alias); for
+ * data/rodata the two aliases coincide. */
+ for (i = 0; i < img->nsegments; ++i) {
+ const LinkSegment* seg = &img->segments[i];
+ size_t mlen = (size_t)align_up_u64(seg->mem_size, page);
+ if (mem->reserve(mem->user, mlen, perms_for(seg->flags),
+ &segs[i]) != 0) {
+ u32 j;
+ for (j = 0; j < i; ++j) mem->release(mem->user, &segs[j]);
+ heap->free(heap, segs, sizeof(*segs) * img->nsegments);
+ compiler_panic(c, no_loc(),
+ "cfree_jit_from_image: execmem.reserve failed");
+ }
+ }
+ /* Reservations are zeroed; BSS is naturally zero. */
+
+ /* Copy each segment's file bytes to its write alias. */
for (i = 0; i < img->nsegments; ++i) {
const LinkSegment* seg = &img->segments[i];
if (seg->file_size == 0) continue;
- memcpy((u8*)base + seg->vaddr,
+ memcpy(segs[i].write,
img->segment_bytes[i],
(size_t)seg->file_size);
}
- /* Apply relocations with runtime base. */
+ /* Apply relocations. The patch site bytes go through the write
+ * alias; PC-relative arithmetic uses the runtime alias address. */
for (i = 0; i < img->nrelocs; ++i) {
const LinkRelocApply* r = &img->relocs[i];
const LinkSymbol* tgt = &img->syms[r->target - 1];
@@ -111,40 +165,53 @@ CfreeJit* cfree_jit_from_image(LinkImage* img)
* already holds the runtime address. */
S = tgt->vaddr;
} else {
- S = tgt->vaddr + (u64)(uintptr_t)base;
+ S = (u64)vaddr_to_runtime(img, segs, tgt->vaddr);
}
- P = r->write_vaddr + (u64)(uintptr_t)base;
- P_bytes = (u8*)base + r->write_vaddr;
+ P = (u64)vaddr_to_runtime(img, segs, r->write_vaddr);
+ P_bytes = (u8*)vaddr_to_write(img, segs, r->write_vaddr);
link_reloc_apply(c, r->kind, P_bytes, S, r->addend, P);
}
- /* Flush the data caches we just wrote and invalidate the icache
- * so the CPU sees the new instructions before the protect flip.
- * Host decides whether this is a no-op (typical on x86). */
- if (mem->flush_icache) mem->flush_icache(mem->user, base, map_size);
-
- /* Flip permissions per segment. */
+ /* Flip the runtime alias of each segment to its final perms. The
+ * write alias is unaffected (still RW for any segment we'd want to
+ * write to from JITed code; for EXEC segments the write alias is
+ * orphaned after this point — JITed code is not expected to write
+ * to its own code). */
for (i = 0; i < img->nsegments; ++i) {
const LinkSegment* seg = &img->segments[i];
- size_t mlen = (size_t)align_up_u64(seg->mem_size, page);
- if (mem->protect(mem->user, (u8*)base + seg->vaddr, mlen,
+ if (mem->protect(mem->user, segs[i].runtime, segs[i].size,
perms_for(seg->flags)) != 0) {
- mem->release(mem->user, base, map_size);
+ u32 j;
+ for (j = 0; j < img->nsegments; ++j)
+ mem->release(mem->user, &segs[j]);
+ heap->free(heap, segs, sizeof(*segs) * img->nsegments);
compiler_panic(c, no_loc(),
"cfree_jit_from_image: execmem.protect failed");
}
}
+ /* Flush only the segments that will be executed, against the
+ * runtime alias (the address from which the CPU will fetch). */
+ if (mem->flush_icache) {
+ for (i = 0; i < img->nsegments; ++i) {
+ const LinkSegment* seg = &img->segments[i];
+ if (seg->flags & SF_EXEC)
+ mem->flush_icache(mem->user, segs[i].runtime, segs[i].size);
+ }
+ }
+
jit = (CfreeJit*)heap->alloc(heap, sizeof(*jit), _Alignof(CfreeJit));
if (!jit) {
- mem->release(mem->user, base, map_size);
+ for (i = 0; i < img->nsegments; ++i)
+ mem->release(mem->user, &segs[i]);
+ heap->free(heap, segs, sizeof(*segs) * img->nsegments);
compiler_panic(c, no_loc(),
"cfree_jit_from_image: oom on jit handle");
}
- jit->c = c;
- jit->image = img;
- jit->base = base;
- jit->map_size = map_size;
+ jit->c = c;
+ jit->image = img;
+ jit->segs = segs;
+ jit->nsegs = img->nsegments;
/* Take ownership of the image: undefer it from the compiler so a
* future panic doesn't reap something we still hold. */
@@ -173,11 +240,18 @@ void cfree_jit_free(CfreeJit* jit)
{
Heap* heap;
const CfreeExecMem* mem;
+ u32 i;
if (!jit) return;
heap = (Heap*)jit->c->env->heap;
mem = jit->c->env->execmem;
- if (jit->base && jit->map_size && mem && mem->release) {
- mem->release(mem->user, jit->base, jit->map_size);
+ if (jit->segs && mem && mem->release) {
+ for (i = 0; i < jit->nsegs; ++i) {
+ if (jit->segs[i].size)
+ mem->release(mem->user, &jit->segs[i]);
+ }
+ }
+ if (jit->segs) {
+ heap->free(heap, jit->segs, sizeof(*jit->segs) * jit->nsegs);
}
if (jit->image) {
/* link_image_free unfederes (no-op now) and releases storage. */
@@ -198,7 +272,7 @@ void* cfree_jit_lookup(CfreeJit* jit, const char* name)
s = &jit->image->syms[id - 1];
if (!s->defined) return NULL;
if (s->kind == SK_ABS) return (void*)(uintptr_t)s->vaddr;
- return (u8*)jit->base + s->vaddr;
+ return (void*)vaddr_to_runtime(jit->image, jit->segs, s->vaddr);
}
/* ---- inspector entries (stubs; out of scope for this cut) ---- */
diff --git a/test/cg/harness/cg_runner.c b/test/cg/harness/cg_runner.c
@@ -54,7 +54,23 @@ static void diag_emit(CfreeDiagSink* s, CfreeDiagKind k, CfreeSrcLoc loc,
}
static CfreeDiagSink g_diag = { diag_emit, NULL, 0, 0 };
-/* posix-backed CfreeExecMem for the JIT path. */
+/* posix-backed CfreeExecMem for the JIT path. Mirrors driver/env.c — see
+ * that file for the strict-W^X dual-mapping rationale. Apple uses
+ * mach_vm_remap; Linux uses memfd_create + dual mmap; other POSIX falls
+ * back to a single mapping with mprotect transitions. */
+#if defined(__APPLE__)
+# include <mach/mach.h>
+# include <mach/mach_vm.h>
+# define XM_DUAL_APPLE 1
+#else
+# define XM_DUAL_APPLE 0
+#endif
+#if defined(__linux__)
+# include <sys/syscall.h>
+# define XM_DUAL_LINUX 1
+#else
+# define XM_DUAL_LINUX 0
+#endif
static int xm_to_posix(int p)
{
int q = 0;
@@ -63,15 +79,82 @@ static int xm_to_posix(int p)
if (p & CFREE_PROT_EXEC) q |= PROT_EXEC;
return q;
}
-static void* xm_reserve(void* u, size_t n, int p)
+typedef struct XmTok { void* w; void* r; size_t n; } XmTok;
+static int xm_reserve_single(size_t n, CfreeExecMemRegion* out)
+{
+ void* p = mmap(NULL, n, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANON, -1, 0);
+ if (p == MAP_FAILED) return 1;
+ out->write = out->runtime = p; out->size = n; out->token = NULL;
+ return 0;
+}
+static int xm_reserve(void* u, size_t n, int p, CfreeExecMemRegion* out)
{
(void)u;
- void* a = mmap(NULL, n, xm_to_posix(p), MAP_PRIVATE | MAP_ANON, -1, 0);
- return a == MAP_FAILED ? NULL : a;
+ if (!out || !n) return 1;
+ if (!(p & CFREE_PROT_EXEC)) return xm_reserve_single(n, out);
+#if XM_DUAL_APPLE
+ {
+ void* w = mmap(NULL, n, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANON, -1, 0);
+ mach_vm_address_t r = 0;
+ vm_prot_t cur = 0, max = 0;
+ XmTok* tok;
+ if (w == MAP_FAILED) return 1;
+ if (mach_vm_remap(mach_task_self(), &r, (mach_vm_size_t)n, 0,
+ VM_FLAGS_ANYWHERE,
+ mach_task_self(), (mach_vm_address_t)(uintptr_t)w,
+ FALSE, &cur, &max, VM_INHERIT_NONE)
+ != KERN_SUCCESS) { munmap(w, n); return 1; }
+ if (mprotect((void*)(uintptr_t)r, n, PROT_READ) != 0) {
+ munmap((void*)(uintptr_t)r, n); munmap(w, n); return 1;
+ }
+ tok = (XmTok*)malloc(sizeof(*tok));
+ if (!tok) { munmap((void*)(uintptr_t)r, n); munmap(w, n); return 1; }
+ tok->w = w; tok->r = (void*)(uintptr_t)r; tok->n = n;
+ out->write = w; out->runtime = (void*)(uintptr_t)r;
+ out->size = n; out->token = tok;
+ return 0;
+ }
+#elif XM_DUAL_LINUX
+ {
+ int fd = (int)syscall(SYS_memfd_create, "cfree-jit-test", 0u);
+ void *w, *r; XmTok* tok;
+ if (fd < 0) return 1;
+ if (ftruncate(fd, (off_t)n) != 0) { close(fd); return 1; }
+ w = mmap(NULL, n, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (w == MAP_FAILED) { close(fd); return 1; }
+ r = mmap(NULL, n, PROT_READ, MAP_SHARED, fd, 0);
+ close(fd);
+ if (r == MAP_FAILED) { munmap(w, n); return 1; }
+ tok = (XmTok*)malloc(sizeof(*tok));
+ if (!tok) { munmap(r, n); munmap(w, n); return 1; }
+ tok->w = w; tok->r = r; tok->n = n;
+ out->write = w; out->runtime = r; out->size = n; out->token = tok;
+ return 0;
+ }
+#else
+ return xm_reserve_single(n, out);
+#endif
}
static int xm_protect(void* u, void* a, size_t n, int p)
{ (void)u; return mprotect(a, n, xm_to_posix(p)); }
-static void xm_release(void* u, void* a, size_t n) { (void)u; munmap(a, n); }
+static void xm_release(void* u, CfreeExecMemRegion* region)
+{
+ (void)u;
+ if (!region || !region->size) return;
+ if (region->token) {
+ XmTok* tok = (XmTok*)region->token;
+ if (tok->r && tok->r != tok->w) munmap(tok->r, tok->n);
+ if (tok->w) munmap(tok->w, tok->n);
+ free(tok);
+ } else if (region->write) {
+ munmap(region->write, region->size);
+ }
+ region->write = region->runtime = NULL;
+ region->size = 0;
+ region->token = NULL;
+}
static void xm_flush(void* u, void* a, size_t n)
{
(void)u;
diff --git a/test/link/harness/jit_runner.c b/test/link/harness/jit_runner.c
@@ -45,6 +45,21 @@ static void diag_fn(CfreeDiagSink* s, CfreeDiagKind k, CfreeSrcLoc loc,
}
static CfreeDiagSink g_diag = { diag_fn, NULL, 0, 0 };
+/* Mirrors driver/env.c — see that file for the strict-W^X dual-mapping
+ * rationale. */
+#if defined(__APPLE__)
+# include <mach/mach.h>
+# include <mach/mach_vm.h>
+# define XM_DUAL_APPLE 1
+#else
+# define XM_DUAL_APPLE 0
+#endif
+#if defined(__linux__)
+# include <sys/syscall.h>
+# define XM_DUAL_LINUX 1
+#else
+# define XM_DUAL_LINUX 0
+#endif
static int xm_to_posix(int p)
{
int q = 0;
@@ -53,15 +68,82 @@ static int xm_to_posix(int p)
if (p & CFREE_PROT_EXEC) q |= PROT_EXEC;
return q;
}
-static void* xm_reserve(void* u, size_t n, int p)
+typedef struct XmTok { void* w; void* r; size_t n; } XmTok;
+static int xm_reserve_single(size_t n, CfreeExecMemRegion* out)
+{
+ void* p = mmap(NULL, n, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANON, -1, 0);
+ if (p == MAP_FAILED) return 1;
+ out->write = out->runtime = p; out->size = n; out->token = NULL;
+ return 0;
+}
+static int xm_reserve(void* u, size_t n, int p, CfreeExecMemRegion* out)
{
(void)u;
- void* a = mmap(NULL, n, xm_to_posix(p), MAP_PRIVATE | MAP_ANON, -1, 0);
- return a == MAP_FAILED ? NULL : a;
+ if (!out || !n) return 1;
+ if (!(p & CFREE_PROT_EXEC)) return xm_reserve_single(n, out);
+#if XM_DUAL_APPLE
+ {
+ void* w = mmap(NULL, n, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANON, -1, 0);
+ mach_vm_address_t r = 0;
+ vm_prot_t cur = 0, max = 0;
+ XmTok* tok;
+ if (w == MAP_FAILED) return 1;
+ if (mach_vm_remap(mach_task_self(), &r, (mach_vm_size_t)n, 0,
+ VM_FLAGS_ANYWHERE,
+ mach_task_self(), (mach_vm_address_t)(uintptr_t)w,
+ FALSE, &cur, &max, VM_INHERIT_NONE)
+ != KERN_SUCCESS) { munmap(w, n); return 1; }
+ if (mprotect((void*)(uintptr_t)r, n, PROT_READ) != 0) {
+ munmap((void*)(uintptr_t)r, n); munmap(w, n); return 1;
+ }
+ tok = (XmTok*)malloc(sizeof(*tok));
+ if (!tok) { munmap((void*)(uintptr_t)r, n); munmap(w, n); return 1; }
+ tok->w = w; tok->r = (void*)(uintptr_t)r; tok->n = n;
+ out->write = w; out->runtime = (void*)(uintptr_t)r;
+ out->size = n; out->token = tok;
+ return 0;
+ }
+#elif XM_DUAL_LINUX
+ {
+ int fd = (int)syscall(SYS_memfd_create, "cfree-jit-test", 0u);
+ void *w, *r; XmTok* tok;
+ if (fd < 0) return 1;
+ if (ftruncate(fd, (off_t)n) != 0) { close(fd); return 1; }
+ w = mmap(NULL, n, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (w == MAP_FAILED) { close(fd); return 1; }
+ r = mmap(NULL, n, PROT_READ, MAP_SHARED, fd, 0);
+ close(fd);
+ if (r == MAP_FAILED) { munmap(w, n); return 1; }
+ tok = (XmTok*)malloc(sizeof(*tok));
+ if (!tok) { munmap(r, n); munmap(w, n); return 1; }
+ tok->w = w; tok->r = r; tok->n = n;
+ out->write = w; out->runtime = r; out->size = n; out->token = tok;
+ return 0;
+ }
+#else
+ return xm_reserve_single(n, out);
+#endif
}
static int xm_protect(void* u, void* a, size_t n, int p)
{ (void)u; return mprotect(a, n, xm_to_posix(p)); }
-static void xm_release(void* u, void* a, size_t n) { (void)u; munmap(a, n); }
+static void xm_release(void* u, CfreeExecMemRegion* region)
+{
+ (void)u;
+ if (!region || !region->size) return;
+ if (region->token) {
+ XmTok* tok = (XmTok*)region->token;
+ if (tok->r && tok->r != tok->w) munmap(tok->r, tok->n);
+ if (tok->w) munmap(tok->w, tok->n);
+ free(tok);
+ } else if (region->write) {
+ munmap(region->write, region->size);
+ }
+ region->write = region->runtime = NULL;
+ region->size = 0;
+ region->token = NULL;
+}
static void xm_flush(void* u, void* a, size_t n)
{
(void)u;
