commit 537cb4cc23c3e08565bb8bd04a6bf6ca9d04c399
parent 064a82b6a17001f25f47a4e05ed1df0cf653d60c
Author: Ryan Sepassi <rsepassi@gmail.com>
Date: Sat, 13 Jun 2026 09:31:02 -0700
perf(mc): typed-store byte sink — mc_emit32 cursor, kill per-insn libc memcpy
aa_emit32/rv64_emit32 staged each instruction word into u8[4] then called the
out-of-line mc_emit_bytes -> buf_write -> memcpy(n=4); the call boundary blocked
const-propagating n=4 so it lowered to a libc memcpy call (~6.94M Ir / 434K
calls, the dominant memcpy in the compile).
Add mc_emit32(MCEmitter*, u32): a typed-store fast path that writes the word
straight through a {cur,cur_end} cursor cached on the MCEmitter base (into the
active section's tail chunk) with one const-width 4-byte store, then bumps
cur_chunk->used + cur_bytes->total inline so the Buf stays coherent at all
times. No staging buffer, no out-of-line memcpy call. The coherent-always
invariant (cur == cur_chunk->data + cur_chunk->used; used/total kept live)
means every random-access reader (obj_pos/buf_patch/mc_pos, plus arch-direct
obj_pos/obj_patch) sees the live frontier with no explicit sync. The cursor is
(re)pointed via mc_cursor_repoint at set_section and after any non-cursor write
(mc_emit_bytes / mc_emit_fill). aarch64 + riscv64 leaf emitters call it
directly; x86-64 (variable-width) keeps mc_emit_bytes unchanged.
Byte-identical: perf identity gate 61/61 PASS (objects -O0/-O1, -g DWARF, -S
asm, linked exe) + cross-arch rv64/x64/aa64 object/-g/-S byte-identity diff.
Diffstat:
4 files changed, 94 insertions(+), 10 deletions(-)
diff --git a/src/arch/aa64/native.c b/src/arch/aa64/native.c
@@ -327,16 +327,15 @@ extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
u32 end_ofs);
static void aa_emit32(MCEmitter* mc, u32 word) {
- u8 b[4];
- wr_u32_le(b, word);
/* obj_pos is a section lookup + buf_pos; only the -g line table needs it, so
- * skip it on the common no-debug compile (one fewer lookup per instruction). */
+ * skip it on the common no-debug compile (one fewer lookup per instruction).
+ * The pre-write offset must be read BEFORE mc_emit32 advances the cursor. */
if (mc->debug) {
u32 ofs = obj_pos(mc->obj, mc->section_id);
- mc_emit_bytes(mc, b, sizeof b);
+ mc_emit32(mc, word);
debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
} else {
- mc_emit_bytes(mc, b, sizeof b);
+ mc_emit32(mc, word);
}
}
diff --git a/src/arch/mc.c b/src/arch/mc.c
@@ -30,6 +30,7 @@
#include "arch/arch.h"
#include "core/arena.h"
#include "core/buf.h"
+#include "core/bytes.h"
#include "core/heap.h"
#include "core/pool.h"
#include "core/strbuf.h"
@@ -234,6 +235,28 @@ ObjSymId mc_label_symbol(MCEmitter* m, MCLabel id) {
/* ---- emission ops (called directly by the arch backends) ---- */
+/* (Re)point the typed-store cursor at the active section's tail chunk write
+ * frontier. Called after set_section and after any non-cursor write path
+ * (mc_emit_bytes / fills) that may have grown or replaced the tail chunk. The
+ * cursor is left NULL/empty (forcing the slow path) when there is no PROGBITS
+ * buffer or the tail chunk is full or absent — buf_write_slow then mints/grows
+ * the chunk and the next repoint picks it up. The invariant maintained
+ * everywhere: cur == cur_chunk->data + cur_chunk->used (when cur_chunk != NULL),
+ * with cur_chunk->used / cur_bytes->total kept coherent by mc_emit32. */
+static void mc_cursor_repoint(MCEmitter* m) {
+ Buf* b = m->cur_bytes;
+ BufChunk* t = b ? b->tail : NULL;
+ if (t) {
+ m->cur_chunk = t;
+ m->cur = t->data + t->used;
+ m->cur_end = t->data + t->cap;
+ } else {
+ m->cur_chunk = NULL;
+ m->cur = NULL;
+ m->cur_end = NULL;
+ }
+}
+
void mc_emit_bytes(MCEmitter* m, const u8* data, size_t n) {
/* Fast path: append straight to the cached section buffer (inlined
* buf_write). cur_bytes is NULL for NOBITS/.bss/none, where obj_write does
@@ -242,6 +265,34 @@ void mc_emit_bytes(MCEmitter* m, const u8* data, size_t n) {
buf_write(m->cur_bytes, data, n);
else
obj_write(m->obj, m->section_id, data, n);
+ /* buf_write may have grown/replaced the tail chunk and bumped used/total;
+ * re-point the typed-store cursor so the next mc_emit32 stays coherent. */
+ mc_cursor_repoint(m);
+}
+
+/* Out-of-line slow path for mc_emit32: the tail chunk is full, absent, or the
+ * section is NOBITS. Route through the normal byte sink (which keeps used/total
+ * coherent and re-points the cursor), so the fast path can branch on a simple
+ * pointer-bounds test. */
+static void mc_emit32_slow(MCEmitter* m, u32 word) {
+ u8 b[4];
+ wr_u32_le(b, word);
+ mc_emit_bytes(m, b, sizeof b);
+}
+
+void mc_emit32(MCEmitter* m, u32 word) {
+ /* Typed-store fast path: one const-width 4-byte store through the cursor,
+ * then bump the chunk/buffer lengths to keep the Buf coherent. No staging
+ * buffer, no out-of-line memcpy call. */
+ u8* cur = m->cur;
+ if (cur && cur + 4 <= m->cur_end) {
+ wr_u32_le(cur, word);
+ m->cur = cur + 4;
+ m->cur_chunk->used += 4u;
+ m->cur_bytes->total += 4u;
+ } else {
+ mc_emit32_slow(m, word);
+ }
}
u32 mc_pos(MCEmitter* m) { return obj_pos(m->obj, m->section_id); }
@@ -254,6 +305,7 @@ void mc_set_section(MCEmitter* m, u32 section_id) {
* per-instruction Sections_at deref + nobits branch. NULL for NOBITS/.bss
* (or none): emit then falls back to obj_write for bss_size accounting. */
m->cur_bytes = obj_section_bytes(m->obj, section_id);
+ mc_cursor_repoint(m);
}
MCLabel mc_label_new(MCEmitter* m) {
@@ -315,6 +367,9 @@ void mc_emit_fill(MCEmitter* m, size_t n, u8 byte) {
obj_write(m->obj, m->section_id, buf, k);
n -= k;
}
+ /* obj_write grew the tail directly (bypassing the typed-store cursor);
+ * re-point so a following mc_emit32 stays coherent. */
+ mc_cursor_repoint(m);
}
void mc_emit_align(MCEmitter* m, u32 align, u8 fill) {
diff --git a/src/arch/mc.h b/src/arch/mc.h
@@ -59,6 +59,23 @@ struct MCEmitter {
* re-points it. */
Buf* cur_bytes;
+ /* Typed-store fast-path cursor into the active section's tail chunk. The
+ * per-instruction emit_word path (mc_emit32) writes a fixed-width word
+ * straight through `cur` and bumps `cur_chunk->used` + `cur_bytes->total`
+ * inline, so the Buf stays coherent at all times — no deferred sync, no
+ * libc memcpy call. `cur` always equals `cur_chunk->data + cur_chunk->used`
+ * and `cur_end` is `cur_chunk->data + cur_chunk->cap`; the cursor is
+ * (re)pointed from cur_bytes->tail by mc_cursor_repoint() at set_section and
+ * after any non-cursor write (mc_emit_bytes / fills) that may have grown or
+ * replaced the tail chunk. All three are NULL/empty when the active section
+ * is NOBITS/.bss (or none): emit then falls to obj_write via mc_emit_bytes.
+ * Because used/total stay coherent, every random-access reader (obj_pos,
+ * buf_patch, mc_pos, arch-direct obj_pos/obj_patch) sees the live frontier
+ * without any explicit sync. */
+ BufChunk* cur_chunk;
+ u8* cur;
+ u8* cur_end;
+
/* Pending source location, updated by set_loc. Promoted to the base so
* arch backends' emit-bytes choke point can read it without reaching
* into the per-arch impl (used to feed debug_emit_row). */
@@ -95,6 +112,15 @@ void mc_free(MCEmitter*);
/* Append machine-code bytes to the active section (the hot emit path). */
void mc_emit_bytes(MCEmitter*, const u8* data, size_t n);
+/* Append one little-endian 4-byte instruction word to the active section.
+ * Typed-store fast path: const-width store straight through the section
+ * cursor (no u8[4] staging, no out-of-line memcpy call). Fixed-width-insn
+ * arches (aarch64, riscv64) call this from their leaf emitter instead of
+ * staging a word into u8[4] + mc_emit_bytes. Variable-width arches (x86-64)
+ * keep using mc_emit_bytes. Keeps the Buf coherent, so any random-access
+ * reader still sees the live frontier. Falls to the buf_write path when the
+ * tail chunk can't hold the word or the section is NOBITS. */
+void mc_emit32(MCEmitter*, u32 word);
/* Current byte offset within the active section. */
u32 mc_pos(MCEmitter*);
/* Stamp the pending source location read by the per-arch emit choke point. */
diff --git a/src/arch/riscv/native.c b/src/arch/riscv/native.c
@@ -79,11 +79,15 @@ extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
/* ============================ low-level emit ============================ */
void rv64_emit32(MCEmitter* mc, u32 word) {
- u8 b[4];
- u32 ofs = obj_pos(mc->obj, mc->section_id);
- wr_u32_le(b, word);
- mc_emit_bytes(mc, b, sizeof b);
- if (mc->debug) debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
+ /* Read the pre-write offset (only the -g line table needs it) before
+ * mc_emit32 advances the cursor. */
+ if (mc->debug) {
+ u32 ofs = obj_pos(mc->obj, mc->section_id);
+ mc_emit32(mc, word);
+ debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
+ } else {
+ mc_emit32(mc, word);
+ }
}
void rv64_emit16(MCEmitter* mc, u32 halfword) {
