kit

pass_jump.c (24895B)

---

 #include <kit/cg.h>
 #include <string.h>
 
 #include "cg/ir_eval.h"
 #include "opt/opt_internal.h"
 
 #define BLOCK_NONE ((u32)~0u)
 
 typedef struct JumpCleanupCtx {
   Func* f;
   u32* emit_index_by_block;
   u32* forwarded_target_by_block;
   u32* forward_path;
   u8* has_label_addr_ref;
 } JumpCleanupCtx;
 
 static u32 emit_order_index(const JumpCleanupCtx* c, u32 block) {
   if (block >= c->f->nblocks) return BLOCK_NONE;
   return c->emit_index_by_block[block];
 }
 
 static u32 next_emit_block(const JumpCleanupCtx* c, u32 block) {
   u32 idx = emit_order_index(c, block);
   if (idx == BLOCK_NONE || idx + 1u >= c->f->emit_order_n) return BLOCK_NONE;
   return c->f->emit_order[idx + 1u];
 }
 
 static void refresh_emit_index_range(JumpCleanupCtx* c, u32 first, u32 last) {
   Func* f = c->f;
   if (!f || first >= f->emit_order_n) return;
   if (last >= f->emit_order_n) last = f->emit_order_n - 1u;
   for (u32 i = first; i <= last; ++i) {
     u32 b = f->emit_order[i];
     if (b < f->nblocks) c->emit_index_by_block[b] = i;
   }
 }
 
 static void mark_label_addr_refs(Func* f, u8* refs) {
   if (!f || !refs) return;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     for (u32 i = 0; i < bl->ninsts; ++i) {
       Inst* in = &bl->insts[i];
       switch ((IROp)in->op) {
         case IR_LOAD_LABEL_ADDR:
           if ((u32)in->extra.imm < f->nblocks) refs[(u32)in->extra.imm] = 1;
           break;
         case IR_LOCAL_STATIC_DATA_LABEL_ADDR: {
           CgIrLocalStaticLabelAux* aux =
               (CgIrLocalStaticLabelAux*)in->extra.aux;
           if (aux && (u32)aux->target < f->nblocks) refs[(u32)aux->target] = 1;
           break;
         }
         default:
           break;
       }
     }
   }
 }
 
 static int block_has_label_addr_ref(const JumpCleanupCtx* c, u32 block) {
   if (!c || block >= c->f->nblocks) return 0;
   return c->has_label_addr_ref && c->has_label_addr_ref[block];
 }
 
 static JumpCleanupCtx jump_cleanup_ctx(Func* f) {
   JumpCleanupCtx c;
   c.f = f;
   c.emit_index_by_block =
       arena_array(f->arena, u32, f->nblocks ? f->nblocks : 1u);
   c.forwarded_target_by_block =
       arena_array(f->arena, u32, f->nblocks ? f->nblocks : 1u);
   c.forward_path = arena_array(f->arena, u32, f->nblocks ? f->nblocks : 1u);
   c.has_label_addr_ref =
       arena_zarray(f->arena, u8, f->nblocks ? f->nblocks : 1u);
   for (u32 b = 0; b < f->nblocks; ++b) c.emit_index_by_block[b] = BLOCK_NONE;
   for (u32 b = 0; b < f->nblocks; ++b)
     c.forwarded_target_by_block[b] = BLOCK_NONE;
   for (u32 i = 0; i < f->emit_order_n; ++i) {
     u32 b = f->emit_order[i];
     if (b < f->nblocks) c.emit_index_by_block[b] = i;
   }
   mark_label_addr_refs(f, c.has_label_addr_ref);
   return c;
 }
 
 static int single_jump_block(const JumpCleanupCtx* c, u32 block,
                              u32* target_out) {
   Func* f = c->f;
   if (block >= f->nblocks) return 0;
   Block* bl = &f->blocks[block];
   if (block_has_label_addr_ref(c, block)) return 0;
   if (bl->ninsts != 1 || bl->nsucc != 1) return 0;
   if ((IROp)bl->insts[0].op != IR_BR) return 0;
   if (target_out) *target_out = bl->succ[0];
   return 1;
 }
 
 static int empty_fallthrough_block(const JumpCleanupCtx* c, u32 block,
                                    u32* target_out) {
   Func* f = c->f;
   if (block >= f->nblocks || block == f->entry) return 0;
   Block* bl = &f->blocks[block];
   if (block_has_label_addr_ref(c, block)) return 0;
   if (bl->ninsts != 0 || bl->nsucc != 0) return 0;
   u32 idx = emit_order_index(c, block);
   if (idx == BLOCK_NONE || idx + 1u >= f->emit_order_n) return 0;
   u32 next = f->emit_order[idx + 1u];
   if (next >= f->nblocks || next == block) return 0;
   if (target_out) *target_out = next;
   return 1;
 }
 
 static int passthrough_succ_block(const JumpCleanupCtx* c, u32 block,
                                   u32* target_out) {
   Func* f = c->f;
   if (block >= f->nblocks || block == f->entry) return 0;
   Block* bl = &f->blocks[block];
   if (block_has_label_addr_ref(c, block)) return 0;
   if (bl->nsucc != 1) return 0;
   if (bl->succ[0] == block) return 0;
   if (bl->ninsts != 0) {
     if (bl->ninsts != 1) return 0;
     switch ((IROp)bl->insts[0].op) {
       case IR_NOP:
       case IR_SCOPE_BEGIN:
       case IR_SCOPE_END:
         break;
       default:
         return 0;
     }
   }
   if (target_out) *target_out = bl->succ[0];
   return 1;
 }
 
 static u32 forward_jump_target_ex(JumpCleanupCtx* c, u32 target,
                                   int allow_empty_fallthrough) {
   u32 cur = target;
   Func* f = c->f;
   u32 npath = 0;
   u32 result = target;
   for (u32 step = 0; step < f->nblocks; ++step) {
     u32 next = BLOCK_NONE;
     if (cur >= f->nblocks) {
       result = cur;
       break;
     }
     if (c->forwarded_target_by_block[cur] != BLOCK_NONE) {
       result = c->forwarded_target_by_block[cur];
       break;
     }
     if (!single_jump_block(c, cur, &next) &&
         (!allow_empty_fallthrough ||
          (!passthrough_succ_block(c, cur, &next) &&
           !empty_fallthrough_block(c, cur, &next)))) {
       result = cur;
       c->forwarded_target_by_block[cur] = result;
       break;
     }
     c->forward_path[npath++] = cur;
     if (next == cur) {
       result = target;
       break;
     }
     cur = next;
     if (step + 1u == f->nblocks) result = target;
   }
   for (u32 i = 0; i < npath; ++i)
     c->forwarded_target_by_block[c->forward_path[i]] = result;
   return result;
 }
 
 static u32 forward_jump_target(JumpCleanupCtx* c, u32 target) {
   return forward_jump_target_ex(c, target, 0);
 }
 
 static int invert_cmp(CmpOp op, CmpOp* out) {
   switch (op) {
     case CMP_EQ:
       *out = CMP_NE;
       return 1;
     case CMP_NE:
       *out = CMP_EQ;
       return 1;
     case CMP_LT_S:
       *out = CMP_GE_S;
       return 1;
     case CMP_LE_S:
       *out = CMP_GT_S;
       return 1;
     case CMP_GT_S:
       *out = CMP_LE_S;
       return 1;
     case CMP_GE_S:
       *out = CMP_LT_S;
       return 1;
     case CMP_LT_U:
       *out = CMP_GE_U;
       return 1;
     case CMP_LE_U:
       *out = CMP_GT_U;
       return 1;
     case CMP_GT_U:
       *out = CMP_LE_U;
       return 1;
     case CMP_GE_U:
       *out = CMP_LT_U;
       return 1;
     /* FP: negation flips ordered<->unordered (the NaN outcome flips too) as
      * well as negating the relation. Kept byte-for-byte in sync with
      * api_invert_cmp (src/cg/fold.c); see that function for the rationale. */
     case CMP_OEQ_F:
       *out = CMP_UNE_F;
       return 1;
     case CMP_ONE_F:
       *out = CMP_UEQ_F;
       return 1;
     case CMP_OLT_F:
       *out = CMP_UGE_F;
       return 1;
     case CMP_OLE_F:
       *out = CMP_UGT_F;
       return 1;
     case CMP_OGT_F:
       *out = CMP_ULE_F;
       return 1;
     case CMP_OGE_F:
       *out = CMP_ULT_F;
       return 1;
     case CMP_UEQ_F:
       *out = CMP_ONE_F;
       return 1;
     case CMP_UNE_F:
       *out = CMP_OEQ_F;
       return 1;
     case CMP_ULT_F:
       *out = CMP_OGE_F;
       return 1;
     case CMP_ULE_F:
       *out = CMP_OGT_F;
       return 1;
     case CMP_UGT_F:
       *out = CMP_OLE_F;
       return 1;
     case CMP_UGE_F:
       *out = CMP_OLT_F;
       return 1;
     default:
       return 0;
   }
 }
 
 static int block_has_only_pred(const Func* f, u32 block, u32 pred) {
   if (block >= f->nblocks) return 0;
   const Block* bl = &f->blocks[block];
   return bl->npreds == 1 && bl->preds && bl->preds[0] == pred;
 }
 
 static void cleanup_branch_targets(JumpCleanupCtx* c) {
   Func* f = c->f;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     u32 nsucc = 0;
     if (!bl->ninsts) continue;
     IROp op = (IROp)bl->insts[bl->ninsts - 1u].op;
     switch (op) {
       case IR_BR:
         nsucc = bl->nsucc;
         break;
       case IR_CMP_BRANCH:
       case IR_CONDBR:
         nsucc = bl->nsucc ? 1u : 0u;
         break;
       default:
         continue;
     }
     for (u32 s = 0; s < nsucc; ++s) {
       u32 target = bl->succ[s];
       u32 forwarded = forward_jump_target_ex(c, target, 1);
       if (forwarded < f->nblocks) bl->succ[s] = forwarded;
     }
   }
 }
 
 static void cleanup_invert_jump_fallthrough(JumpCleanupCtx* c) {
   Func* f = c->f;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     if (!bl->ninsts || bl->nsucc != 2) continue;
     Inst* last = &bl->insts[bl->ninsts - 1u];
     if ((IROp)last->op != IR_CMP_BRANCH) continue;
 
     u32 taken = bl->succ[0];
     u32 fallthrough = bl->succ[1];
     u32 next = next_emit_block(c, b);
     u32 fallthrough_idx = emit_order_index(c, fallthrough);
     u32 jump_target = BLOCK_NONE;
     CmpOp inverted;
     if (next != fallthrough) continue;
     if (fallthrough_idx == BLOCK_NONE ||
         fallthrough_idx + 1u >= f->emit_order_n)
       continue;
     if (f->emit_order[fallthrough_idx + 1u] != taken) continue;
     if (!block_has_only_pred(f, fallthrough, b)) continue;
     if (!single_jump_block(c, fallthrough, &jump_target)) continue;
     jump_target = forward_jump_target(c, jump_target);
     if (jump_target >= f->nblocks) continue;
     if (!invert_cmp((CmpOp)last->extra.imm, &inverted)) continue;
 
     last->extra.imm = inverted;
     bl->succ[0] = jump_target;
     bl->succ[1] = taken;
   }
 }
 
 static void cleanup_invert_taken_fallthrough(JumpCleanupCtx* c) {
   Func* f = c->f;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     if (!bl->ninsts || bl->nsucc != 2) continue;
     Inst* last = &bl->insts[bl->ninsts - 1u];
     if ((IROp)last->op != IR_CMP_BRANCH) continue;
 
     u32 taken = bl->succ[0];
     u32 fallthrough = bl->succ[1];
     if (next_emit_block(c, b) != taken) continue;
     CmpOp inverted;
     if (!invert_cmp((CmpOp)last->extra.imm, &inverted)) continue;
 
     last->extra.imm = inverted;
     bl->succ[0] = fallthrough;
     bl->succ[1] = taken;
   }
 }
 
 /* Greedy chain-extension reorder: pick the entry block, then repeatedly
  * extend the current chain to its preferred unvisited successor. For
  * conditional branches that's `succ[1]` (fallthrough); for IR_BR or any
  * 1-succ block it's `succ[0]`. When the chain stalls, start a new one from
  * the lowest unvisited block id. After this pass, every chained pair `(p,
  * n)` is adjacent in emit_order, so the existing branch-invert and
  * trailing-branch-strip cleanups can collapse the jump.
  *
  * kit's frontend loop lowering puts `inc` between `head` and `body`
  * (because `continue` jumps to `inc`), forcing `b body` and `b inc` per
  * iteration. Without reordering the trailing-branch strip can't fire —
  * `body`'s next block in original order is `exit`, not `inc`. After
  * reordering, body→inc is adjacent and the back-edge collapses. */
 static u32 chain_preferred_succ(const Block* bl) {
   if (!bl->ninsts) {
     return bl->nsucc ? bl->succ[0] : BLOCK_NONE;
   }
   IROp op = (IROp)bl->insts[bl->ninsts - 1u].op;
   switch (op) {
     case IR_CMP_BRANCH:
     case IR_CONDBR:
       return bl->nsucc >= 2u ? bl->succ[1] : BLOCK_NONE;
     case IR_BR:
       return bl->nsucc >= 1u ? bl->succ[0] : BLOCK_NONE;
     default:
       /* RET/SWITCH/INDIRECT_BRANCH: no implicit fallthrough preference. */
       return BLOCK_NONE;
   }
 }
 
 static void cleanup_reorder_for_fallthrough(JumpCleanupCtx* c) {
   Func* f = c->f;
   if (f->nblocks < 2u || f->emit_order_n < 2u) return;
   u8* visited = arena_zarray(f->arena, u8, f->nblocks);
   u32* new_order = arena_array(f->arena, u32, f->emit_order_n);
   u32 w = 0;
 
   /* Entry must come first regardless. */
   u32 entry = f->entry;
   if (entry >= f->nblocks) return;
   new_order[w++] = entry;
   visited[entry] = 1;
   u32 cur = entry;
 
   /* Extend by preferred successor. When the chain stalls, restart from the
    * next unvisited block in original emit_order to preserve some locality. */
   u32 scan_cursor = 0;
   for (;;) {
     u32 next = chain_preferred_succ(&f->blocks[cur]);
     if (next >= f->nblocks || visited[next]) {
       /* Fall back: any unvisited successor of cur — extends the chain even
        * if it's the "taken" arm of a conditional. The subsequent
        * cleanup_invert_taken_fallthrough will invert the branch. */
       const Block* cb = &f->blocks[cur];
       next = BLOCK_NONE;
       for (u32 s = 0; s < cb->nsucc; ++s) {
         u32 cand = cb->succ[s];
         if (cand < f->nblocks && !visited[cand]) {
           next = cand;
           break;
         }
       }
     }
     if (next >= f->nblocks || visited[next]) {
       /* Start a new chain from the lowest unvisited block in original order. */
       next = BLOCK_NONE;
       while (scan_cursor < f->emit_order_n) {
         u32 cand = f->emit_order[scan_cursor++];
         if (cand < f->nblocks && !visited[cand]) {
           next = cand;
           break;
         }
       }
       if (next >= f->nblocks) break;
     }
     new_order[w++] = next;
     visited[next] = 1;
     cur = next;
   }
 
   if (w != f->emit_order_n) return; /* Conservative: bail if we missed any. */
   memcpy(f->emit_order, new_order, sizeof(u32) * w);
   /* Refresh the cached emit-index map so subsequent cleanups see the new
    * order. */
   for (u32 b = 0; b < f->nblocks; ++b) c->emit_index_by_block[b] = BLOCK_NONE;
   for (u32 i = 0; i < f->emit_order_n; ++i) {
     u32 b = f->emit_order[i];
     if (b < f->nblocks) c->emit_index_by_block[b] = i;
   }
 }
 
 /* Rotate simple counted loops so the test sits at the bottom: the back-edge
  * becomes the per-iteration conditional branch and the body falls through to
  * the test, eliminating the unconditional back-jump the head-test shape emits
  * every iteration.
  *
  * Pattern (after the greedy reorder):
  *
  *   H (emit i):  IR_CMP_BRANCH, succ[0]=E (exit, taken), succ[1]=B (body entry,
  *               the fallthrough, placed right after H by the greedy reorder)
  *   ... body chain ...
  *   L (emit k):  the latch — a predecessor of H at a later emit position, i.e.
  *               the source of the loop's back-edge to H
  *
  * The rewrite moves H from position i to just after L (rotating the emit-order
  * run [i..k] left by one) and inverts H's test in place: H's taken edge becomes
  * the back-edge to the body entry (the per-iteration conditional) and its
  * fallthrough becomes the exit E. No CFG edges move — reordering emit_order is
  * always valid because pass_native_emit emits an explicit jump for any
  * successor that isn't the textual next block. After the move L falls through
  * to H (its back-edge `b H` is stripped), and the preheader keeps its now-
  * forward `b H` (executed once, serving as the zero-trip guard); the
  * unconditional back-jump every iteration is gone.
  *
  * Conservative guards: the body entry must be H's fallthrough and immediately
  * follow H (the greedy chain), and H must have exactly one back-edge
  * predecessor after it in emit order (reducible, single-latch loop). */
 static void cleanup_rotate_loops(JumpCleanupCtx* c) {
   Func* f = c->f;
   if (f->emit_order_n < 2u) return;
   for (u32 i = 0; i + 1u < f->emit_order_n; ++i) {
     u32 h = f->emit_order[i];
     if (h >= f->nblocks) continue;
     Block* H = &f->blocks[h];
     if (!H->ninsts || H->nsucc != 2) continue;
     Inst* hterm = &H->insts[H->ninsts - 1u];
     if ((IROp)hterm->op != IR_CMP_BRANCH) continue;
     u32 body = H->succ[1]; /* loop body entry (fallthrough) */
     if (f->emit_order[i + 1u] != body) continue;
     /* Latch: the single predecessor of H later in emit order (back-edge
      * source). Zero or multiple => not a simple single-latch loop; skip. */
     u32 latch_pos = BLOCK_NONE;
     int ok = 1;
     for (u32 p = 0; p < H->npreds; ++p) {
       u32 pos = emit_order_index(c, H->preds[p]);
       if (pos == BLOCK_NONE || pos <= i) continue;
       if (latch_pos != BLOCK_NONE) {
         ok = 0;
         break;
       }
       latch_pos = pos;
     }
     if (!ok || latch_pos == BLOCK_NONE) continue;
     CmpOp inverted;
     if (!invert_cmp((CmpOp)hterm->extra.imm, &inverted)) continue;
     u32 exit = H->succ[0];
     for (u32 k = i; k < latch_pos; ++k)
       f->emit_order[k] = f->emit_order[k + 1u];
     f->emit_order[latch_pos] = h;
     hterm->extra.imm = inverted;
     H->succ[0] = body; /* taken: back-edge to the loop body */
     H->succ[1] = exit; /* fallthrough: loop exit */
     refresh_emit_index_range(c, i, latch_pos);
   }
 }
 
 static void cleanup_layout_fallthrough_branches(const JumpCleanupCtx* c) {
   Func* f = c->f;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     if (!bl->ninsts || bl->nsucc != 1) continue;
     Inst* last = &bl->insts[bl->ninsts - 1u];
     if ((IROp)last->op != IR_BR) continue;
     if (next_emit_block(c, b) != bl->succ[0]) continue;
     memset(last, 0, sizeof *last);
     --bl->ninsts;
   }
 }
 
 static int forward_empty_fallthrough_chain(JumpCleanupCtx* c, u32 target,
                                            u32* target_out) {
   u32 cur = target;
   Func* f = c->f;
   for (u32 step = 0; step < f->nblocks; ++step) {
     u32 next_block = BLOCK_NONE;
     if (!empty_fallthrough_block(c, cur, &next_block)) return 0;
     if (next_block >= f->nblocks) return 0;
     if (!empty_fallthrough_block(c, next_block, NULL)) {
       if (target_out) *target_out = next_block;
       return 1;
     }
     cur = next_block;
   }
   return 0;
 }
 
 static int full_cond_fallthrough_forwardable(JumpCleanupCtx* c, u32 pred,
                                              u32 target, u32* target_out) {
   u32 next = next_emit_block(c, pred);
   if (next != target) return 0;
   return forward_empty_fallthrough_chain(c, target, target_out);
 }
 
 static u32 full_forwardable_successors(const Block* bl, const Inst* term) {
   switch ((IROp)term->op) {
     case IR_BR:
     case IR_SWITCH:
       return bl->nsucc;
     case IR_CONDBR:
     case IR_CMP_BRANCH:
       return bl->nsucc < 2u ? bl->nsucc : 2u;
     default:
       return 0;
   }
 }
 
 static void full_rewrite_successor(Func* f, u32 b, u32 old_succ, u32 new_succ) {
   opt_replace_succ_ref(f, b, old_succ, new_succ);
 }
 
 static int full_forward_branch_targets(JumpCleanupCtx* c) {
   Func* f = c->f;
   int changed = 0;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     if (!bl->ninsts) continue;
     Inst* term = &bl->insts[bl->ninsts - 1u];
     u32 nsucc = full_forwardable_successors(bl, term);
     for (u32 s = 0; s < nsucc; ++s) {
       u32 target = bl->succ[s];
       u32 forwarded = forward_jump_target_ex(c, target, 1);
       if (((IROp)term->op == IR_CONDBR || (IROp)term->op == IR_CMP_BRANCH) &&
           s == 1u) {
         if (!full_cond_fallthrough_forwardable(c, b, target, &forwarded))
           continue;
       }
       if (forwarded >= f->nblocks || forwarded == target) continue;
       full_rewrite_successor(f, b, target, forwarded);
       changed = 1;
     }
   }
   return changed;
 }
 
 static int full_collapse_same_target_branches(Func* f) {
   int changed = 0;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     if (!bl->ninsts || bl->nsucc != 2) continue;
     Inst* term = &bl->insts[bl->ninsts - 1u];
     IROp op = (IROp)term->op;
     if (op != IR_CONDBR && op != IR_CMP_BRANCH) continue;
     if (bl->succ[0] != bl->succ[1]) continue;
     InstId id = term->id;
     SrcLoc loc = term->loc;
     memset(term, 0, sizeof *term);
     term->op = IR_BR;
     term->id = id;
     term->loc = loc;
     bl->nsucc = 1;
     changed = 1;
   }
   return changed;
 }
 
 /* O1.md W10 — constant `cmp_branch` folding (no SSA).
  *
  * Fold an `IR_CMP_BRANCH` whose outcome is statically known by *local* facts
  * only, then rewrite the terminator to a plain `IR_BR` at the selected
  * successor (dropping the now-dead CFG edge). Two cases are decidable here:
  *
  *   1. both operands are immediates (`OPK_IMM`), or
  *   2. an integer same-register identity (`x <cmp> x`).
  *
  * Both are evaluated through the shared `kit_ir_eval_cmp` integer evaluator,
  * which masks/sign-extends to the operand type width and returns 0 for any FP
  * predicate (`op >= CMP_OEQ_F`) — so floating compares (where NaN makes even
  * `x == x` non-trivial) are never folded. The operand type width also gates the
  * fold: if the width is unknown we skip. We deliberately do *not* chase
  * `IR_LOAD_IMM` definitions: at O1 PRegs are mutable, so only direct operands
  * and the same-reg identity are sound without SSA. */
 static u32 cmp_branch_operand_width(Func* f, const Operand* op) {
   if (!op) return 0;
   u32 w = kit_cg_type_int_width((KitCompiler*)f->c, op->type);
   if (w && w <= 64u) return w;
   if (kit_cg_type_kind((KitCompiler*)f->c, op->type) == KIT_CG_TYPE_PTR) {
     u64 size = kit_cg_type_size((KitCompiler*)f->c, op->type);
     if (size && size <= 8u) return (u32)(size * 8u);
   }
   return 0;
 }
 
 /* Returns 1 and writes *taken_out (the i1 compare result) when the branch
  * condition is locally decidable; 0 to leave the branch alone. */
 static int cmp_branch_const_outcome(Func* f, const Inst* term, int* taken_out) {
   if ((IROp)term->op != IR_CMP_BRANCH || term->nopnds < 2) return 0;
   CmpOp op = (CmpOp)term->extra.imm;
   if (op >= CMP_OEQ_F) return 0; /* never fold FP (NaN). */
   const Operand* a = &term->opnds[0];
   const Operand* b = &term->opnds[1];
 
   /* Integer same-register identity: x <cmp> x. The width only needs to be
    * known to be a real integer/pointer scalar; the value cancels out. */
   if (a->kind == OPK_REG && b->kind == OPK_REG && a->v.reg == b->v.reg) {
     u32 w = cmp_branch_operand_width(f, a);
     if (!w) return 0;
     i64 r;
     if (!kit_ir_eval_cmp(op, w, 0, 0, &r)) return 0;
     *taken_out = r != 0;
     return 1;
   }
 
   /* Two immediates. */
   if (a->kind == OPK_IMM && b->kind == OPK_IMM) {
     u32 wa = cmp_branch_operand_width(f, a);
     u32 wb = cmp_branch_operand_width(f, b);
     u32 w = wa ? wa : wb;
     if (!w) return 0;
     i64 r;
     if (!kit_ir_eval_cmp(op, w, a->v.imm, b->v.imm, &r)) return 0;
     *taken_out = r != 0;
     return 1;
   }
   return 0;
 }
 
 static int one_pass_fold_const_cmp_branch(Func* f) {
   int changed = 0;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     if (!bl->ninsts || bl->nsucc != 2) continue;
     Inst* term = &bl->insts[bl->ninsts - 1u];
     int taken;
     if (!cmp_branch_const_outcome(f, term, &taken)) continue;
     /* succ[0] = taken target, succ[1] = fallthrough/false target. */
     u32 keep = taken ? bl->succ[0] : bl->succ[1];
     InstId id = term->id;
     SrcLoc loc = term->loc;
     memset(term, 0, sizeof *term);
     term->op = IR_BR;
     term->id = id;
     term->loc = loc;
     bl->succ[0] = keep;
     bl->nsucc = 1;
     changed = 1;
   }
   return changed;
 }
 
 /* O1.md W9 — one-pass branch cleanup subset (linear).
  *
  * The single-pass, obviously-linear core of `opt_jump_opt` with its
  * fixed-point loop removed (the loop is intentionally not an O1 fit). Runs the
  * W10 constant cmp_branch fold first (it can create same-target / pass-through
  * blocks the W9 passes then clean up), then forwards branch targets through
  * trivial pass-through blocks once and collapses same-target conditional
  * branches once. The forwarding walk uses the memoized
  * `forward_jump_target_ex`, so repeated target queries are amortized linear for
  * the function; the label-address guard (`has_label_addr_ref`) keeps
  * computed-goto-visible blocks from being bypassed. At most one CFG rebuild
  * after the rewrites. */
 void opt_jump_cleanup_o1(Func* f) {
   if (!f) return;
   opt_analysis_invalidate(
       f, OPT_ANALYSIS_DEF_USE | OPT_ANALYSIS_DOM | OPT_ANALYSIS_LOOP);
 
   int changed = one_pass_fold_const_cmp_branch(f);
 
   JumpCleanupCtx c = jump_cleanup_ctx(f);
   changed |= full_forward_branch_targets(&c);
   changed |= full_collapse_same_target_branches(f);
 
   if (changed) {
     opt_analysis_invalidate(
         f, OPT_ANALYSIS_DEF_USE | OPT_ANALYSIS_DOM | OPT_ANALYSIS_LOOP);
     opt_build_cfg(f);
   }
 }
 
 void opt_jump_cleanup(Func* f, OptJumpCleanupStage stage) {
   if (!f) return;
   opt_analysis_invalidate(
       f, OPT_ANALYSIS_DEF_USE | OPT_ANALYSIS_DOM | OPT_ANALYSIS_LOOP);
   JumpCleanupCtx c = jump_cleanup_ctx(f);
   if (stage == OPT_JUMP_CLEANUP_CFG) {
     cleanup_invert_jump_fallthrough(&c);
     cleanup_branch_targets(&c);
   } else if (stage == OPT_JUMP_CLEANUP_LAYOUT) {
     cleanup_reorder_for_fallthrough(&c);
     cleanup_rotate_loops(&c);
     cleanup_invert_taken_fallthrough(&c);
     cleanup_layout_fallthrough_branches(&c);
   }
 }
 
 void opt_jump_opt(Func* f) {
   if (!f) return;
   int changed = 0;
 
   opt_analysis_invalidate(
       f, OPT_ANALYSIS_DEF_USE | OPT_ANALYSIS_DOM | OPT_ANALYSIS_LOOP);
 
   for (u32 iter = 0; iter < f->nblocks; ++iter) {
     JumpCleanupCtx c = jump_cleanup_ctx(f);
     int iter_changed = 0;
     iter_changed |= full_forward_branch_targets(&c);
     opt_build_cfg(f);
     c = jump_cleanup_ctx(f);
     cleanup_invert_jump_fallthrough(&c);
     cleanup_branch_targets(&c);
     iter_changed |= full_collapse_same_target_branches(f);
     if (!iter_changed) break;
     changed = 1;
     opt_build_cfg(f);
   }
 
   if (changed) {
     opt_analysis_invalidate(
         f, OPT_ANALYSIS_DEF_USE | OPT_ANALYSIS_DOM | OPT_ANALYSIS_LOOP);
   }
   opt_build_cfg(f);
 }