kernel.S (10448B)
1 /* seed kernel — arm64 boot protocol entry, vector table, SVC handler, 2 * plus C-callable thunks for ops that can't be expressed in plain C 3 * (sysreg msr/mrs, barriers, cache/TLB ops, PSCI conduits, cpu pause). */ 4 5 .section .head.text, "ax" 6 .globl _head 7 _head: 8 /* arm64 Image header (Documentation/arm64/booting.rst). 9 * code0 must be a valid instruction (a branch, in our case). */ 10 b stext 11 .long 0 12 .quad 0x80000 /* text_offset (preferred load offset within RAM) */ 13 .quad _image_end - _head /* image_size */ 14 .quad 0xa /* flags: 4K pages, anywhere in physmem, LE */ 15 .quad 0 16 .quad 0 17 .quad 0 18 .ascii "ARM\x64" /* magic */ 19 .long 0 /* PE COFF offset (none) */ 20 21 stext: 22 /* Entry contract: x0 = DTB phys, MMU off, caches off, EL2 or EL1. */ 23 msr daifset, #0xf 24 25 /* If we entered at EL2, drop to EL1. Otherwise we're already at EL1. */ 26 mrs x9, CurrentEL 27 lsr x9, x9, #2 28 cmp x9, #2 29 b.ne in_el1 30 31 /* EL2 → EL1: set HCR_EL2.RW=1 (EL1 is AArch64), CNTHCTL/CNTVOFF defaults, 32 * SPSR=EL1h with DAIF masked, ELR=in_el1, eret. */ 33 mov x9, #(1 << 31) 34 msr hcr_el2, x9 35 mov x9, #0x3c5 /* EL1h, DAIF=1111 */ 36 msr spsr_el2, x9 37 adr x9, in_el1 38 msr elr_el2, x9 39 /* Make sure SP_EL1 is set before we eret to EL1 (else we land with 40 * an undefined SP). Use the same kernel stack we're about to install. */ 41 adrp x9, kstack_top 42 add x9, x9, :lo12:kstack_top 43 msr sp_el1, x9 44 eret 45 46 in_el1: 47 /* Stack. */ 48 adrp x9, kstack_top 49 add x9, x9, :lo12:kstack_top 50 mov sp, x9 51 52 /* Vector table. */ 53 adrp x9, vector_table 54 add x9, x9, :lo12:vector_table 55 msr vbar_el1, x9 56 isb 57 58 /* Zero BSS. */ 59 adrp x1, __bss_start 60 add x1, x1, :lo12:__bss_start 61 adrp x2, __bss_end 62 add x2, x2, :lo12:__bss_end 63 1: cmp x1, x2 64 b.ge 2f 65 str xzr, [x1], #8 66 b 1b 67 2: 68 /* Hand control to C. x0 still = DTB phys (not clobbered above). */ 69 bl kmain 70 71 /* kmain shouldn't return. */ 72 hang: 73 wfe 74 b hang 75 76 77 /* ─── Exception vector table ──────────────────────────────────────────── */ 78 79 .macro VENTRY label 80 .balign 0x80 81 b \label 82 .endm 83 84 .section .text, "ax" 85 .balign 0x800 86 .globl vector_table 87 vector_table: 88 /* Current EL with SP_EL0 (we never run kernel like this — only user). */ 89 VENTRY el1_sp0_sync /* 0x000: SVC from EL1t (our "user") */ 90 VENTRY unhandled /* 0x080 */ 91 VENTRY unhandled /* 0x100 */ 92 VENTRY unhandled /* 0x180 */ 93 /* Current EL with SP_ELx (kernel internal). */ 94 VENTRY el1_spx_sync /* 0x200: panic on kernel sync fault */ 95 VENTRY unhandled /* 0x280 */ 96 VENTRY unhandled /* 0x300 */ 97 VENTRY unhandled /* 0x380 */ 98 /* Lower EL using AArch64 (EL0). Unused in this design but wired. */ 99 VENTRY el1_sp0_sync /* 0x400 */ 100 VENTRY unhandled /* 0x480 */ 101 VENTRY unhandled /* 0x500 */ 102 VENTRY unhandled /* 0x580 */ 103 /* Lower EL using AArch32 (unused). */ 104 VENTRY unhandled /* 0x600 */ 105 VENTRY unhandled /* 0x680 */ 106 VENTRY unhandled /* 0x700 */ 107 VENTRY unhandled /* 0x780 */ 108 109 110 /* ─── Trap entry/exit ───────────────────────────────────────────────────── 111 * Save x0..x30 + ELR_EL1 + SPSR_EL1 onto the kernel stack as a trapframe, 112 * call C trap_sync(esr, &tf), restore, eret. The C handler reads/writes 113 * tf->x[0..7] for syscall args and return value, plus tf->x[8] for the 114 * syscall number. 115 */ 116 117 .macro SAVE_TF 118 sub sp, sp, #272 119 stp x0, x1, [sp, #0] 120 stp x2, x3, [sp, #16] 121 stp x4, x5, [sp, #32] 122 stp x6, x7, [sp, #48] 123 stp x8, x9, [sp, #64] 124 stp x10, x11, [sp, #80] 125 stp x12, x13, [sp, #96] 126 stp x14, x15, [sp, #112] 127 stp x16, x17, [sp, #128] 128 stp x18, x19, [sp, #144] 129 stp x20, x21, [sp, #160] 130 stp x22, x23, [sp, #176] 131 stp x24, x25, [sp, #192] 132 stp x26, x27, [sp, #208] 133 stp x28, x29, [sp, #224] 134 str x30, [sp, #240] 135 mrs x10, elr_el1 136 mrs x11, spsr_el1 137 stp x10, x11, [sp, #248] 138 .endm 139 140 .macro RESTORE_TF 141 ldp x10, x11, [sp, #248] 142 msr elr_el1, x10 143 msr spsr_el1, x11 144 ldr x30, [sp, #240] 145 ldp x28, x29, [sp, #224] 146 ldp x26, x27, [sp, #208] 147 ldp x24, x25, [sp, #192] 148 ldp x22, x23, [sp, #176] 149 ldp x20, x21, [sp, #160] 150 ldp x18, x19, [sp, #144] 151 ldp x16, x17, [sp, #128] 152 ldp x14, x15, [sp, #112] 153 ldp x12, x13, [sp, #96] 154 ldp x10, x11, [sp, #80] 155 ldp x8, x9, [sp, #64] 156 ldp x6, x7, [sp, #48] 157 ldp x4, x5, [sp, #32] 158 ldp x2, x3, [sp, #16] 159 ldp x0, x1, [sp, #0] 160 add sp, sp, #272 161 .endm 162 163 el1_sp0_sync: 164 SAVE_TF 165 mrs x0, esr_el1 166 mov x1, sp 167 bl trap_sync 168 RESTORE_TF 169 eret 170 171 el1_spx_sync: 172 /* Same shape as user sync — let C distinguish via SPSR/ESR if needed. */ 173 SAVE_TF 174 mrs x0, esr_el1 175 mov x1, sp 176 bl trap_kernel 177 RESTORE_TF 178 eret 179 180 unhandled: 181 SAVE_TF 182 mrs x0, esr_el1 183 mov x1, sp 184 bl trap_unhandled 185 RESTORE_TF 186 eret 187 188 189 /* ─── eret_to_user(entry, sp) ───────────────────────────────────────────── 190 * Drop into the loaded user program. Runs at EL1t (same EL as kernel, 191 * but uses SP_EL0 — gives us a separate user stack without setting up 192 * an MMU). DAIF stays masked since we don't service interrupts. 193 */ 194 .globl eret_to_user 195 eret_to_user: 196 msr sp_el0, x1 197 msr elr_el1, x0 198 mov x9, #0x3c4 /* EL1t, DAIF=1111 */ 199 msr spsr_el1, x9 200 /* Clear all GP regs so user starts clean. argc/argv come in via the 201 * SysV stack layout, which the user reads directly off SP_EL0. Some 202 * boot0/1 seed-stage binaries (notably M0) read xN before any write, 203 * so leaking kernel register state past the eret would fault them. */ 204 mov x0, xzr 205 mov x1, xzr 206 mov x2, xzr 207 mov x3, xzr 208 mov x4, xzr 209 mov x5, xzr 210 mov x6, xzr 211 mov x7, xzr 212 mov x8, xzr 213 mov x9, xzr 214 mov x10, xzr 215 mov x11, xzr 216 mov x12, xzr 217 mov x13, xzr 218 mov x14, xzr 219 mov x15, xzr 220 mov x16, xzr 221 mov x17, xzr 222 mov x18, xzr 223 mov x19, xzr 224 mov x20, xzr 225 mov x21, xzr 226 mov x22, xzr 227 mov x23, xzr 228 mov x24, xzr 229 mov x25, xzr 230 mov x26, xzr 231 mov x27, xzr 232 mov x28, xzr 233 mov x29, xzr 234 mov x30, xzr 235 eret 236 237 238 /* ─── C-callable thunks ─────────────────────────────────────────────────── 239 * The arm64 sysreg name is encoded in the msr/mrs opcode itself, so each 240 * register needs its own emit site — sysreg_read/sysreg_write dispatch on 241 * an integer id that must match the SR_* enum in kernel.c. Likewise for 242 * arm64_barrier (BAR_*) and cpu_pause (PAUSE_*). 243 */ 244 245 /* SR_* — matches kernel.c enum, declaration order. */ 246 #define SR_MAIR_EL1 0 247 #define SR_TCR_EL1 1 248 #define SR_TTBR0_EL1 2 249 #define SR_SCTLR_EL1 3 250 #define SR_CPACR_EL1 4 251 #define SR_SP_EL0 5 252 #define SR_FAR_EL1 6 253 254 .globl sysreg_read 255 sysreg_read: 256 cmp x0, #SR_SCTLR_EL1 257 b.eq .Lrd_sctlr_el1 258 cmp x0, #SR_SP_EL0 259 b.eq .Lrd_sp_el0 260 cmp x0, #SR_FAR_EL1 261 b.eq .Lrd_far_el1 262 mov x0, xzr 263 ret 264 .Lrd_sctlr_el1: mrs x0, sctlr_el1; ret 265 .Lrd_sp_el0: mrs x0, sp_el0; ret 266 .Lrd_far_el1: mrs x0, far_el1; ret 267 268 .globl sysreg_write 269 sysreg_write: 270 cmp x0, #SR_MAIR_EL1 271 b.eq .Lwr_mair_el1 272 cmp x0, #SR_TCR_EL1 273 b.eq .Lwr_tcr_el1 274 cmp x0, #SR_TTBR0_EL1 275 b.eq .Lwr_ttbr0_el1 276 cmp x0, #SR_SCTLR_EL1 277 b.eq .Lwr_sctlr_el1 278 cmp x0, #SR_CPACR_EL1 279 b.eq .Lwr_cpacr_el1 280 cmp x0, #SR_SP_EL0 281 b.eq .Lwr_sp_el0 282 ret 283 .Lwr_mair_el1: msr mair_el1, x1; ret 284 .Lwr_tcr_el1: msr tcr_el1, x1; ret 285 .Lwr_ttbr0_el1: msr ttbr0_el1, x1; ret 286 .Lwr_sctlr_el1: msr sctlr_el1, x1; ret 287 .Lwr_cpacr_el1: msr cpacr_el1, x1; ret 288 .Lwr_sp_el0: msr sp_el0, x1; ret 289 290 /* BAR_* — matches kernel.c enum. */ 291 #define BAR_DSB_SY 0 292 #define BAR_DSB_ISH 1 293 #define BAR_DMB_ISH 2 294 #define BAR_DMB_ISHST 3 295 #define BAR_ISB 4 296 297 .globl arm64_barrier 298 arm64_barrier: 299 cmp x0, #BAR_DSB_SY 300 b.eq .Lbar_dsb_sy 301 cmp x0, #BAR_DSB_ISH 302 b.eq .Lbar_dsb_ish 303 cmp x0, #BAR_DMB_ISH 304 b.eq .Lbar_dmb_ish 305 cmp x0, #BAR_DMB_ISHST 306 b.eq .Lbar_dmb_ishst 307 isb 308 ret 309 .Lbar_dsb_sy: dsb sy; ret 310 .Lbar_dsb_ish: dsb ish; ret 311 .Lbar_dmb_ish: dmb ish; ret 312 .Lbar_dmb_ishst: dmb ishst; ret 313 314 /* Bare cache/TLB primitives — kernel.c brackets them with arm64_barrier 315 * calls so the dsb scope at each call site stays explicit (the kernel 316 * uses both `dsb ish` and `dsb sy` patterns, not a single canonical 317 * sequence). */ 318 .globl arm64_ic_iallu 319 arm64_ic_iallu: 320 ic iallu 321 ret 322 323 .globl arm64_tlbi_vmalle1 324 arm64_tlbi_vmalle1: 325 tlbi vmalle1 326 ret 327 328 /* PAUSE_* — matches kernel.c enum. */ 329 #define PAUSE_WFE 0 330 #define PAUSE_WFI 1 331 #define PAUSE_YIELD 2 332 333 .globl cpu_pause 334 cpu_pause: 335 cmp x0, #PAUSE_WFI 336 b.eq .Lp_wfi 337 cmp x0, #PAUSE_YIELD 338 b.eq .Lp_yield 339 wfe 340 ret 341 .Lp_wfi: wfi; ret 342 .Lp_yield: yield; ret 343 344 /* PSCI / SMCCC: x0 = conduit (0=HVC, 1=SMC), x1 = function id. 345 * Returns the call's x0. */ 346 .globl arm64_psci_call 347 arm64_psci_call: 348 mov x9, x0 349 mov x0, x1 350 cbnz x9, .Lpsci_smc 351 hvc #0 352 ret 353 .Lpsci_smc: 354 smc #0 355 ret