kernel.S (15849B)
1 /* seed kernel — arm64 boot protocol entry, vector table, SVC handler, 2 * plus C-callable thunks for ops that can't be expressed in plain C 3 * (sysreg msr/mrs, barriers, cache/TLB ops, PSCI conduits, cpu pause). */ 4 5 .section .text, "ax" 6 .globl _start 7 _start: 8 /* arm64 Image header (Documentation/arm64/booting.rst). 9 * code0 must be a valid instruction (a branch, in our case). 10 * 11 * The header lives at the top of `.text` (not a separate 12 * `.head.text` section) because we link kernel.S first, so its 13 * `.text` content lands at file offset 0 — QEMU's arm64 boot 14 * protocol detects this image by `ARM\x64` magic at offset 0x38, 15 * which is the `.ascii "ARM\x64"` line below. No ld-script 16 * section ordering required. 17 * 18 * Entry symbol is `_start` (not `_head`) so a script-less linker 19 * — tcc3 in particular — picks us up via its hard-coded default. 20 * Image-size end marker is `_end`, which both gnu ld and tcc3 21 * auto-define at the end of `.bss`. */ 22 b stext 23 .long 0 24 .quad 0x80000 /* text_offset (preferred load offset within RAM) */ 25 .quad _end - _start /* image_size */ 26 .quad 0xa /* flags: 4K pages, anywhere in physmem, LE */ 27 .quad 0 28 .quad 0 29 .quad 0 30 .ascii "ARM\x64" /* magic */ 31 .long 0 /* PE COFF offset (none) */ 32 33 stext: 34 /* Entry contract: x0 = DTB phys, MMU off, caches off, EL2 or EL1. */ 35 msr daifset, #0xf 36 37 /* If we entered at EL2, drop to EL1. Otherwise we're already at EL1. */ 38 mrs x9, CurrentEL 39 lsr x9, x9, #2 40 cmp x9, #2 41 b.ne in_el1 42 43 /* EL2 → EL1: set HCR_EL2.RW=1 (EL1 is AArch64), CNTHCTL/CNTVOFF defaults, 44 * SPSR=EL1h with DAIF masked, ELR=in_el1, eret. 45 * 46 * Address loads use `ldr Xn, =sym` rather than `adrp/add :lo12:` so this 47 * file assembles under tcc3 (the MOVW_UABS_G{0..3} reloc chain that 48 * `ldr =sym` lowers to is the same chain `arm64-gen.c` uses for every 49 * compiler-emitted symbol load). */ 50 mov x9, #(1 << 31) 51 msr hcr_el2, x9 52 mov x9, #0x3c5 /* EL1h, DAIF=1111 */ 53 msr spsr_el2, x9 54 ldr x9, =in_el1 55 msr elr_el2, x9 56 /* Make sure SP_EL1 is set before we eret to EL1 (else we land with 57 * an undefined SP). Use the same kernel stack we're about to install. */ 58 ldr x9, =kstack_top 59 msr sp_el1, x9 60 eret 61 62 in_el1: 63 /* Stack. */ 64 ldr x9, =kstack_top 65 mov sp, x9 66 67 /* Vector table. */ 68 ldr x9, =vector_table 69 msr vbar_el1, x9 70 isb 71 72 /* Enable FP/SIMD at EL1+EL0 (CPACR_EL1.FPEN = 0b11) before any C 73 * call. The gcc build sets FPEN later from kmain() and gets away 74 * with it because `-mgeneral-regs-only` ensures gcc never emits 75 * FP/SIMD instructions; tcc has no equivalent flag and saves the 76 * callee-saved SIMD regs in every function prologue, so the very 77 * first `bl kmain` would take an FP-trapped exception. */ 78 mov x9, #(3 << 20) 79 msr cpacr_el1, x9 80 isb 81 82 /* Zero BSS. `__bss_start` and `_end` are auto-defined by both 83 * gnu ld (via the kernel.lds bracket) and tcc3 (via the 84 * `bss-start-symbol` simple-patch + tcc's existing `_end` def). */ 85 ldr x1, =__bss_start 86 ldr x2, =_end 87 1: cmp x1, x2 88 b.ge 2f 89 str xzr, [x1], #8 90 b 1b 91 2: 92 /* Hand control to C. x0 still = DTB phys (not clobbered above). */ 93 bl kmain 94 95 /* kmain shouldn't return. */ 96 hang: 97 wfe 98 b hang 99 100 101 /* ─── Exception vector table ──────────────────────────────────────────── */ 102 103 /* The 16 entries are unrolled (rather than emitted via `.macro VENTRY` + 104 * 16 invocations) because tcc 0.9.26's `tccasm.c` does not implement 105 * `.macro` / `.endm`. Each entry is `.balign 0x80` (puts the entry on its 106 * 128-byte slot) followed by a single `b <handler>`. */ 107 108 .section .text, "ax" 109 .balign 0x800 110 .globl vector_table 111 vector_table: 112 /* Current EL with SP_EL0 (we never run kernel like this — only user). */ 113 .balign 0x80 /* 0x000: SVC from EL1t (our "user") */ 114 b el1_sp0_sync 115 .balign 0x80 /* 0x080 */ 116 b unhandled 117 .balign 0x80 /* 0x100 */ 118 b unhandled 119 .balign 0x80 /* 0x180 */ 120 b unhandled 121 /* Current EL with SP_ELx (kernel internal). */ 122 .balign 0x80 /* 0x200: panic on kernel sync fault */ 123 b el1_spx_sync 124 .balign 0x80 /* 0x280 */ 125 b unhandled 126 .balign 0x80 /* 0x300 */ 127 b unhandled 128 .balign 0x80 /* 0x380 */ 129 b unhandled 130 /* Lower EL using AArch64 (EL0). Unused in this design but wired. */ 131 .balign 0x80 /* 0x400 */ 132 b el1_sp0_sync 133 .balign 0x80 /* 0x480 */ 134 b unhandled 135 .balign 0x80 /* 0x500 */ 136 b unhandled 137 .balign 0x80 /* 0x580 */ 138 b unhandled 139 /* Lower EL using AArch32 (unused). */ 140 .balign 0x80 /* 0x600 */ 141 b unhandled 142 .balign 0x80 /* 0x680 */ 143 b unhandled 144 .balign 0x80 /* 0x700 */ 145 b unhandled 146 .balign 0x80 /* 0x780 */ 147 b unhandled 148 149 150 /* ─── Trap entry/exit ───────────────────────────────────────────────────── 151 * Save x0..x30 + ELR_EL1 + SPSR_EL1 onto the kernel stack as a trapframe, 152 * call a C handler with (esr, &tf), restore, eret. The C handler reads 153 * and writes tf->x[0..7] for syscall args and return value, plus tf->x[8] 154 * for the syscall number. 155 * 156 * Save/restore are unrolled at each of the three entries (sp0_sync, 157 * spx_sync, unhandled) rather than being factored through `.macro` — 158 * tcc 0.9.26's tccasm.c does not implement .macro, and the obvious 159 * shared-subroutine factoring (`b save_tf; ...; b restore_tf` with a 160 * linkage register) would clobber the user's x18 before saving it. 161 */ 162 163 el1_sp0_sync: 164 sub sp, sp, #272 165 stp x0, x1, [sp, #0] 166 stp x2, x3, [sp, #16] 167 stp x4, x5, [sp, #32] 168 stp x6, x7, [sp, #48] 169 stp x8, x9, [sp, #64] 170 stp x10, x11, [sp, #80] 171 stp x12, x13, [sp, #96] 172 stp x14, x15, [sp, #112] 173 stp x16, x17, [sp, #128] 174 stp x18, x19, [sp, #144] 175 stp x20, x21, [sp, #160] 176 stp x22, x23, [sp, #176] 177 stp x24, x25, [sp, #192] 178 stp x26, x27, [sp, #208] 179 stp x28, x29, [sp, #224] 180 str x30, [sp, #240] 181 mrs x10, elr_el1 182 mrs x11, spsr_el1 183 stp x10, x11, [sp, #248] 184 mrs x0, esr_el1 185 mov x1, sp 186 bl trap_sync 187 ldp x10, x11, [sp, #248] 188 msr elr_el1, x10 189 msr spsr_el1, x11 190 ldr x30, [sp, #240] 191 ldp x28, x29, [sp, #224] 192 ldp x26, x27, [sp, #208] 193 ldp x24, x25, [sp, #192] 194 ldp x22, x23, [sp, #176] 195 ldp x20, x21, [sp, #160] 196 ldp x18, x19, [sp, #144] 197 ldp x16, x17, [sp, #128] 198 ldp x14, x15, [sp, #112] 199 ldp x12, x13, [sp, #96] 200 ldp x10, x11, [sp, #80] 201 ldp x8, x9, [sp, #64] 202 ldp x6, x7, [sp, #48] 203 ldp x4, x5, [sp, #32] 204 ldp x2, x3, [sp, #16] 205 ldp x0, x1, [sp, #0] 206 add sp, sp, #272 207 eret 208 209 el1_spx_sync: 210 /* Same shape as user sync — let C distinguish via SPSR/ESR if needed. */ 211 sub sp, sp, #272 212 stp x0, x1, [sp, #0] 213 stp x2, x3, [sp, #16] 214 stp x4, x5, [sp, #32] 215 stp x6, x7, [sp, #48] 216 stp x8, x9, [sp, #64] 217 stp x10, x11, [sp, #80] 218 stp x12, x13, [sp, #96] 219 stp x14, x15, [sp, #112] 220 stp x16, x17, [sp, #128] 221 stp x18, x19, [sp, #144] 222 stp x20, x21, [sp, #160] 223 stp x22, x23, [sp, #176] 224 stp x24, x25, [sp, #192] 225 stp x26, x27, [sp, #208] 226 stp x28, x29, [sp, #224] 227 str x30, [sp, #240] 228 mrs x10, elr_el1 229 mrs x11, spsr_el1 230 stp x10, x11, [sp, #248] 231 mrs x0, esr_el1 232 mov x1, sp 233 bl trap_kernel 234 ldp x10, x11, [sp, #248] 235 msr elr_el1, x10 236 msr spsr_el1, x11 237 ldr x30, [sp, #240] 238 ldp x28, x29, [sp, #224] 239 ldp x26, x27, [sp, #208] 240 ldp x24, x25, [sp, #192] 241 ldp x22, x23, [sp, #176] 242 ldp x20, x21, [sp, #160] 243 ldp x18, x19, [sp, #144] 244 ldp x16, x17, [sp, #128] 245 ldp x14, x15, [sp, #112] 246 ldp x12, x13, [sp, #96] 247 ldp x10, x11, [sp, #80] 248 ldp x8, x9, [sp, #64] 249 ldp x6, x7, [sp, #48] 250 ldp x4, x5, [sp, #32] 251 ldp x2, x3, [sp, #16] 252 ldp x0, x1, [sp, #0] 253 add sp, sp, #272 254 eret 255 256 unhandled: 257 sub sp, sp, #272 258 stp x0, x1, [sp, #0] 259 stp x2, x3, [sp, #16] 260 stp x4, x5, [sp, #32] 261 stp x6, x7, [sp, #48] 262 stp x8, x9, [sp, #64] 263 stp x10, x11, [sp, #80] 264 stp x12, x13, [sp, #96] 265 stp x14, x15, [sp, #112] 266 stp x16, x17, [sp, #128] 267 stp x18, x19, [sp, #144] 268 stp x20, x21, [sp, #160] 269 stp x22, x23, [sp, #176] 270 stp x24, x25, [sp, #192] 271 stp x26, x27, [sp, #208] 272 stp x28, x29, [sp, #224] 273 str x30, [sp, #240] 274 mrs x10, elr_el1 275 mrs x11, spsr_el1 276 stp x10, x11, [sp, #248] 277 mrs x0, esr_el1 278 mov x1, sp 279 bl trap_unhandled 280 ldp x10, x11, [sp, #248] 281 msr elr_el1, x10 282 msr spsr_el1, x11 283 ldr x30, [sp, #240] 284 ldp x28, x29, [sp, #224] 285 ldp x26, x27, [sp, #208] 286 ldp x24, x25, [sp, #192] 287 ldp x22, x23, [sp, #176] 288 ldp x20, x21, [sp, #160] 289 ldp x18, x19, [sp, #144] 290 ldp x16, x17, [sp, #128] 291 ldp x14, x15, [sp, #112] 292 ldp x12, x13, [sp, #96] 293 ldp x10, x11, [sp, #80] 294 ldp x8, x9, [sp, #64] 295 ldp x6, x7, [sp, #48] 296 ldp x4, x5, [sp, #32] 297 ldp x2, x3, [sp, #16] 298 ldp x0, x1, [sp, #0] 299 add sp, sp, #272 300 eret 301 302 303 /* ─── eret_to_user(entry, sp) ───────────────────────────────────────────── 304 * Drop into the loaded user program. Runs at EL1t (same EL as kernel, 305 * but uses SP_EL0 — gives us a separate user stack without setting up 306 * an MMU). DAIF stays masked since we don't service interrupts. 307 */ 308 .globl eret_to_user 309 eret_to_user: 310 msr sp_el0, x1 311 msr elr_el1, x0 312 mov x9, #0x3c4 /* EL1t, DAIF=1111 */ 313 msr spsr_el1, x9 314 /* Clear all GP regs so user starts clean. argc/argv come in via the 315 * SysV stack layout, which the user reads directly off SP_EL0. Some 316 * boot0/1 seed-stage binaries (notably M0) read xN before any write, 317 * so leaking kernel register state past the eret would fault them. */ 318 mov x0, xzr 319 mov x1, xzr 320 mov x2, xzr 321 mov x3, xzr 322 mov x4, xzr 323 mov x5, xzr 324 mov x6, xzr 325 mov x7, xzr 326 mov x8, xzr 327 mov x9, xzr 328 mov x10, xzr 329 mov x11, xzr 330 mov x12, xzr 331 mov x13, xzr 332 mov x14, xzr 333 mov x15, xzr 334 mov x16, xzr 335 mov x17, xzr 336 mov x18, xzr 337 mov x19, xzr 338 mov x20, xzr 339 mov x21, xzr 340 mov x22, xzr 341 mov x23, xzr 342 mov x24, xzr 343 mov x25, xzr 344 mov x26, xzr 345 mov x27, xzr 346 mov x28, xzr 347 mov x29, xzr 348 mov x30, xzr 349 eret 350 351 352 /* ─── C-callable thunks ─────────────────────────────────────────────────── 353 * The arm64 sysreg name is encoded in the msr/mrs opcode itself, so each 354 * register needs its own emit site — sysreg_read/sysreg_write dispatch on 355 * an integer id that must match the SR_* enum in kernel.c. Likewise for 356 * arm64_barrier (BAR_*) and cpu_pause (PAUSE_*). 357 */ 358 359 /* SR_* — matches kernel.c enum, declaration order. */ 360 #define SR_MAIR_EL1 0 361 #define SR_TCR_EL1 1 362 #define SR_TTBR0_EL1 2 363 #define SR_SCTLR_EL1 3 364 #define SR_CPACR_EL1 4 365 #define SR_SP_EL0 5 366 #define SR_FAR_EL1 6 367 368 .globl sysreg_read 369 sysreg_read: 370 cmp x0, #SR_SCTLR_EL1 371 b.eq .Lrd_sctlr_el1 372 cmp x0, #SR_SP_EL0 373 b.eq .Lrd_sp_el0 374 cmp x0, #SR_FAR_EL1 375 b.eq .Lrd_far_el1 376 mov x0, xzr 377 ret 378 .Lrd_sctlr_el1: mrs x0, sctlr_el1; ret 379 .Lrd_sp_el0: mrs x0, sp_el0; ret 380 .Lrd_far_el1: mrs x0, far_el1; ret 381 382 .globl sysreg_write 383 sysreg_write: 384 cmp x0, #SR_MAIR_EL1 385 b.eq .Lwr_mair_el1 386 cmp x0, #SR_TCR_EL1 387 b.eq .Lwr_tcr_el1 388 cmp x0, #SR_TTBR0_EL1 389 b.eq .Lwr_ttbr0_el1 390 cmp x0, #SR_SCTLR_EL1 391 b.eq .Lwr_sctlr_el1 392 cmp x0, #SR_CPACR_EL1 393 b.eq .Lwr_cpacr_el1 394 cmp x0, #SR_SP_EL0 395 b.eq .Lwr_sp_el0 396 ret 397 .Lwr_mair_el1: msr mair_el1, x1; ret 398 .Lwr_tcr_el1: msr tcr_el1, x1; ret 399 .Lwr_ttbr0_el1: msr ttbr0_el1, x1; ret 400 .Lwr_sctlr_el1: msr sctlr_el1, x1; ret 401 .Lwr_cpacr_el1: msr cpacr_el1, x1; ret 402 .Lwr_sp_el0: msr sp_el0, x1; ret 403 404 /* BAR_* — matches kernel.c enum. */ 405 #define BAR_DSB_SY 0 406 #define BAR_DSB_ISH 1 407 #define BAR_DMB_ISH 2 408 #define BAR_DMB_ISHST 3 409 #define BAR_ISB 4 410 411 .globl arm64_barrier 412 arm64_barrier: 413 cmp x0, #BAR_DSB_SY 414 b.eq .Lbar_dsb_sy 415 cmp x0, #BAR_DSB_ISH 416 b.eq .Lbar_dsb_ish 417 cmp x0, #BAR_DMB_ISH 418 b.eq .Lbar_dmb_ish 419 cmp x0, #BAR_DMB_ISHST 420 b.eq .Lbar_dmb_ishst 421 isb 422 ret 423 .Lbar_dsb_sy: dsb sy; ret 424 .Lbar_dsb_ish: dsb ish; ret 425 .Lbar_dmb_ish: dmb ish; ret 426 .Lbar_dmb_ishst: dmb ishst; ret 427 428 /* Bare cache/TLB primitives — kernel.c brackets them with arm64_barrier 429 * calls so the dsb scope at each call site stays explicit (the kernel 430 * uses both `dsb ish` and `dsb sy` patterns, not a single canonical 431 * sequence). */ 432 .globl arm64_ic_iallu 433 arm64_ic_iallu: 434 ic iallu 435 ret 436 437 .globl arm64_tlbi_vmalle1 438 arm64_tlbi_vmalle1: 439 tlbi vmalle1 440 ret 441 442 /* PAUSE_* — matches kernel.c enum. */ 443 #define PAUSE_WFE 0 444 #define PAUSE_WFI 1 445 #define PAUSE_YIELD 2 446 447 .globl cpu_pause 448 cpu_pause: 449 cmp x0, #PAUSE_WFI 450 b.eq .Lp_wfi 451 cmp x0, #PAUSE_YIELD 452 b.eq .Lp_yield 453 wfe 454 ret 455 .Lp_wfi: wfi; ret 456 .Lp_yield: yield; ret 457 458 /* PSCI / SMCCC: x0 = conduit (0=HVC, 1=SMC), x1 = function id. 459 * Returns the call's x0. */ 460 .globl arm64_psci_call 461 arm64_psci_call: 462 mov x9, x0 463 mov x0, x1 464 cbnz x9, .Lpsci_smc 465 hvc #0 466 ret 467 .Lpsci_smc: 468 smc #0 469 ret 470 471 472 /* ─── Kernel stack ──────────────────────────────────────────────────────── 473 * 64 KB reserved as plain `.bss`. Keeping the stack inside `.bss` (rather 474 * than a custom `.stack` section) means the only link-time service the 475 * kernel needs is bracketing the merged `.bss` with __bss_start/__bss_end 476 * — no custom output-section ordering, no link-script reservations. The 477 * stext bss-zero loop runs before any stack write (it only touches x1/x2), 478 * so zeroing our own stack region first is safe. */ 479 .section .bss, "aw", %nobits 480 .balign 16 481 .skip 0x10000 482 .globl kstack_top 483 kstack_top: