emu.h (18397B)
1 #ifndef KIT_EMU_INTERNAL_H 2 #define KIT_EMU_INTERNAL_H 3 4 /* Internal API for libkit's guest-ISA emulator. Public surface is 5 * kit_emu_* in <kit/emu.h>; the implementation in src/emu/emu.c 6 * composes the pieces declared here. See doc/EMU.md for design. 7 * 8 * Layering: emu.c owns KitEmu lifecycle and the translate/dispatch 9 * loop; per-ISA decoders/lifters live behind ArchImpl hooks, while CPUState 10 * synthesis, the JIT code cache and reserved-VA region, and the runtime helper 11 * trampolines each live behind one of the surfaces below. */ 12 13 #include <kit/cg.h> 14 #include <kit/emu.h> 15 #include <kit/jit.h> 16 17 #include "arch/arch.h" 18 #include "core/core.h" 19 #include "obj/obj.h" 20 21 typedef struct LinkImage LinkImage; 22 typedef struct Linker Linker; 23 typedef struct EmuCPUState EmuCPUState; 24 typedef struct ObjFormatImpl ObjFormatImpl; 25 typedef struct KitOsImpl KitOsImpl; 26 typedef struct EmuProcess EmuProcess; 27 typedef struct EmuThread EmuThread; 28 typedef struct EmuExternalBindings EmuExternalBindings; 29 typedef struct ObjFormatEmuOps ObjFormatEmuOps; 30 typedef struct EmuLoadedObject EmuLoadedObject; 31 typedef struct EmuDynNeededIter EmuDynNeededIter; 32 typedef struct EmuDynSymbol EmuDynSymbol; 33 typedef struct EmuDynRelocIter EmuDynRelocIter; 34 typedef struct EmuDynReloc EmuDynReloc; 35 typedef struct EmuObjectFormatData EmuObjectFormatData; 36 37 /* ---- Configuration knobs ---------------------------------------- */ 38 39 /* Bounded so the translator can stack-allocate the decoded instruction buffer. 40 */ 41 #define EMU_MAX_INSTS_PER_BLOCK 64u 42 43 /* Reserved JIT code region. emu_runtime mmap's PROT_NONE up front and 44 * commits pages as cold blocks land. Sized for v1 — chaining and the 45 * code cache assume host VAs of placed sections never move, so this 46 * region also never grows. */ 47 #define EMU_CODE_REGION_SIZE (128ull * 1024ull * 1024ull) 48 49 /* ---- Per-emu JIT host wiring ------------------------------------ */ 50 /* `host` is borrowed and must outlive the KitEmu. The setter remains for 51 * internal callers that construct an emu before wiring driver-owned JIT state. 52 */ 53 void emu_set_jit_host(KitEmu*, const KitJitHost*); 54 const KitJitHost* emu_get_jit_host(const KitEmu*); 55 56 /* ---- Guest executable image ------------------------------------- */ 57 58 typedef enum EmuMemPerm { 59 EMU_MEM_READ = 1u << 0, 60 EMU_MEM_WRITE = 1u << 1, 61 EMU_MEM_EXEC = 1u << 2, 62 } EmuMemPerm; 63 64 typedef enum EmuMapKind { 65 EMU_MAP_ANON, 66 EMU_MAP_FILE, 67 EMU_MAP_GUARD, 68 } EmuMapKind; 69 70 typedef enum EmuFaultKind { 71 EMU_FAULT_NONE = 0, 72 EMU_FAULT_UNMAPPED, 73 EMU_FAULT_PROT, 74 EMU_FAULT_EXEC_INVALIDATED, 75 } EmuFaultKind; 76 77 typedef struct EmuMemFault { 78 EmuFaultKind kind; 79 u64 addr; 80 u8 access; 81 } EmuMemFault; 82 83 typedef struct EmuMap { 84 u64 start; 85 u64 end; 86 u8 perms; 87 EmuMapKind kind; 88 u32 flags; 89 u64 generation; 90 u8* bytes; 91 u8* dirty_pages; 92 u8* translated_pages; 93 } EmuMap; 94 95 typedef struct EmuAddrSpace { 96 Compiler* compiler; 97 Heap* heap; 98 u64 page_size; 99 EmuMap* maps; 100 u32 nmaps; 101 u32 maps_cap; 102 u64 generation; 103 u64 brk_base; 104 u64 brk_cur; 105 u64 brk_max; 106 EmuMemFault last_fault; 107 } EmuAddrSpace; 108 109 typedef struct EmuDynamicImport { 110 KitSlice object_name; 111 KitSlice symbol_name; 112 u64 got_vaddr; 113 u64 thunk_vaddr; 114 KitEmuImportSignature signature; 115 } EmuDynamicImport; 116 117 typedef struct EmuImportBinding { 118 KitSlice object_name; 119 KitSlice symbol_name; 120 u64 got_vaddr; 121 u64 thunk_vaddr; 122 u64 resolved_guest_addr; 123 void* resolved_host_fn; 124 u32 flags; 125 KitEmuImportSignature signature; 126 } EmuImportBinding; 127 128 typedef struct EmuObjectImports { 129 KitSlice needed[16]; 130 u32 nneeded; 131 } EmuObjectImports; 132 133 typedef struct EmuObjectInitFini { 134 u64 init; 135 u64 fini; 136 u64 init_array; 137 u64 init_arraysz; 138 u64 fini_array; 139 u64 fini_arraysz; 140 } EmuObjectInitFini; 141 142 struct EmuObjectFormatData { 143 void* data; 144 size_t size; 145 size_t align; 146 }; 147 148 typedef struct EmuObjectProcessInfo { 149 u64 headers_vaddr; 150 u64 header_entry_size; 151 u64 header_count; 152 KitSlice interpreter_path; 153 } EmuObjectProcessInfo; 154 155 typedef u32 EmuDynRelocClass; 156 #define EMU_DYN_RELOC_NONE 0u 157 #define EMU_DYN_RELOC_RELATIVE 1u 158 #define EMU_DYN_RELOC_SYMBOLIC 2u 159 #define EMU_DYN_RELOC_IMPORT_SLOT 3u 160 161 struct EmuDynNeededIter { 162 const EmuLoadedObject* object; 163 u32 index; 164 }; 165 166 struct EmuDynSymbol { 167 KitSlice name; 168 u64 value; 169 u64 size; 170 u64 index; 171 u32 flags; 172 int defined; 173 }; 174 175 struct EmuDynReloc { 176 u64 patch_addr; 177 u64 symbol_index; 178 u64 wire_type; 179 i64 addend; 180 u32 width; 181 }; 182 183 typedef u32 EmuDynRelocTableKind; 184 #define EMU_DYN_RELOC_TABLE_MAIN 0u 185 #define EMU_DYN_RELOC_TABLE_PLT 1u 186 187 struct EmuDynRelocIter { 188 const EmuLoadedObject* object; 189 EmuDynRelocTableKind table; 190 u64 cursor; 191 }; 192 193 typedef struct EmuTlsModule { 194 u64 image_vaddr; 195 u64 filesz; 196 u64 memsz; 197 u64 align; 198 u32 module_id; 199 } EmuTlsModule; 200 201 typedef struct EmuLoadedObject { 202 KitSlice name; 203 KitSlice soname; 204 u64 load_bias; 205 u64 map_start; 206 u64 map_end; 207 EmuObjectImports imports; 208 EmuObjectInitFini init_fini; 209 EmuObjectFormatData format; 210 EmuTlsModule tls; 211 u32 flags; 212 } EmuLoadedObject; 213 214 typedef struct EmuLinkMap { 215 EmuLoadedObject* objects; 216 u32 nobjects; 217 u32 objects_cap; 218 u32 main_object; 219 u32 global_scope_head; 220 } EmuLinkMap; 221 222 typedef struct EmuDlPolicy { 223 u32 flags; 224 u32 global_scope_head; 225 u32 search_path_head; 226 } EmuDlPolicy; 227 228 typedef struct EmuTlsState { 229 EmuTlsModule* modules; 230 u32 nmodules; 231 u32 modules_cap; 232 u64 static_size; 233 u64 static_align; 234 } EmuTlsState; 235 236 typedef struct EmuTlsBlock { 237 u32 module_id; 238 u64 base; 239 u64 memsz; 240 } EmuTlsBlock; 241 242 typedef struct EmuTlsBlocks { 243 EmuTlsBlock* blocks; 244 u32 nblocks; 245 u32 blocks_cap; 246 } EmuTlsBlocks; 247 248 typedef struct EmuLoadedImage { 249 EmuAddrSpace addr_space; 250 u64 entry_pc; /* guest VA of the program entry point */ 251 u64 initial_sp; /* guest VA of the initial stack pointer */ 252 EmuObjectProcessInfo process_info; 253 EmuDynamicImport* imports; 254 u32 nimports; 255 EmuImportBinding* import_bindings; 256 u32 nimport_bindings; 257 u32 import_bindings_cap; 258 u64 import_thunk_base; 259 u64 import_thunk_size; 260 u64 import_thunk_next; 261 EmuLinkMap link_map; 262 } EmuLoadedImage; 263 264 typedef struct EmuLoadOptions { 265 KitSlice name; 266 KitSlice bytes; 267 KitTargetSpec guest_target; 268 const char* const* argv; 269 const char* const* envp; 270 const KitOsImpl* os; 271 const EmuExternalBindings* bindings; 272 EmuProcess* process; 273 } EmuLoadOptions; 274 275 typedef struct EmuSyscallRequest { 276 u64 number; 277 u64 args[6]; 278 } EmuSyscallRequest; 279 280 typedef struct EmuSyscallResult { 281 i64 result; 282 i32 guest_errno; 283 u32 flags; 284 } EmuSyscallResult; 285 286 #define EMU_SYSCALL_RESULT_SKIP_ENCODE 1u 287 288 struct EmuExternalBindings { 289 KitStatus (*syscall)(void* user, EmuProcess*, EmuThread*, 290 const EmuSyscallRequest*, EmuSyscallResult* out); 291 KitStatus (*resolve_import)(void* user, EmuProcess*, const EmuDynamicImport*, 292 KitEmuResolvedImport* out); 293 KitStatus (*resolve_object)(void* user, EmuProcess*, KitSlice object_name, 294 KitSlice* out_bytes); 295 void* user; 296 }; 297 298 typedef struct EmuFaultEvent { 299 EmuFaultKind kind; 300 u64 addr; 301 u64 pc; 302 u64 next_pc; 303 u8 access; 304 } EmuFaultEvent; 305 306 struct EmuProcess { 307 Compiler* compiler; 308 KitTargetSpec guest_target; 309 const ObjFormatImpl* obj_format; 310 const ArchImpl* arch; 311 const KitOsImpl* os; 312 void* os_private; 313 EmuLoadedImage image; 314 EmuExternalBindings bindings; 315 EmuDlPolicy dl_policy; 316 EmuTlsState tls_state; 317 }; 318 319 struct EmuThread { 320 EmuProcess* process; 321 EmuCPUState* cpu; 322 void* os_private; 323 }; 324 325 struct KitOsImpl { 326 KitOSKind kind; 327 const char* name; 328 KitStatus (*emu_init_process_private)(Compiler*, EmuProcess*); 329 void (*emu_destroy_process_private)(Compiler*, EmuProcess*); 330 KitStatus (*emu_init_thread_private)(Compiler*, EmuProcess*, EmuThread*); 331 void (*emu_destroy_thread_private)(Compiler*, EmuThread*); 332 KitStatus (*emu_init_process)(Compiler*, EmuProcess*, const EmuLoadOptions*, 333 const EmuLoadedImage*); 334 KitStatus (*emu_init_thread)(Compiler*, EmuProcess*, EmuThread*); 335 KitStatus (*emu_decode_syscall)(EmuProcess*, EmuThread*, 336 EmuSyscallRequest* out); 337 KitStatus (*emu_encode_syscall_result)(EmuProcess*, EmuThread*, 338 const EmuSyscallResult*); 339 u64 (*emu_syscall_next_pc)(EmuProcess*, EmuThread*, const EmuSyscallRequest*, 340 u64 next_pc); 341 KitStatus (*emu_find_map_region)(EmuProcess*, u64 nbytes, u64 align, 342 u32 purpose, u64* out); 343 void (*emu_note_map_region)(EmuProcess*, u64 base, u64 nbytes, u32 purpose); 344 KitStatus (*emu_default_syscall)(void* user, EmuProcess*, EmuThread*, 345 const EmuSyscallRequest*, 346 EmuSyscallResult* out); 347 KitStatus (*emu_deliver_fault)(EmuProcess*, EmuThread*, const EmuFaultEvent*, 348 u64* next_pc_out); 349 }; 350 351 const KitOsImpl* os_lookup(KitOSKind); 352 353 #define EMU_OS_MAP_MMAP 1u 354 #define EMU_OS_MAP_DL_THUNKS 2u 355 #define EMU_OS_MAP_TLS 3u 356 357 int emu_loaded_image_attach_cpu(EmuCPUState*, EmuLoadedImage*); 358 void emu_unload_image(Compiler*, EmuLoadedImage*); 359 KitStatus emu_dl_init_process(Compiler*, EmuProcess*); 360 KitStatus emu_dl_load_dependencies_and_relocate(Compiler*, EmuProcess*, 361 const EmuLoadOptions*, 362 const ObjFormatEmuOps*); 363 KitStatus emu_dl_lookup_symbol(EmuProcess*, KitSlice symbol, u64* out_addr); 364 KitStatus emu_dl_resolve_import_thunk(EmuProcess*, u64 target, 365 EmuImportBinding** out); 366 KitStatus emu_call_host_import(EmuThread*, EmuImportBinding*, const u64* args, 367 u32 nargs, u64* result_out); 368 KitStatus emu_object_format_data_alloc(Compiler*, EmuLoadedObject*, size_t, 369 size_t, void** out); 370 KitStatus emu_apply_reloc_bytes(Compiler*, RelocKind, u8* P_bytes, u64 S, i64 A, 371 u64 P); 372 KitStatus emu_tls_rebuild_modules(Compiler*, EmuProcess*); 373 KitStatus emu_tls_blocks_add(Compiler*, EmuTlsBlocks*, u32 module_id, u64 base, 374 u64 memsz); 375 KitStatus emu_tls_copy_module_image(EmuProcess*, const EmuTlsModule*, u64 base); 376 void emu_tls_destroy_process(Compiler*, EmuProcess*); 377 void emu_tls_destroy_blocks(Compiler*, EmuTlsBlocks*); 378 KitStatus emu_fault_deliver(EmuProcess*, EmuThread*, const EmuFaultEvent*, 379 u64* next_pc_out); 380 KitStatus emu_process_os_alloc(Compiler*, EmuProcess*, size_t size, 381 size_t align); 382 void emu_process_os_free(Compiler*, EmuProcess*, size_t size); 383 KitStatus emu_thread_os_alloc(Compiler*, EmuThread*, size_t size, size_t align); 384 void emu_thread_os_free(Compiler*, EmuThread*, size_t size); 385 KitStatus emu_addr_space_init(EmuAddrSpace*, Compiler*, u64 page_size); 386 void emu_addr_space_destroy(EmuAddrSpace*); 387 KitStatus emu_addr_space_map(EmuAddrSpace*, u64 va, u64 nbytes, u8 perms, 388 EmuMapKind kind); 389 KitStatus emu_addr_space_unmap(EmuAddrSpace*, u64 va, u64 nbytes); 390 KitStatus emu_addr_space_protect(EmuAddrSpace*, u64 va, u64 nbytes, u8 perms); 391 KitStatus emu_addr_space_find_gap(EmuAddrSpace*, u64 nbytes, u64 align, 392 u64 min_va, u64 max_va, u64* out); 393 KitStatus emu_addr_space_set_brk(EmuAddrSpace*, u64 requested, u64* actual_out); 394 KitStatus emu_addr_space_copy_in(EmuAddrSpace*, u64 va, const void* src, 395 u64 nbytes); 396 KitStatus emu_addr_space_set_perm(EmuAddrSpace*, u64 va, u64 nbytes, u8 perms); 397 u8* emu_addr_space_ptr(EmuAddrSpace*, u64 va, u64 nbytes, u8 need_perms); 398 u64 emu_addr_space_contig_len(EmuAddrSpace*, u64 va, u8 need_perms); 399 const EmuMemFault* emu_addr_space_last_fault(const EmuAddrSpace*); 400 void emu_addr_space_mark_translated(EmuAddrSpace*, u64 va, u64 nbytes); 401 void emu_addr_space_invalidate(EmuAddrSpace*, u64 va, u64 nbytes); 402 403 /* ---- CPU state -------------------------------------------------- */ 404 405 typedef enum EmuTrapReason { 406 EMU_TRAP_NONE = 0, 407 EMU_TRAP_EXIT, /* guest exit syscall; exit_code valid */ 408 EMU_TRAP_FAULT, /* unmapped access / decode failure */ 409 } EmuTrapReason; 410 411 EmuCPUState* emu_cpu_new_with_arch_state(Compiler*, KitArchKind, u64 initial_pc, 412 size_t arch_state_size, 413 size_t arch_state_align); 414 void emu_cpu_free(EmuCPUState*); 415 void* emu_cpu_arch_state(EmuCPUState*); 416 void emu_cpu_set_thread(EmuCPUState*, EmuThread*); 417 EmuThread* emu_cpu_thread(const EmuCPUState*); 418 u64 emu_cpu_pc(const EmuCPUState*); 419 void emu_cpu_set_pc(EmuCPUState*, u64); 420 EmuTrapReason emu_cpu_trap_reason(const EmuCPUState*); 421 int emu_cpu_exit_code(const EmuCPUState*); 422 void emu_cpu_attach_addr_space(EmuCPUState*, EmuAddrSpace*); 423 u8* emu_cpu_va_to_host_perm(EmuCPUState*, u64 va, u64 nbytes, u8 need_perms); 424 u64 emu_cpu_brk_cur(const EmuCPUState*); 425 u64 emu_cpu_brk_max(const EmuCPUState*); 426 void emu_cpu_set_brk_cur(EmuCPUState*, u64 v); 427 void emu_cpu_trap_exit(EmuCPUState*, int code); 428 void emu_cpu_trap_fault(EmuCPUState*); 429 void emu_cpu_clear_trap(EmuCPUState*); 430 EmuCPUState* emu_thread_cpu(EmuThread*); 431 432 /* The interned codegen pointer type representing EmuThread for JIT helper 433 * calls. CPUState remains arch-owned state below the thread. */ 434 KitCgTypeId emu_thread_type(Compiler*); 435 436 /* The function type `u64 (EmuThread*)` used for every lifted block. 437 * Returned interned. */ 438 KitCgTypeId emu_block_fn_type(Compiler*); 439 440 /* ---- Lifter ----------------------------------------------------- */ 441 442 typedef struct EmuLiftCtx { 443 Compiler* compiler; 444 KitArchKind arch; 445 KitCgTypeId thread_type; /* from emu_thread_type */ 446 KitCgTypeId block_fn_type; /* from emu_block_fn_type */ 447 KitCgSym block_sym; /* function symbol for this block */ 448 u64 guest_pc; /* PC of first instruction in the block */ 449 } EmuLiftCtx; 450 451 /* ---- Code cache ------------------------------------------------- */ 452 453 typedef struct EmuCodeCache EmuCodeCache; 454 455 EmuCodeCache* emu_cache_new(Compiler*); 456 void emu_cache_free(EmuCodeCache*); 457 void emu_cache_insert(EmuCodeCache*, u64 guest_pc, void* host_entry); 458 void* emu_cache_lookup(const EmuCodeCache*, u64 guest_pc); 459 460 /* ---- Code region (reserved VA) ---------------------------------- */ 461 /* PROT_NONE mmap that backs the linker's bump-allocated VA range. 462 * Pages are committed and flipped to RX after each link_resolve_extend 463 * lands new sections. The base address is fed to link_resolve_at as the 464 * image's runtime VA. The KitExecMem is borrowed (from the emu's 465 * JitHost) and must outlive the region. */ 466 typedef struct EmuCodeRegion EmuCodeRegion; 467 468 EmuCodeRegion* emu_code_region_new(Compiler*, const KitExecMem*, 469 size_t reserve_size); 470 void emu_code_region_free(EmuCodeRegion*); 471 uintptr_t emu_code_region_base(const EmuCodeRegion*); 472 size_t emu_code_region_size(const EmuCodeRegion*); 473 474 /* Commits and mprotects RX every page covering [base, end). `end` must 475 * lie inside the reserved range and must be monotonically non-decreasing 476 * across calls — the chaining invariant depends on previously committed 477 * pages remaining RX. */ 478 void emu_code_region_commit_rx_to(EmuCodeRegion*, uintptr_t end); 479 480 /* ---- Runtime helpers -------------------------------------------- */ 481 482 /* Names of the runtime helper symbols the lifter emits as undefined 483 * externs. The extern resolver maps each one to the host address of 484 * the matching helper. Kept centralized so decode/lift/runtime agree. */ 485 #define EMU_SYM_CPU_STATE "__emu_cpu_state" 486 #define EMU_SYM_LOAD8 "__emu_load8" 487 #define EMU_SYM_LOAD16 "__emu_load16" 488 #define EMU_SYM_LOAD32 "__emu_load32" 489 #define EMU_SYM_LOAD64 "__emu_load64" 490 #define EMU_SYM_LOAD8_CHECKED "__emu_load8_checked" 491 #define EMU_SYM_LOAD16_CHECKED "__emu_load16_checked" 492 #define EMU_SYM_LOAD32_CHECKED "__emu_load32_checked" 493 #define EMU_SYM_LOAD64_CHECKED "__emu_load64_checked" 494 #define EMU_SYM_STORE8 "__emu_store8" 495 #define EMU_SYM_STORE16 "__emu_store16" 496 #define EMU_SYM_STORE32 "__emu_store32" 497 #define EMU_SYM_STORE64 "__emu_store64" 498 #define EMU_SYM_SYSCALL "__emu_syscall" 499 #define EMU_SYM_DISPATCH "__emu_dispatch" 500 501 /* The block-symbol name format: emu_block_<hex_pc>. Kept short; the 502 * linker globals table only has to find it once per cold miss. */ 503 Sym emu_block_sym_name(Compiler*, u64 guest_pc); 504 505 /* External resolver passed to link_set_extern_resolver. `user` is 506 * the KitEmu*. Returns NULL for unrecognized names — the linker 507 * promotes that to a fatal undefined-symbol diagnostic. */ 508 void* emu_runtime_extern_resolver(void* user, KitSlice name); 509 510 /* Memory helpers; called from JITted blocks. The host process owns 511 * the guest AS, so loads/stores bounds-check against the EmuCPUState's 512 * mapped guest range and trap on miss (writing EMU_TRAP_FAULT into the 513 * CPU state and falling back to the dispatcher). */ 514 u8 emu_mem_load8(EmuThread*, u64 addr); 515 u16 emu_mem_load16(EmuThread*, u64 addr); 516 u32 emu_mem_load32(EmuThread*, u64 addr); 517 u64 emu_mem_load64(EmuThread*, u64 addr); 518 u64 emu_mem_load8_checked(EmuThread*, u64 addr, u64 fault_pc, u64 next_pc, 519 u64* value_out); 520 u64 emu_mem_load16_checked(EmuThread*, u64 addr, u64 fault_pc, u64 next_pc, 521 u64* value_out); 522 u64 emu_mem_load32_checked(EmuThread*, u64 addr, u64 fault_pc, u64 next_pc, 523 u64* value_out); 524 u64 emu_mem_load64_checked(EmuThread*, u64 addr, u64 fault_pc, u64 next_pc, 525 u64* value_out); 526 u64 emu_mem_store8(EmuThread*, u64 addr, u8, u64 fault_pc, u64 next_pc); 527 u64 emu_mem_store16(EmuThread*, u64 addr, u16, u64 fault_pc, u64 next_pc); 528 u64 emu_mem_store32(EmuThread*, u64 addr, u32, u64 fault_pc, u64 next_pc); 529 u64 emu_mem_store64(EmuThread*, u64 addr, u64, u64 fault_pc, u64 next_pc); 530 531 /* Reads syscall number / args from the guest registers, forwards to 532 * the host OS, and writes the return into the guest return register. */ 533 void emu_syscall(EmuThread*); 534 u64 emu_syscall_next(EmuThread*, u64 next_pc); 535 536 /* ---- Tracing ---------------------------------------------------- */ 537 538 void emu_trace_pc(Compiler*, u64 guest_pc); 539 void emu_trace_block(Compiler*, u64 guest_pc); 540 void emu_trace_insn(Compiler*, u64 guest_pc, const KitDecodedInsn*); 541 542 #endif