emu.h (16993B)
1 #ifndef KIT_EMU_INTERNAL_H 2 #define KIT_EMU_INTERNAL_H 3 4 /* Internal API for libkit's guest-ISA emulator. Public surface is 5 * kit_emu_* in <kit/emu.h>; the implementation in src/emu/emu.c 6 * composes the pieces declared here. See doc/EMU.md for design. 7 * 8 * Layering: emu.c owns KitEmu lifecycle and the translate/dispatch 9 * loop; per-ISA decoders/lifters live behind ArchImpl hooks, while CPUState 10 * synthesis, the JIT code cache and reserved-VA region, and the runtime helper 11 * trampolines each live behind one of the surfaces below. */ 12 13 #include <kit/cg.h> 14 #include <kit/emu.h> 15 #include <kit/jit.h> 16 #include <kit/os.h> 17 18 #include "arch/arch.h" 19 #include "core/core.h" 20 #include "obj/obj.h" 21 22 typedef struct LinkImage LinkImage; 23 typedef struct Linker Linker; 24 typedef struct EmuCPUState EmuCPUState; 25 typedef struct ObjFormatImpl ObjFormatImpl; 26 typedef struct EmuProcess EmuProcess; 27 typedef struct EmuThread EmuThread; 28 typedef struct EmuExternalBindings EmuExternalBindings; 29 typedef struct ObjFormatEmuOps ObjFormatEmuOps; 30 typedef struct EmuLoadedObject EmuLoadedObject; 31 typedef struct EmuDynNeededIter EmuDynNeededIter; 32 typedef struct EmuDynSymbol EmuDynSymbol; 33 typedef struct EmuDynRelocIter EmuDynRelocIter; 34 typedef struct EmuDynReloc EmuDynReloc; 35 typedef struct EmuObjectFormatData EmuObjectFormatData; 36 37 /* ---- Configuration knobs ---------------------------------------- */ 38 39 /* Bounded so the translator can stack-allocate the decoded instruction buffer. 40 */ 41 #define EMU_MAX_INSTS_PER_BLOCK 64u 42 43 /* Reserved JIT code region. emu_runtime mmap's PROT_NONE up front and 44 * commits pages as cold blocks land. Sized for v1 — chaining and the 45 * code cache assume host VAs of placed sections never move, so this 46 * region also never grows. */ 47 #define EMU_CODE_REGION_SIZE (128ull * 1024ull * 1024ull) 48 49 /* ---- Per-emu JIT host wiring ------------------------------------ */ 50 /* `host` is borrowed and must outlive the KitEmu. The setter remains for 51 * internal callers that construct an emu before wiring driver-owned JIT state. 52 */ 53 void emu_set_jit_host(KitEmu*, const KitJitHost*); 54 const KitJitHost* emu_get_jit_host(const KitEmu*); 55 56 /* ---- Guest executable image ------------------------------------- */ 57 58 typedef enum EmuMemPerm { 59 EMU_MEM_READ = 1u << 0, 60 EMU_MEM_WRITE = 1u << 1, 61 EMU_MEM_EXEC = 1u << 2, 62 } EmuMemPerm; 63 64 typedef enum EmuMapKind { 65 EMU_MAP_ANON, 66 EMU_MAP_FILE, 67 EMU_MAP_GUARD, 68 } EmuMapKind; 69 70 typedef enum EmuFaultKind { 71 EMU_FAULT_NONE = 0, 72 EMU_FAULT_UNMAPPED, 73 EMU_FAULT_PROT, 74 EMU_FAULT_EXEC_INVALIDATED, 75 } EmuFaultKind; 76 77 typedef struct EmuMemFault { 78 EmuFaultKind kind; 79 u64 addr; 80 u8 access; 81 } EmuMemFault; 82 83 typedef struct EmuMap { 84 u64 start; 85 u64 end; 86 u8 perms; 87 EmuMapKind kind; 88 u32 flags; 89 u64 generation; 90 u8* bytes; 91 u8* dirty_pages; 92 u8* translated_pages; 93 } EmuMap; 94 95 typedef struct EmuAddrSpace { 96 Compiler* compiler; 97 Heap* heap; 98 u64 page_size; 99 EmuMap* maps; 100 u32 nmaps; 101 u32 maps_cap; 102 u64 generation; 103 u64 brk_base; 104 u64 brk_cur; 105 u64 brk_max; 106 EmuMemFault last_fault; 107 } EmuAddrSpace; 108 109 typedef struct EmuDynamicImport { 110 KitSlice object_name; 111 KitSlice symbol_name; 112 u64 got_vaddr; 113 u64 thunk_vaddr; 114 KitEmuImportSignature signature; 115 } EmuDynamicImport; 116 117 typedef struct EmuImportBinding { 118 KitSlice object_name; 119 KitSlice symbol_name; 120 u64 got_vaddr; 121 u64 thunk_vaddr; 122 u64 resolved_guest_addr; 123 void* resolved_host_fn; 124 u32 flags; 125 KitEmuImportSignature signature; 126 } EmuImportBinding; 127 128 /* Max DT_NEEDED entries recorded per object; overflow is silently truncated by 129 * the ELF emu loader (src/obj/elf/emu_load.c). */ 130 #define EMU_MAX_NEEDED 16 131 132 typedef struct EmuObjectImports { 133 KitSlice needed[EMU_MAX_NEEDED]; 134 u32 nneeded; 135 } EmuObjectImports; 136 137 typedef struct EmuObjectInitFini { 138 u64 init; 139 u64 fini; 140 u64 init_array; 141 u64 init_arraysz; 142 u64 fini_array; 143 u64 fini_arraysz; 144 } EmuObjectInitFini; 145 146 struct EmuObjectFormatData { 147 void* data; 148 size_t size; 149 size_t align; 150 }; 151 152 typedef struct EmuObjectProcessInfo { 153 u64 headers_vaddr; 154 u64 header_entry_size; 155 u64 header_count; 156 KitSlice interpreter_path; 157 } EmuObjectProcessInfo; 158 159 typedef u32 EmuDynRelocClass; 160 #define EMU_DYN_RELOC_NONE 0u 161 #define EMU_DYN_RELOC_RELATIVE 1u 162 #define EMU_DYN_RELOC_SYMBOLIC 2u 163 #define EMU_DYN_RELOC_IMPORT_SLOT 3u 164 165 struct EmuDynNeededIter { 166 const EmuLoadedObject* object; 167 u32 index; 168 }; 169 170 struct EmuDynSymbol { 171 KitSlice name; 172 u64 value; 173 u64 size; 174 u64 index; 175 u32 flags; 176 int defined; 177 }; 178 179 struct EmuDynReloc { 180 u64 patch_addr; 181 u64 symbol_index; 182 u64 wire_type; 183 i64 addend; 184 u32 width; 185 }; 186 187 typedef u32 EmuDynRelocTableKind; 188 #define EMU_DYN_RELOC_TABLE_MAIN 0u 189 #define EMU_DYN_RELOC_TABLE_PLT 1u 190 191 struct EmuDynRelocIter { 192 const EmuLoadedObject* object; 193 EmuDynRelocTableKind table; 194 u64 cursor; 195 }; 196 197 typedef struct EmuTlsModule { 198 u64 image_vaddr; 199 u64 filesz; 200 u64 memsz; 201 u64 align; 202 u32 module_id; 203 } EmuTlsModule; 204 205 typedef struct EmuLoadedObject { 206 KitSlice name; 207 KitSlice soname; 208 u64 load_bias; 209 u64 map_start; 210 u64 map_end; 211 EmuObjectImports imports; 212 EmuObjectInitFini init_fini; 213 EmuObjectFormatData format; 214 EmuTlsModule tls; 215 u32 flags; 216 } EmuLoadedObject; 217 218 typedef struct EmuLinkMap { 219 EmuLoadedObject* objects; 220 u32 nobjects; 221 u32 objects_cap; 222 u32 main_object; 223 u32 global_scope_head; 224 } EmuLinkMap; 225 226 typedef struct EmuDlPolicy { 227 u32 flags; 228 u32 global_scope_head; 229 u32 search_path_head; 230 } EmuDlPolicy; 231 232 typedef struct EmuTlsState { 233 EmuTlsModule* modules; 234 u32 nmodules; 235 u32 modules_cap; 236 u64 static_size; 237 u64 static_align; 238 } EmuTlsState; 239 240 typedef struct EmuTlsBlock { 241 u32 module_id; 242 u64 base; 243 u64 memsz; 244 } EmuTlsBlock; 245 246 typedef struct EmuTlsBlocks { 247 EmuTlsBlock* blocks; 248 u32 nblocks; 249 u32 blocks_cap; 250 } EmuTlsBlocks; 251 252 typedef struct EmuLoadedImage { 253 EmuAddrSpace addr_space; 254 u64 entry_pc; /* guest VA of the program entry point */ 255 u64 initial_sp; /* guest VA of the initial stack pointer */ 256 EmuObjectProcessInfo process_info; 257 EmuDynamicImport* imports; 258 u32 nimports; 259 EmuImportBinding* import_bindings; 260 u32 nimport_bindings; 261 u32 import_bindings_cap; 262 u64 import_thunk_base; 263 u64 import_thunk_size; 264 u64 import_thunk_next; 265 EmuLinkMap link_map; 266 } EmuLoadedImage; 267 268 typedef struct EmuLoadOptions { 269 KitSlice name; 270 KitSlice bytes; 271 KitTargetSpec guest_target; 272 const char* const* argv; 273 const char* const* envp; 274 const KitOsImpl* os; 275 const EmuExternalBindings* bindings; 276 EmuProcess* process; 277 } EmuLoadOptions; 278 279 typedef struct EmuSyscallRequest { 280 u64 number; 281 u64 args[6]; 282 } EmuSyscallRequest; 283 284 typedef struct EmuSyscallResult { 285 i64 result; 286 i32 guest_errno; 287 u32 flags; 288 } EmuSyscallResult; 289 290 #define EMU_SYSCALL_RESULT_SKIP_ENCODE 1u 291 292 struct EmuExternalBindings { 293 KitStatus (*syscall)(void* user, EmuProcess*, EmuThread*, 294 const EmuSyscallRequest*, EmuSyscallResult* out); 295 KitStatus (*resolve_import)(void* user, EmuProcess*, const EmuDynamicImport*, 296 KitEmuResolvedImport* out); 297 KitStatus (*resolve_object)(void* user, EmuProcess*, KitSlice object_name, 298 KitSlice* out_bytes); 299 void* user; 300 }; 301 302 typedef struct EmuFaultEvent { 303 EmuFaultKind kind; 304 u64 addr; 305 u64 pc; 306 u64 next_pc; 307 u8 access; 308 } EmuFaultEvent; 309 310 struct EmuProcess { 311 Compiler* compiler; 312 KitTargetSpec guest_target; 313 const ObjFormatImpl* obj_format; 314 const ArchImpl* arch; 315 const KitOsImpl* os; 316 void* os_private; 317 EmuLoadedImage image; 318 EmuExternalBindings bindings; 319 EmuDlPolicy dl_policy; 320 EmuTlsState tls_state; 321 }; 322 323 struct EmuThread { 324 EmuProcess* process; 325 EmuCPUState* cpu; 326 void* os_private; 327 }; 328 329 #define EMU_OS_MAP_MMAP 1u 330 #define EMU_OS_MAP_DL_THUNKS 2u 331 #define EMU_OS_MAP_TLS 3u 332 333 int emu_loaded_image_attach_cpu(EmuCPUState*, EmuLoadedImage*); 334 void emu_unload_image(Compiler*, EmuLoadedImage*); 335 KitStatus emu_dl_init_process(Compiler*, EmuProcess*); 336 KitStatus emu_dl_load_dependencies_and_relocate(Compiler*, EmuProcess*, 337 const EmuLoadOptions*, 338 const ObjFormatEmuOps*); 339 KitStatus emu_dl_lookup_symbol(EmuProcess*, KitSlice symbol, u64* out_addr); 340 KitStatus emu_dl_resolve_import_thunk(EmuProcess*, u64 target, 341 EmuImportBinding** out); 342 KitStatus emu_call_host_import(EmuThread*, EmuImportBinding*, const u64* args, 343 u32 nargs, u64* result_out); 344 KitStatus emu_object_format_data_alloc(Compiler*, EmuLoadedObject*, size_t, 345 size_t, void** out); 346 KitStatus emu_apply_reloc_bytes(Compiler*, RelocKind, u8* P_bytes, u64 S, i64 A, 347 u64 P); 348 KitStatus emu_tls_rebuild_modules(Compiler*, EmuProcess*); 349 KitStatus emu_tls_blocks_add(Compiler*, EmuTlsBlocks*, u32 module_id, u64 base, 350 u64 memsz); 351 KitStatus emu_tls_copy_module_image(EmuProcess*, const EmuTlsModule*, u64 base); 352 void emu_tls_destroy_process(Compiler*, EmuProcess*); 353 void emu_tls_destroy_blocks(Compiler*, EmuTlsBlocks*); 354 KitStatus emu_fault_deliver(EmuProcess*, EmuThread*, const EmuFaultEvent*, 355 u64* next_pc_out); 356 KitStatus emu_process_os_alloc(Compiler*, EmuProcess*, size_t size, 357 size_t align); 358 void emu_process_os_free(Compiler*, EmuProcess*, size_t size); 359 KitStatus emu_thread_os_alloc(Compiler*, EmuThread*, size_t size, size_t align); 360 void emu_thread_os_free(Compiler*, EmuThread*, size_t size); 361 KitStatus emu_addr_space_init(EmuAddrSpace*, Compiler*, u64 page_size); 362 void emu_addr_space_destroy(EmuAddrSpace*); 363 KitStatus emu_addr_space_map(EmuAddrSpace*, u64 va, u64 nbytes, u8 perms, 364 EmuMapKind kind); 365 KitStatus emu_addr_space_unmap(EmuAddrSpace*, u64 va, u64 nbytes); 366 KitStatus emu_addr_space_protect(EmuAddrSpace*, u64 va, u64 nbytes, u8 perms); 367 KitStatus emu_addr_space_find_gap(EmuAddrSpace*, u64 nbytes, u64 align, 368 u64 min_va, u64 max_va, u64* out); 369 KitStatus emu_addr_space_set_brk(EmuAddrSpace*, u64 requested, u64* actual_out); 370 KitStatus emu_addr_space_copy_in(EmuAddrSpace*, u64 va, const void* src, 371 u64 nbytes); 372 u8* emu_addr_space_ptr(EmuAddrSpace*, u64 va, u64 nbytes, u8 need_perms); 373 u64 emu_addr_space_contig_len(EmuAddrSpace*, u64 va, u8 need_perms); 374 const EmuMemFault* emu_addr_space_last_fault(const EmuAddrSpace*); 375 void emu_addr_space_mark_translated(EmuAddrSpace*, u64 va, u64 nbytes); 376 void emu_addr_space_invalidate(EmuAddrSpace*, u64 va, u64 nbytes); 377 378 /* ---- CPU state -------------------------------------------------- */ 379 380 typedef enum EmuTrapReason { 381 EMU_TRAP_NONE = 0, 382 EMU_TRAP_EXIT, /* guest exit syscall; exit_code valid */ 383 EMU_TRAP_FAULT, /* unmapped access / decode failure */ 384 } EmuTrapReason; 385 386 EmuCPUState* emu_cpu_new_with_arch_state(Compiler*, KitArchKind, u64 initial_pc, 387 size_t arch_state_size, 388 size_t arch_state_align); 389 void emu_cpu_free(EmuCPUState*); 390 void* emu_cpu_arch_state(EmuCPUState*); 391 void emu_cpu_set_thread(EmuCPUState*, EmuThread*); 392 EmuThread* emu_cpu_thread(const EmuCPUState*); 393 u64 emu_cpu_pc(const EmuCPUState*); 394 void emu_cpu_set_pc(EmuCPUState*, u64); 395 EmuTrapReason emu_cpu_trap_reason(const EmuCPUState*); 396 int emu_cpu_exit_code(const EmuCPUState*); 397 void emu_cpu_attach_addr_space(EmuCPUState*, EmuAddrSpace*); 398 u8* emu_cpu_va_to_host_perm(EmuCPUState*, u64 va, u64 nbytes, u8 need_perms); 399 u64 emu_cpu_brk_cur(const EmuCPUState*); 400 u64 emu_cpu_brk_max(const EmuCPUState*); 401 void emu_cpu_set_brk_cur(EmuCPUState*, u64 v); 402 void emu_cpu_trap_exit(EmuCPUState*, int code); 403 void emu_cpu_trap_fault(EmuCPUState*); 404 void emu_cpu_clear_trap(EmuCPUState*); 405 EmuCPUState* emu_thread_cpu(EmuThread*); 406 407 /* The interned codegen pointer type representing EmuThread for JIT helper 408 * calls. CPUState remains arch-owned state below the thread. */ 409 KitCgTypeId emu_thread_type(Compiler*); 410 411 /* The function type `u64 (EmuThread*)` used for every lifted block. 412 * Returned interned. */ 413 KitCgTypeId emu_block_fn_type(Compiler*); 414 415 /* ---- Lifter ----------------------------------------------------- */ 416 417 typedef struct EmuLiftCtx { 418 Compiler* compiler; 419 KitArchKind arch; 420 KitCgTypeId thread_type; /* from emu_thread_type */ 421 KitCgTypeId block_fn_type; /* from emu_block_fn_type */ 422 KitCgSym block_sym; /* function symbol for this block */ 423 u64 guest_pc; /* PC of first instruction in the block */ 424 } EmuLiftCtx; 425 426 /* ---- Code cache ------------------------------------------------- */ 427 428 typedef struct EmuCodeCache EmuCodeCache; 429 430 EmuCodeCache* emu_cache_new(Compiler*); 431 void emu_cache_free(EmuCodeCache*); 432 void emu_cache_insert(EmuCodeCache*, u64 guest_pc, void* host_entry); 433 void* emu_cache_lookup(const EmuCodeCache*, u64 guest_pc); 434 435 /* ---- Code region (reserved VA) ---------------------------------- */ 436 /* PROT_NONE mmap that backs the linker's bump-allocated VA range. 437 * Pages are committed and flipped to RX after each link_resolve_extend 438 * lands new sections. The base address is fed to link_resolve_at as the 439 * image's runtime VA. The KitExecMem is borrowed (from the emu's 440 * JitHost) and must outlive the region. */ 441 typedef struct EmuCodeRegion EmuCodeRegion; 442 443 EmuCodeRegion* emu_code_region_new(Compiler*, const KitExecMem*, 444 size_t reserve_size); 445 void emu_code_region_free(EmuCodeRegion*); 446 uintptr_t emu_code_region_base(const EmuCodeRegion*); 447 size_t emu_code_region_size(const EmuCodeRegion*); 448 449 /* Commits and mprotects RX every page covering [base, end). `end` must 450 * lie inside the reserved range and must be monotonically non-decreasing 451 * across calls — the chaining invariant depends on previously committed 452 * pages remaining RX. */ 453 void emu_code_region_commit_rx_to(EmuCodeRegion*, uintptr_t end); 454 455 /* ---- Runtime helpers -------------------------------------------- */ 456 457 /* Names of the runtime helper symbols the lifter emits as undefined 458 * externs. The extern resolver maps each one to the host address of 459 * the matching helper. Kept centralized so decode/lift/runtime agree. */ 460 #define EMU_SYM_CPU_STATE "__emu_cpu_state" 461 #define EMU_SYM_LOAD8 "__emu_load8" 462 #define EMU_SYM_LOAD16 "__emu_load16" 463 #define EMU_SYM_LOAD32 "__emu_load32" 464 #define EMU_SYM_LOAD64 "__emu_load64" 465 #define EMU_SYM_LOAD8_CHECKED "__emu_load8_checked" 466 #define EMU_SYM_LOAD16_CHECKED "__emu_load16_checked" 467 #define EMU_SYM_LOAD32_CHECKED "__emu_load32_checked" 468 #define EMU_SYM_LOAD64_CHECKED "__emu_load64_checked" 469 #define EMU_SYM_STORE8 "__emu_store8" 470 #define EMU_SYM_STORE16 "__emu_store16" 471 #define EMU_SYM_STORE32 "__emu_store32" 472 #define EMU_SYM_STORE64 "__emu_store64" 473 #define EMU_SYM_SYSCALL "__emu_syscall" 474 #define EMU_SYM_DISPATCH "__emu_dispatch" 475 476 /* The block-symbol name format: emu_block_<hex_pc>. Kept short; the 477 * linker globals table only has to find it once per cold miss. */ 478 Sym emu_block_sym_name(Compiler*, u64 guest_pc); 479 480 /* External resolver passed to link_set_extern_resolver. `user` is 481 * the KitEmu*. Returns NULL for unrecognized names — the linker 482 * promotes that to a fatal undefined-symbol diagnostic. */ 483 void* emu_runtime_extern_resolver(void* user, KitSlice name); 484 485 /* Memory helpers; called from JITted blocks. The host process owns 486 * the guest AS, so loads/stores bounds-check against the EmuCPUState's 487 * mapped guest range and trap on miss (writing EMU_TRAP_FAULT into the 488 * CPU state and falling back to the dispatcher). */ 489 u8 emu_mem_load8(EmuThread*, u64 addr); 490 u16 emu_mem_load16(EmuThread*, u64 addr); 491 u32 emu_mem_load32(EmuThread*, u64 addr); 492 u64 emu_mem_load64(EmuThread*, u64 addr); 493 u64 emu_mem_load8_checked(EmuThread*, u64 addr, u64 fault_pc, u64 next_pc, 494 u64* value_out); 495 u64 emu_mem_load16_checked(EmuThread*, u64 addr, u64 fault_pc, u64 next_pc, 496 u64* value_out); 497 u64 emu_mem_load32_checked(EmuThread*, u64 addr, u64 fault_pc, u64 next_pc, 498 u64* value_out); 499 u64 emu_mem_load64_checked(EmuThread*, u64 addr, u64 fault_pc, u64 next_pc, 500 u64* value_out); 501 u64 emu_mem_store8(EmuThread*, u64 addr, u8, u64 fault_pc, u64 next_pc); 502 u64 emu_mem_store16(EmuThread*, u64 addr, u16, u64 fault_pc, u64 next_pc); 503 u64 emu_mem_store32(EmuThread*, u64 addr, u32, u64 fault_pc, u64 next_pc); 504 u64 emu_mem_store64(EmuThread*, u64 addr, u64, u64 fault_pc, u64 next_pc); 505 506 /* Reads syscall number / args from the guest registers, forwards to 507 * the host OS, and writes the return into the guest return register. */ 508 void emu_syscall(EmuThread*); 509 u64 emu_syscall_next(EmuThread*, u64 next_pc); 510 511 /* ---- Tracing ---------------------------------------------------- */ 512 513 void emu_trace_pc(Compiler*, u64 guest_pc); 514 void emu_trace_block(Compiler*, u64 guest_pc); 515 void emu_trace_insn(Compiler*, u64 guest_pc, const KitDecodedInsn*); 516 517 #endif