/* * x86_64 application-processor startup trampoline * * a SIPI starts an AP in 16-bit real mode at a low-memory page chosen by the * BSP. this trampoline transitions through protected mode into long mode and * finally calls the AP's C entry point. * * the BSP copies this image below 1 MiB and patches the fields marked below. * * the trampoline is position-independent with respect to its copied page: * %bx/%ebx/%rbx holds the runtime page base, and references within the page * use page-relative displacements. do not replace them with link-time absolute * addresses. * * raw byte encodings are used where the assembler would otherwise emit * absolute ADDR32 relocations. after ANY change you make, verify: * * - llvm-readelf -r shows no relocation for ap_lm or far-jump targets. * - BSP-patched slot offsets still match this layout. */ .text /* * selectors from the flat GDT at the bottom of this file. kept as names so a * change to the table layout cannot drift away from the far jumps that use it. */ .set GDT_SEL_CODE32, 0x08 .set GDT_SEL_DATA32, 0x10 .set GDT_SEL_CODE64, 0x18 .set GDT_SEL_TSS, 0x20 /* model-specific registers, named because the raw numbers are untraceable */ .set MSR_EFER, 0xC0000080 .set MSR_KERNEL_GS_BASE, 0xC0000102 .set CR0_PE, (1 << 0) .set CR0_PG, (1 << 31) .set CR4_PAE, (1 << 5) /* EFER.LME: sets LMA once CR0.PG is also on */ .set EFER_LME, (1 << 8) /* * bump the progress marker, so a BSP sampling ap_stage can tell how far an AP * got. one macro per address size because the displacement width is encoded * separately from the opcode (0x87/0x83 are incb with different ModRM). * the base register is not encoded here either, so the disp32 form serves the * 64-bit section too: incb disp32(%ebx) and disp32(%rbx) are the same bytes. */ .macro AP_BUMP_STAGE16 .byte 0xfe, 0x87 /* incb disp16(%bx) */ .word ap_stage - ap_trampoline .endm .macro AP_BUMP_STAGE .byte 0xfe, 0x83 /* incb disp32(%ebx) */ .long ap_stage - ap_trampoline .endm .globl ap_trampoline ap_trampoline: .code16 /* entry is real mode, paging off, interrupts masked. */ .byte 0xfa /* cli */ .byte 0xfc /* cld */ .byte 0x31, 0xc0 /* xorw %ax, %ax */ .byte 0x8e, 0xd8 /* movw %ax, %ds */ .byte 0x8e, 0xc0 /* movw %ax, %es */ .byte 0x8e, 0xd0 /* movw %ax, %ss */ /* * load the runtime page base into %bx. * * INIT leaves SP=0, so a call/pop position-discovery trick is unsafe here. * the BSP already knows the copied page address, so it patches that value * directly into ap_rm_base_slot and patches this instruction's absolute * 16-bit operand to point at the slot. * * slots embedded in the instruction stream must be jumped over. */ jmp 1f .globl ap_rm_base_slot ap_rm_base_slot: .word 0 /* patched: trampoline page base */ 1: .byte 0x8b, 0x1e /* movw abs16, %bx */ .globl ap_rm_base_addr16 ap_rm_base_addr16: .word 0 /* patched: address of ap_rm_base_slot */ /* * install a real-mode stack within this page * * SS=0, so SP must contain the absolute low-memory address of the stack * top. the displacement and loaded value are distinct words. * * ModRM is 0x87 (mod=10, reg=SP, rm=111) so the disp16 is present. 0x27 * drops the displacement and loads SP from the code bytes instead. */ jmp 1f .globl ap_rm_sp_slot ap_rm_sp_slot: .word 0 /* patched: page base + stack-top offset */ 1: .byte 0x8b, 0x87 /* movw disp16(%bx), %sp */ .globl ap_rm_sp_disp16 ap_rm_sp_disp16: .word ap_rm_sp_slot - ap_trampoline /* * load our GDT pointer. * * 0x66 selects the 32-bit LGDT operand format: limit:2, base:4. * addressing remains 16-bit. * * ModRM 0x97: * mod=10 disp16 * reg=010 LGDT (/2) * rm=111 [BX] * * rm=011 would be [BP+DI], not [BX], and reg=001 is SIDT rather than LGDT. */ .byte 0x66 .byte 0x0f, 0x01, 0x97 /* lgdt disp16(%bx) */ .word ap_gdt_pm_ptr - ap_trampoline AP_BUMP_STAGE16 /* * enter protected mode before the far jump. without CR0.PE, the jump uses * real-mode CS semantics instead of loading selector 0x08 from the GDT. */ .byte 0x66, 0x0f, 0x20, 0xc0 /* mov %cr0, %eax */ .byte 0x66, 0x83, 0xc8, 0x01 /* or $CR0_PE, %eax */ .byte 0x66, 0x0f, 0x22, 0xc0 /* mov %eax, %cr0 */ AP_BUMP_STAGE16 /* * far jump to flat 32-bit code. * * since the GDT code-segment base is zero, the jump offset must be the * absolute runtime address, not a within-page offset. the BSP patches it. */ .byte 0xea .globl ap_pm32_off16 ap_pm32_off16: .word 0 /* patched: page base + ap_pm32 offset */ .word GDT_SEL_CODE32 .code32 .globl ap_pm32 ap_pm32: /* load flat data selectors. */ movw $GDT_SEL_DATA32, %ax movw %ax, %ds movw %ax, %es movw %ax, %ss /* * reload the GDT. required because the far jump out of this block loads * CS with a 64-bit (L=1) descriptor. * * same pointer record as the real-mode load: the memory layout follows the * operand size, not the execution mode. here that size is 32-bit. * * ModRM 0x93: mod=10, reg=010 (LGDT), rm=011 ([EBX]). */ .byte 0x0f, 0x01, 0x93 /* lgdt disp32(%ebx) */ .long ap_gdt_pm_ptr - ap_trampoline AP_BUMP_STAGE /* * CR4.PAE, required for the 4-level tables long mode uses. * * CR4 is zeroed rather than OR-ed into %ecx: INIT leaves %ecx with * whatever the delivery path last used, and a stray CR4.VMXE/SMXE bit puts * QEMU into nested virtualisation and spins it on the next CR write. */ xorl %eax, %eax movl %eax, %cr4 orl $CR4_PAE, %eax movl %eax, %cr4 AP_BUMP_STAGE movl $MSR_EFER, %ecx rdmsr AP_BUMP_STAGE orl $EFER_LME, %eax wrmsr AP_BUMP_STAGE /* * load CR3 with the PML4 the BSP built. paging is still off, so this does * not change the translation regime yet. */ .byte 0x8b, 0x83 /* movl disp32(%ebx), %eax */ .long ap_cr3_off32 - ap_trampoline movl %eax, %cr3 AP_BUMP_STAGE /* * enable paging, which is what actually turns long mode on: EFER.LMA is * only set once LME and CR0.PG are both set. * * CR0 is read-modify-written rather than zeroed, because zeroing would * briefly drop PE. the mask clears CR0.NW/CD, which firmware leaves set. */ movl %eax, %cr0 /* read CR0 */ andl $0x9fffffff, %eax /* clear NW (29), CD (30) */ orl $(CR0_PG | CR0_PE), %eax movl %eax, %cr0 /* write CR0 */ AP_BUMP_STAGE /* * CR4.PSE is unnecessary in long mode (PAE already provides 2 MiB pages), * and CR4.FSGSBASE is left clear because qemu64 does not advertise it, * which is why per-CPU state below goes through swapgs. */ /* * far jump to the long-mode (L=1) segment. same absolute-offset rule as * the 16-bit far jump above ; the BSP patches this one with the page base. */ .byte 0xea .globl ap_lm_off32 ap_lm_off32: .long 0 /* patched: page base + ap_lm offset */ .word GDT_SEL_CODE64 .code64 .globl ap_lm ap_lm: /* * long mode, paging live on the identity map the BSP built. * * select the AP's per-CPU area via IA32_KERNEL_GS_BASE + swapgs. GS.base * is still 0 on a freshly INIT'd AP, so the swap cannot corrupt anything. */ movl $MSR_KERNEL_GS_BASE, %ecx .byte 0x48, 0x8b, 0x83 /* movq disp32(%rbx), %rax */ .long ap_percpu - ap_trampoline /* * WRMSR writes (RDX << 32) | RAX, so the HIGH half is what needs zeroing. * clearing %eax instead puts the pointer at bit 32 and yields a * non-canonical GS base that faults in C much later. */ xorl %edx, %edx wrmsr swapgs AP_BUMP_STAGE /* * load the AP's own TSS before the IDT. * * ORDER MATTERS: the IDT below names IST stacks, and an IST gate raises * #GP (which itself faults, giving a triple fault and a reset) unless the * processor already has a TSS loaded. the descriptor lives in this AP's own * GDT at GDT_SEL_TSS and its base was patched per AP, because the TSS holds * that AP's IST stack pointers and cannot be shared. * * ModRM 0x9b here is reg=011 (LTR /3), the same addressing as the LIDT. */ .byte 0x0f, 0x00, 0x9b, 0x83 /* ltr disp32(%rbx) */ .long ap_tss_sel - ap_trampoline AP_BUMP_STAGE /* * load the AP IDT the BSP built. * * separate from the BSP's table because its gates select 0x08, which is a * 32-bit code segment ; a long-mode interrupt gate must name a 64-bit one. * the IST entries the BSP uses ARE mirrored here now that the AP has a TSS, * so a #DF on an AP lands on a dedicated stack instead of a dead one. * * ModRM 0x9b: mod=10, reg=011 (LIDT), rm=011 ([RBX]). */ .byte 0x0f, 0x01, 0x9b, 0x83 /* lidt disp32(%rbx) */ .long ap_idt_pm_ptr - ap_trampoline AP_BUMP_STAGE /* * move to this AP's own stack, leaving the firmware stack behind. * * ModRM is 0xa3 (reg=rsp), not 0x83 (reg=rax) which silently discards the * value and leaves RSP at 0. */ .byte 0x48, 0x8b, 0xa3 /* movq disp32(%rbx), %rsp */ .long ap_stack_top - ap_trampoline xorl %ebp, %ebp /* terminate frame-pointer chains */ AP_BUMP_STAGE /* call the AP entry point, patched by the BSP. */ .byte 0xff, 0x93 /* call *disp32(%rbx) */ .long ap_entry - ap_trampoline /* park if it ever returns. */ 1: hlt jmp 1b /* * GDT pointer record, read by both LGDTs above. LGDT loads a (limit, base) * pair, not the descriptor table, so this is kept apart from the table itself. * the BSP fills both fields ; the base is only known at run time. */ .balign 8 .globl ap_gdt_ptr ap_gdt_ptr: .globl ap_gdt_pm_ptr ap_gdt_pm_ptr: .word 0 /* patched: GDTR limit */ .long 0 /* patched: GDTR base */ .word 0 /* pad */ /* * IDT pointer record, read by the LIDT in ap_lm. limit then base, same as the * GDT record. both fields are absolute addresses: the table lives in the * kernel image rather than in this page. */ .balign 8 .globl ap_idt_pm_ptr ap_idt_pm_ptr: .word 0 /* patched: IDT limit */ .long 0 /* patched: IDT base */ .word 0 /* pad */ /* * BSP-patched slots. each holds an absolute value, never a within-page offset: * reading the offset where an address belongs installs a small integer as a * stack pointer or function pointer. ap_cr3_off32 is a slot for the same * reason, since CR3 names a page table outside this page. */ .balign 8 .globl ap_cr3_off32 ap_cr3_off32: .quad 0 /* patched: absolute physical address of the PML4 */ .globl ap_percpu ap_percpu: .quad 0 /* patched: &percpu[slot] */ .globl ap_stack_top ap_stack_top: .quad 0 /* patched: top of this AP's stack */ .globl ap_entry ap_entry: .quad 0 /* patched: ap_main() */ .globl ap_slot ap_slot: .quad 0 /* patched: dense per-CPU slot for this AP */ .globl ap_tss_sel ap_tss_sel: .word GDT_SEL_TSS /* selector, read by the LTR above */ .quad 0 /* * progress marker, bumped at each step above so the BSP can report how far an * AP got. the page is reloaded before every AP, so the value always belongs to * the AP being started. debug aid ; the BSP only reads it. */ .globl ap_stage ap_stage: .byte 0 .space 3 /* * minimal flat GDT: null, 32-bit code, 32-bit data, 64-bit code. loaded by the * real-mode prologue and again before the long-mode far jump, which needs the * L=1 descriptor at selector 0x18. * * a selector is the descriptor's byte offset from the table base, so the .set * names above and the layout below have to agree. the assembler resolves * `.if` at build time, so a mismatch fails the build rather than the boot. * the checks compare label differences because `.if` requires an absolute * expression, which `label + constant` is not. */ .balign 8 .globl ap_gdt_sym ap_gdt_sym: ap_gdt: .quad 0x0000000000000000 /* null */ ap_gdt_code32: .quad 0x00cf9a000000ffff /* 0x08: 32-bit code, base 0, limit 4G */ ap_gdt_data32: .quad 0x00cf92000000ffff /* 0x10: 32-bit data, base 0, limit 4G */ ap_gdt_code64: .quad 0x00af9a000000ffff /* 0x18: 64-bit code, base 0, limit 4G */ .globl ap_gdt_tss ap_gdt_tss: /* * 64-bit available TSS descriptor, 16 bytes, base patched per AP by * patch_trampoline(): the TSS holds that AP's IST stack pointers, so it * cannot be a fixed address. */ .quad 0 .quad 0 .globl ap_gdt_end ap_gdt_end: .if (ap_gdt_code32 - ap_gdt) != GDT_SEL_CODE32 .error "ap_gdt: 32-bit code descriptor is not at GDT_SEL_CODE32" .endif .if (ap_gdt_data32 - ap_gdt) != GDT_SEL_DATA32 .error "ap_gdt: 32-bit data descriptor is not at GDT_SEL_DATA32" .endif .if (ap_gdt_code64 - ap_gdt) != GDT_SEL_CODE64 .error "ap_gdt: 64-bit code descriptor is not at GDT_SEL_CODE64" .endif .if (ap_gdt_tss - ap_gdt) != GDT_SEL_TSS .error "ap_gdt: TSS descriptor is not at GDT_SEL_TSS" .endif .if (ap_gdt_end - ap_gdt) != 48 .error "ap_gdt: table is not 5 descriptors; a selector above is wrong" .endif /* * top of the 16-bit real-mode stack. SS=0 here, so this is an offset within * the page ; keep it above all code and patched fields. */ .balign 16 ap_rm_stack: .space 64 .globl ap_rm_stack_top ap_rm_stack_top: .balign 8 .globl ap_trampoline_end ap_trampoline_end: /* size of the image, for the BSP to copy. */ .globl ap_trampoline_bytes ap_trampoline_bytes: .quad ap_trampoline_end - ap_trampoline