λ

furix

cute os

settings

arch/x86_64/ap_boot.S

14195 bytes raw

/*
    * x86_64 application-processor startup trampoline
    *
    * a SIPI starts an AP in 16-bit real mode at a low-memory page chosen by the
    * BSP. this trampoline transitions through protected mode into long mode and
    * finally calls the AP's C entry point.
    *
    * the BSP copies this image below 1 MiB and patches the fields marked below.
    *
    * the trampoline is position-independent with respect to its copied page:
    * %bx/%ebx/%rbx holds the runtime page base, and references within the page
    * use page-relative displacements. do not replace them with link-time absolute
    * addresses.
    *
    * raw byte encodings are used where the assembler would otherwise emit
    * absolute ADDR32 relocations. after ANY change you make, verify:
    *
    *   - llvm-readelf -r shows no relocation for ap_lm or far-jump targets.
    *   - BSP-patched slot offsets still match this layout.
    */

.text

/*
    * selectors from the flat GDT at the bottom of this file. kept as names so a
    * change to the table layout cannot drift away from the far jumps that use it.
    */
.set GDT_SEL_CODE32, 0x08
.set GDT_SEL_DATA32, 0x10
.set GDT_SEL_CODE64, 0x18
.set GDT_SEL_TSS, 0x20

/* model-specific registers, named because the raw numbers are untraceable */
.set MSR_EFER, 0xC0000080
.set MSR_KERNEL_GS_BASE, 0xC0000102

.set CR0_PE, (1 << 0)
.set CR0_PG, (1 << 31)
.set CR4_PAE, (1 << 5)

/* EFER.LME: sets LMA once CR0.PG is also on */

.set EFER_LME, (1 << 8)

/*
    * bump the progress marker, so a BSP sampling ap_stage can tell how far an AP
    * got. one macro per address size because the displacement width is encoded
    * separately from the opcode (0x87/0x83 are incb with different ModRM).
    * the base register is not encoded here either, so the disp32 form serves the
    * 64-bit section too: incb disp32(%ebx) and disp32(%rbx) are the same bytes.
    */
.macro AP_BUMP_STAGE16
    .byte 0xfe, 0x87 /* incb disp16(%bx) */
    .word ap_stage - ap_trampoline
.endm

.macro AP_BUMP_STAGE
    .byte 0xfe, 0x83 /* incb disp32(%ebx) */
    .long ap_stage - ap_trampoline
.endm

.globl ap_trampoline


ap_trampoline:

    .code16

    /* entry is real mode, paging off, interrupts masked. */
    .byte 0xfa /* cli */
    .byte 0xfc /* cld */

    .byte 0x31, 0xc0 /* xorw %ax, %ax */
    .byte 0x8e, 0xd8 /* movw %ax, %ds */
    .byte 0x8e, 0xc0 /* movw %ax, %es */
    .byte 0x8e, 0xd0 /* movw %ax, %ss */

    /*
    * load the runtime page base into %bx.
    *
    * INIT leaves SP=0, so a call/pop position-discovery trick is unsafe here.
    * the BSP already knows the copied page address, so it patches that value
    * directly into ap_rm_base_slot and patches this instruction's absolute
    * 16-bit operand to point at the slot.
    *
    * slots embedded in the instruction stream must be jumped over.
    */
    jmp 1f
.globl ap_rm_base_slot


ap_rm_base_slot:

    .word 0 /* patched: trampoline page base */


1:

    .byte 0x8b, 0x1e /* movw abs16, %bx */
.globl ap_rm_base_addr16


ap_rm_base_addr16:

    .word 0 /* patched: address of ap_rm_base_slot */

    /*
    * install a real-mode stack within this page
    *
    * SS=0, so SP must contain the absolute low-memory address of the stack
    * top. the displacement and loaded value are distinct words.
    *
    * ModRM is 0x87 (mod=10, reg=SP, rm=111) so the disp16 is present. 0x27
    * drops the displacement and loads SP from the code bytes instead.
    */
    jmp 1f
.globl ap_rm_sp_slot


ap_rm_sp_slot:

    .word 0 /* patched: page base + stack-top offset */


1:

    .byte 0x8b, 0x87 /* movw disp16(%bx), %sp */
.globl ap_rm_sp_disp16


ap_rm_sp_disp16:

    .word ap_rm_sp_slot - ap_trampoline

    /*
    * load our GDT pointer.
    *
    * 0x66 selects the 32-bit LGDT operand format: limit:2, base:4.
    * addressing remains 16-bit.
    *
    * ModRM 0x97:
    *   mod=10  disp16
    *   reg=010 LGDT (/2)
    *   rm=111  [BX]
    *
    * rm=011 would be [BP+DI], not [BX], and reg=001 is SIDT rather than LGDT.
    */
    .byte 0x66
    .byte 0x0f, 0x01, 0x97 /* lgdt disp16(%bx) */
    .word ap_gdt_pm_ptr - ap_trampoline
    AP_BUMP_STAGE16

    /*
    * enter protected mode before the far jump. without CR0.PE, the jump uses
    * real-mode CS semantics instead of loading selector 0x08 from the GDT.
    */
    .byte 0x66, 0x0f, 0x20, 0xc0 /* mov %cr0, %eax */
    .byte 0x66, 0x83, 0xc8, 0x01 /* or $CR0_PE, %eax */
    .byte 0x66, 0x0f, 0x22, 0xc0 /* mov %eax, %cr0 */
    AP_BUMP_STAGE16

    /*
    * far jump to flat 32-bit code.
    *
    * since the GDT code-segment base is zero, the jump offset must be the
    * absolute runtime address, not a within-page offset. the BSP patches it.
    */
    .byte 0xea
.globl ap_pm32_off16


ap_pm32_off16:

    .word 0 /* patched: page base + ap_pm32 offset */
    .word GDT_SEL_CODE32

    .code32
.globl ap_pm32


ap_pm32:

    /* load flat data selectors. */
    movw $GDT_SEL_DATA32, %ax
    movw %ax, %ds
    movw %ax, %es
    movw %ax, %ss

    /*
    * reload the GDT. required because the far jump out of this block loads
    * CS with a 64-bit (L=1) descriptor.
    *
    * same pointer record as the real-mode load: the memory layout follows the
    * operand size, not the execution mode. here that size is 32-bit.
    *
    * ModRM 0x93: mod=10, reg=010 (LGDT), rm=011 ([EBX]).
    */
    .byte 0x0f, 0x01, 0x93 /* lgdt disp32(%ebx) */
    .long ap_gdt_pm_ptr - ap_trampoline
    AP_BUMP_STAGE

    /*
    * CR4.PAE, required for the 4-level tables long mode uses.
    *
    * CR4 is zeroed rather than OR-ed into %ecx: INIT leaves %ecx with
    * whatever the delivery path last used, and a stray CR4.VMXE/SMXE bit puts
    * QEMU into nested virtualisation and spins it on the next CR write.
    */
    xorl %eax, %eax
    movl %eax, %cr4
    orl $CR4_PAE, %eax
    movl %eax, %cr4
    AP_BUMP_STAGE

    movl $MSR_EFER, %ecx
    rdmsr
    AP_BUMP_STAGE
    orl $EFER_LME, %eax
    wrmsr
    AP_BUMP_STAGE

    /*
    * load CR3 with the PML4 the BSP built. paging is still off, so this does
    * not change the translation regime yet.
    */
    .byte 0x8b, 0x83 /* movl disp32(%ebx), %eax */
    .long ap_cr3_off32 - ap_trampoline
    movl %eax, %cr3
    AP_BUMP_STAGE

    /*
    * enable paging, which is what actually turns long mode on: EFER.LMA is
    * only set once LME and CR0.PG are both set.
    *
    * CR0 is read-modify-written rather than zeroed, because zeroing would
    * briefly drop PE. the mask clears CR0.NW/CD, which firmware leaves set.
    */
    movl %eax, %cr0 /* read CR0 */
    andl $0x9fffffff, %eax /* clear NW (29), CD (30) */
    orl $(CR0_PG | CR0_PE), %eax
    movl %eax, %cr0 /* write CR0 */
    AP_BUMP_STAGE

    /*
    * CR4.PSE is unnecessary in long mode (PAE already provides 2 MiB pages),
    * and CR4.FSGSBASE is left clear because qemu64 does not advertise it,
    * which is why per-CPU state below goes through swapgs.
    */

    /*
    * far jump to the long-mode (L=1) segment. same absolute-offset rule as
    * the 16-bit far jump above                                                        ; the BSP patches this one with the page base.
    */
    .byte 0xea
.globl ap_lm_off32


ap_lm_off32:

    .long 0 /* patched: page base + ap_lm offset */
    .word GDT_SEL_CODE64

    .code64
.globl ap_lm


ap_lm:

    /*
    * long mode, paging live on the identity map the BSP built.
    *
    * select the AP's per-CPU area via IA32_KERNEL_GS_BASE + swapgs. GS.base
    * is still 0 on a freshly INIT'd AP, so the swap cannot corrupt anything.
    */
    movl $MSR_KERNEL_GS_BASE, %ecx
    .byte 0x48, 0x8b, 0x83 /* movq disp32(%rbx), %rax */
    .long ap_percpu - ap_trampoline

    /*
    * WRMSR writes (RDX << 32) | RAX, so the HIGH half is what needs zeroing.
    * clearing %eax instead puts the pointer at bit 32 and yields a
    * non-canonical GS base that faults in C much later.
    */
    xorl %edx, %edx
    wrmsr
    swapgs
    AP_BUMP_STAGE

    /*
    * load the AP's own TSS before the IDT.
    *
    * ORDER MATTERS: the IDT below names IST stacks, and an IST gate raises
    * #GP (which itself faults, giving a triple fault and a reset) unless the
    * processor already has a TSS loaded. the descriptor lives in this AP's own
    * GDT at GDT_SEL_TSS and its base was patched per AP, because the TSS holds
    * that AP's IST stack pointers and cannot be shared.
    *
    * ModRM 0x9b here is reg=011 (LTR /3), the same addressing as the LIDT.
    */
    .byte 0x0f, 0x00, 0x9b, 0x83 /* ltr disp32(%rbx) */
    .long ap_tss_sel - ap_trampoline
    AP_BUMP_STAGE

    /*
    * load the AP IDT the BSP built.
    *
    * separate from the BSP's table because its gates select 0x08, which is a
    * 32-bit code segment                                                              ; a long-mode interrupt gate must name a 64-bit one.
    * the IST entries the BSP uses ARE mirrored here now that the AP has a TSS,
    * so a #DF on an AP lands on a dedicated stack instead of a dead one.
    *
    * ModRM 0x9b: mod=10, reg=011 (LIDT), rm=011 ([RBX]).
    */
    .byte 0x0f, 0x01, 0x9b, 0x83 /* lidt disp32(%rbx) */
    .long ap_idt_pm_ptr - ap_trampoline
    AP_BUMP_STAGE

    /*
    * move to this AP's own stack, leaving the firmware stack behind.
    *
    * ModRM is 0xa3 (reg=rsp), not 0x83 (reg=rax) which silently discards the
    * value and leaves RSP at 0.
    */
    .byte 0x48, 0x8b, 0xa3 /* movq disp32(%rbx), %rsp */
    .long ap_stack_top - ap_trampoline
    xorl %ebp, %ebp /* terminate frame-pointer chains */
    AP_BUMP_STAGE

    /* call the AP entry point, patched by the BSP. */
    .byte 0xff, 0x93 /* call *disp32(%rbx) */
    .long ap_entry - ap_trampoline

    /* park if it ever returns. */


1:

    hlt
    jmp 1b

/*
    * GDT pointer record, read by both LGDTs above. LGDT loads a (limit, base)
    * pair, not the descriptor table, so this is kept apart from the table itself.
    * the BSP fills both fields                                                        ; the base is only known at run time.
    */
    .balign 8
.globl ap_gdt_ptr


ap_gdt_ptr:

.globl ap_gdt_pm_ptr


ap_gdt_pm_ptr:

    .word 0 /* patched: GDTR limit */
    .long 0 /* patched: GDTR base */
    .word 0 /* pad */

/*
    * IDT pointer record, read by the LIDT in ap_lm. limit then base, same as the
    * GDT record. both fields are absolute addresses: the table lives in the
    * kernel image rather than in this page.
    */
    .balign 8
.globl ap_idt_pm_ptr


ap_idt_pm_ptr:

    .word 0 /* patched: IDT limit */
    .long 0 /* patched: IDT base */
    .word 0 /* pad */

/*
    * BSP-patched slots. each holds an absolute value, never a within-page offset:
    * reading the offset where an address belongs installs a small integer as a
    * stack pointer or function pointer. ap_cr3_off32 is a slot for the same
    * reason, since CR3 names a page table outside this page.
    */
    .balign 8
.globl ap_cr3_off32


ap_cr3_off32:

    .quad 0 /* patched: absolute physical address of the PML4 */

.globl ap_percpu


ap_percpu:

    .quad 0 /* patched: &percpu[slot] */

.globl ap_stack_top


ap_stack_top:

    .quad 0 /* patched: top of this AP's stack */

.globl ap_entry


ap_entry:

    .quad 0 /* patched: ap_main() */

.globl ap_slot


ap_slot:

    .quad 0 /* patched: dense per-CPU slot for this AP */

.globl ap_tss_sel


ap_tss_sel:

    .word GDT_SEL_TSS /* selector, read by the LTR above */

    .quad 0

/*
    * progress marker, bumped at each step above so the BSP can report how far an
    * AP got. the page is reloaded before every AP, so the value always belongs to
    * the AP being started. debug aid                                                  ; the BSP only reads it.
    */
    .globl ap_stage


ap_stage:

    .byte 0
    .space 3

/*
    * minimal flat GDT: null, 32-bit code, 32-bit data, 64-bit code. loaded by the
    * real-mode prologue and again before the long-mode far jump, which needs the
    * L=1 descriptor at selector 0x18.
    *
    * a selector is the descriptor's byte offset from the table base, so the .set
    * names above and the layout below have to agree. the assembler resolves
    * `.if` at build time, so a mismatch fails the build rather than the boot.
    * the checks compare label differences because `.if` requires an absolute
    * expression, which `label + constant` is not.
    */
    .balign 8
.globl ap_gdt_sym


ap_gdt_sym:



ap_gdt:

    .quad 0x0000000000000000 /* null */


ap_gdt_code32:

    .quad 0x00cf9a000000ffff /* 0x08: 32-bit code, base 0, limit 4G */


ap_gdt_data32:

    .quad 0x00cf92000000ffff /* 0x10: 32-bit data, base 0, limit 4G */


ap_gdt_code64:

    .quad 0x00af9a000000ffff /* 0x18: 64-bit code, base 0, limit 4G */

.globl ap_gdt_tss


ap_gdt_tss:

    /*
    * 64-bit available TSS descriptor, 16 bytes, base patched per AP by
    * patch_trampoline(): the TSS holds that AP's IST stack pointers, so it
    * cannot be a fixed address.
    */
    .quad 0
    .quad 0
.globl ap_gdt_end


ap_gdt_end:

.if (ap_gdt_code32 - ap_gdt) != GDT_SEL_CODE32
    .error "ap_gdt: 32-bit code descriptor is not at GDT_SEL_CODE32"
.endif
.if (ap_gdt_data32 - ap_gdt) != GDT_SEL_DATA32
    .error "ap_gdt: 32-bit data descriptor is not at GDT_SEL_DATA32"
.endif
.if (ap_gdt_code64 - ap_gdt) != GDT_SEL_CODE64
    .error "ap_gdt: 64-bit code descriptor is not at GDT_SEL_CODE64"
.endif
.if (ap_gdt_tss - ap_gdt) != GDT_SEL_TSS
    .error "ap_gdt: TSS descriptor is not at GDT_SEL_TSS"
.endif
.if (ap_gdt_end - ap_gdt) != 48
    .error "ap_gdt: table is not 5 descriptors; a selector above is wrong"
.endif

/*
    * top of the 16-bit real-mode stack. SS=0 here, so this is an offset within
    * the page                                                                         ; keep it above all code and patched fields.
    */
    .balign 16


ap_rm_stack:

    .space 64
.globl ap_rm_stack_top


ap_rm_stack_top:

    .balign 8
.globl ap_trampoline_end


ap_trampoline_end:

/* size of the image, for the BSP to copy. */
.globl ap_trampoline_bytes


ap_trampoline_bytes:

    .quad ap_trampoline_end - ap_trampoline