furix
cute os
diff
+4306 -312
.astylerc added +7 -0
diff --git a/.astylerc b/.astylerc new file mode 100644 index 0000000000000000000000000000000000000000..5e71cb957b6ab9c1e21d849dce72511ca3e80b87 --- /dev/null +++ b/.astylerc @@ -0,0 +1,7 @@ +style=java +convert-tabs +align-pointer=name +pad-oper +pad-comma +pad-header +suffix=none
Makefile modified +22 -9
diff --git a/Makefile b/Makefile index 67049e7ecb0c0b6a115d06b904539d95c3825a26..65bd569ef1e75aafb48f4a8f73167542d195c680 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,26 @@ ARCH ?= x86_64 -CFLAGS := \ - -std=c99 -Wall -Werror -Wextra -pedantic-errors -MMD -MP -g \ - -ffreestanding -fno-builtin -nostdlib -nostdinc -Iinclude - -LDFLAGS := \ - -ffreestanding -nostdlib +CC := clang KERNEL_SRC := \ + kernel/acpi.c \ kernel/kernel.c BUILD := build/obj BUILD_BOOT := build/boot/$(ARCH) +EFI := $(BUILD_BOOT)/furix.efi -KERNEL_ELF := $(BUILD_BOOT)/kernel.elf +CFLAGS := \ + -std=c11 -Wall -Wextra -pedantic-errors -Werror -MMD -MP -g \ + -Wconversion -Wsign-conversion -Wshadow -Wcast-qual -Wformat=2 \ + -Wstrict-prototypes -Wold-style-definition -Wmissing-prototypes \ + -Wredundant-decls -Wundef -Wdouble-promotion -Wfloat-equal \ + -Winit-self -Wnull-dereference -Wswitch-enum -Wbad-function-cast \ + -ffreestanding -fno-builtin -nostdinc -fshort-wchar -fno-stack-protector \ + -Iinclude .PHONY: all -all: $(KERNEL_ELF) +all: $(EFI) .PHONY: clean clean: @@ -27,4 +31,13 @@ compile_flags.txt: @echo $(CFLAGS) | tr ' ' '\n' | tee $@ include arch/$(ARCH)/Makefile -include boot/$(ARCH)/Makefile +include boot/uefi/Makefile + +$(BUILD)/%.o: % + @mkdir -p $(dir $@) + $(CC) $(CFLAGS) -o $@ -c $< + +# asm isn't C, no language/standard/warning flags, just the target +$(BUILD)/%.S.o: %.S + @mkdir -p $(dir $@) + $(CC) -target x86_64-unknown-windows -c -o $@ $<
arch/x86_64/Makefile modified +15 -30
diff --git a/arch/x86_64/Makefile b/arch/x86_64/Makefile index 9d25e79ee43490b9f8034a1df1c77cfe0f18b669..6b8a90821983c3c6d9c579e9df1296078d080269 100644 --- a/arch/x86_64/Makefile +++ b/arch/x86_64/Makefile @@ -1,34 +1,19 @@ -ifndef KERNEL_SRC -$(error KERNEL_SRC undefined) -endif - -ifndef KERNEL_ELF -$(error KERNEL_ELF undefined) -endif - -ifndef BUILD -$(error BUILD undefined) -endif - -CC := x86_64-elf-gcc - CFLAGS += \ - -mno-red-zone -mgeneral-regs-only \ - -Iarch/x86_64/include - -LDFLAGS += \ - -Tarch/x86_64/linker.ld + -target x86_64-unknown-windows \ + -mgeneral-regs-only \ + -Iinclude/arch/x86_64 ARCH_SRC := \ - arch/x86_64/entry.S - -SRC := $(KERNEL_SRC) $(ARCH_SRC) -OBJ := $(patsubst %,$(BUILD)/%.o,$(SRC)) - -$(KERNEL_ELF): $(OBJ) - @mkdir -p $(dir $@) - $(CC) $(LDFLAGS) -o $@ $^ + arch/x86_64/ap_boot.c \ + arch/x86_64/gdt.c \ + arch/x86_64/idt.c \ + arch/x86_64/ioapic.c \ + arch/x86_64/lapic.c \ + arch/x86_64/pic.c \ + arch/x86_64/percpu.c \ + arch/x86_64/serial.c -$(BUILD)/%.o: % - @mkdir -p $(dir $@) - $(CC) $(CFLAGS) -o $@ -c $< +ARCH_ASM_SRC := \ + arch/x86_64/ap_boot.S \ + arch/x86_64/asm.S \ + arch/x86_64/isr.S
arch/x86_64/ap_boot.S added +538 -0
diff --git a/arch/x86_64/ap_boot.S b/arch/x86_64/ap_boot.S new file mode 100644 index 0000000000000000000000000000000000000000..1f92c92a787ce81de45a83deb680b744b5d321f7 --- /dev/null +++ b/arch/x86_64/ap_boot.S @@ -0,0 +1,538 @@ +/* + * x86_64 application-processor startup trampoline + * + * a SIPI starts an AP in 16-bit real mode at a low-memory page chosen by the + * BSP. this trampoline transitions through protected mode into long mode and + * finally calls the AP's C entry point. + * + * the BSP copies this image below 1 MiB and patches the fields marked below. + * + * the trampoline is position-independent with respect to its copied page: + * %bx/%ebx/%rbx holds the runtime page base, and references within the page + * use page-relative displacements. do not replace them with link-time absolute + * addresses. + * + * raw byte encodings are used where the assembler would otherwise emit + * absolute ADDR32 relocations. after ANY change you make, verify: + * + * - llvm-readelf -r shows no relocation for ap_lm or far-jump targets. + * - BSP-patched slot offsets still match this layout. + */ + +.text + +/* + * selectors from the flat GDT at the bottom of this file. kept as names so a + * change to the table layout cannot drift away from the far jumps that use it. + */ +.set GDT_SEL_CODE32, 0x08 +.set GDT_SEL_DATA32, 0x10 +.set GDT_SEL_CODE64, 0x18 +.set GDT_SEL_TSS, 0x20 + +/* model-specific registers, named because the raw numbers are untraceable */ +.set MSR_EFER, 0xC0000080 +.set MSR_KERNEL_GS_BASE, 0xC0000102 + +.set CR0_PE, (1 << 0) +.set CR0_PG, (1 << 31) +.set CR4_PAE, (1 << 5) + +/* EFER.LME: sets LMA once CR0.PG is also on */ + +.set EFER_LME, (1 << 8) + +/* + * bump the progress marker, so a BSP sampling ap_stage can tell how far an AP + * got. one macro per address size because the displacement width is encoded + * separately from the opcode (0x87/0x83 are incb with different ModRM). + * the base register is not encoded here either, so the disp32 form serves the + * 64-bit section too: incb disp32(%ebx) and disp32(%rbx) are the same bytes. + */ +.macro AP_BUMP_STAGE16 + .byte 0xfe, 0x87 /* incb disp16(%bx) */ + .word ap_stage - ap_trampoline +.endm + +.macro AP_BUMP_STAGE + .byte 0xfe, 0x83 /* incb disp32(%ebx) */ + .long ap_stage - ap_trampoline +.endm + +.globl ap_trampoline + + +ap_trampoline: + + .code16 + + /* entry is real mode, paging off, interrupts masked. */ + .byte 0xfa /* cli */ + .byte 0xfc /* cld */ + + .byte 0x31, 0xc0 /* xorw %ax, %ax */ + .byte 0x8e, 0xd8 /* movw %ax, %ds */ + .byte 0x8e, 0xc0 /* movw %ax, %es */ + .byte 0x8e, 0xd0 /* movw %ax, %ss */ + + /* + * load the runtime page base into %bx. + * + * INIT leaves SP=0, so a call/pop position-discovery trick is unsafe here. + * the BSP already knows the copied page address, so it patches that value + * directly into ap_rm_base_slot and patches this instruction's absolute + * 16-bit operand to point at the slot. + * + * slots embedded in the instruction stream must be jumped over. + */ + jmp 1f +.globl ap_rm_base_slot + + +ap_rm_base_slot: + + .word 0 /* patched: trampoline page base */ + + +1: + + .byte 0x8b, 0x1e /* movw abs16, %bx */ +.globl ap_rm_base_addr16 + + +ap_rm_base_addr16: + + .word 0 /* patched: address of ap_rm_base_slot */ + + /* + * install a real-mode stack within this page + * + * SS=0, so SP must contain the absolute low-memory address of the stack + * top. the displacement and loaded value are distinct words. + * + * ModRM is 0x87 (mod=10, reg=SP, rm=111) so the disp16 is present. 0x27 + * drops the displacement and loads SP from the code bytes instead. + */ + jmp 1f +.globl ap_rm_sp_slot + + +ap_rm_sp_slot: + + .word 0 /* patched: page base + stack-top offset */ + + +1: + + .byte 0x8b, 0x87 /* movw disp16(%bx), %sp */ +.globl ap_rm_sp_disp16 + + +ap_rm_sp_disp16: + + .word ap_rm_sp_slot - ap_trampoline + + /* + * load our GDT pointer. + * + * 0x66 selects the 32-bit LGDT operand format: limit:2, base:4. + * addressing remains 16-bit. + * + * ModRM 0x97: + * mod=10 disp16 + * reg=010 LGDT (/2) + * rm=111 [BX] + * + * rm=011 would be [BP+DI], not [BX], and reg=001 is SIDT rather than LGDT. + */ + .byte 0x66 + .byte 0x0f, 0x01, 0x97 /* lgdt disp16(%bx) */ + .word ap_gdt_pm_ptr - ap_trampoline + AP_BUMP_STAGE16 + + /* + * enter protected mode before the far jump. without CR0.PE, the jump uses + * real-mode CS semantics instead of loading selector 0x08 from the GDT. + */ + .byte 0x66, 0x0f, 0x20, 0xc0 /* mov %cr0, %eax */ + .byte 0x66, 0x83, 0xc8, 0x01 /* or $CR0_PE, %eax */ + .byte 0x66, 0x0f, 0x22, 0xc0 /* mov %eax, %cr0 */ + AP_BUMP_STAGE16 + + /* + * far jump to flat 32-bit code. + * + * since the GDT code-segment base is zero, the jump offset must be the + * absolute runtime address, not a within-page offset. the BSP patches it. + */ + .byte 0xea +.globl ap_pm32_off16 + + +ap_pm32_off16: + + .word 0 /* patched: page base + ap_pm32 offset */ + .word GDT_SEL_CODE32 + + .code32 +.globl ap_pm32 + + +ap_pm32: + + /* load flat data selectors. */ + movw $GDT_SEL_DATA32, %ax + movw %ax, %ds + movw %ax, %es + movw %ax, %ss + + /* + * reload the GDT. required because the far jump out of this block loads + * CS with a 64-bit (L=1) descriptor. + * + * same pointer record as the real-mode load: the memory layout follows the + * operand size, not the execution mode. here that size is 32-bit. + * + * ModRM 0x93: mod=10, reg=010 (LGDT), rm=011 ([EBX]). + */ + .byte 0x0f, 0x01, 0x93 /* lgdt disp32(%ebx) */ + .long ap_gdt_pm_ptr - ap_trampoline + AP_BUMP_STAGE + + /* + * CR4.PAE, required for the 4-level tables long mode uses. + * + * CR4 is zeroed rather than OR-ed into %ecx: INIT leaves %ecx with + * whatever the delivery path last used, and a stray CR4.VMXE/SMXE bit puts + * QEMU into nested virtualisation and spins it on the next CR write. + */ + xorl %eax, %eax + movl %eax, %cr4 + orl $CR4_PAE, %eax + movl %eax, %cr4 + AP_BUMP_STAGE + + movl $MSR_EFER, %ecx + rdmsr + AP_BUMP_STAGE + orl $EFER_LME, %eax + wrmsr + AP_BUMP_STAGE + + /* + * load CR3 with the PML4 the BSP built. paging is still off, so this does + * not change the translation regime yet. + */ + .byte 0x8b, 0x83 /* movl disp32(%ebx), %eax */ + .long ap_cr3_off32 - ap_trampoline + movl %eax, %cr3 + AP_BUMP_STAGE + + /* + * enable paging, which is what actually turns long mode on: EFER.LMA is + * only set once LME and CR0.PG are both set. + * + * CR0 is read-modify-written rather than zeroed, because zeroing would + * briefly drop PE. the mask clears CR0.NW/CD, which firmware leaves set. + */ + movl %eax, %cr0 /* read CR0 */ + andl $0x9fffffff, %eax /* clear NW (29), CD (30) */ + orl $(CR0_PG | CR0_PE), %eax + movl %eax, %cr0 /* write CR0 */ + AP_BUMP_STAGE + + /* + * CR4.PSE is unnecessary in long mode (PAE already provides 2 MiB pages), + * and CR4.FSGSBASE is left clear because qemu64 does not advertise it, + * which is why per-CPU state below goes through swapgs. + */ + + /* + * far jump to the long-mode (L=1) segment. same absolute-offset rule as + * the 16-bit far jump above ; the BSP patches this one with the page base. + */ + .byte 0xea +.globl ap_lm_off32 + + +ap_lm_off32: + + .long 0 /* patched: page base + ap_lm offset */ + .word GDT_SEL_CODE64 + + .code64 +.globl ap_lm + + +ap_lm: + + /* + * long mode, paging live on the identity map the BSP built. + * + * select the AP's per-CPU area via IA32_KERNEL_GS_BASE + swapgs. GS.base + * is still 0 on a freshly INIT'd AP, so the swap cannot corrupt anything. + */ + movl $MSR_KERNEL_GS_BASE, %ecx + .byte 0x48, 0x8b, 0x83 /* movq disp32(%rbx), %rax */ + .long ap_percpu - ap_trampoline + + /* + * WRMSR writes (RDX << 32) | RAX, so the HIGH half is what needs zeroing. + * clearing %eax instead puts the pointer at bit 32 and yields a + * non-canonical GS base that faults in C much later. + */ + xorl %edx, %edx + wrmsr + swapgs + AP_BUMP_STAGE + + /* + * load the AP's own TSS before the IDT. + * + * ORDER MATTERS: the IDT below names IST stacks, and an IST gate raises + * #GP (which itself faults, giving a triple fault and a reset) unless the + * processor already has a TSS loaded. the descriptor lives in this AP's own + * GDT at GDT_SEL_TSS and its base was patched per AP, because the TSS holds + * that AP's IST stack pointers and cannot be shared. + * + * ModRM 0x9b here is reg=011 (LTR /3), the same addressing as the LIDT. + */ + .byte 0x0f, 0x00, 0x9b, 0x83 /* ltr disp32(%rbx) */ + .long ap_tss_sel - ap_trampoline + AP_BUMP_STAGE + + /* + * load the AP IDT the BSP built. + * + * separate from the BSP's table because its gates select 0x08, which is a + * 32-bit code segment ; a long-mode interrupt gate must name a 64-bit one. + * the IST entries the BSP uses ARE mirrored here now that the AP has a TSS, + * so a #DF on an AP lands on a dedicated stack instead of a dead one. + * + * ModRM 0x9b: mod=10, reg=011 (LIDT), rm=011 ([RBX]). + */ + .byte 0x0f, 0x01, 0x9b, 0x83 /* lidt disp32(%rbx) */ + .long ap_idt_pm_ptr - ap_trampoline + AP_BUMP_STAGE + + /* + * move to this AP's own stack, leaving the firmware stack behind. + * + * ModRM is 0xa3 (reg=rsp), not 0x83 (reg=rax) which silently discards the + * value and leaves RSP at 0. + */ + .byte 0x48, 0x8b, 0xa3 /* movq disp32(%rbx), %rsp */ + .long ap_stack_top - ap_trampoline + xorl %ebp, %ebp /* terminate frame-pointer chains */ + AP_BUMP_STAGE + + /* call the AP entry point, patched by the BSP. */ + .byte 0xff, 0x93 /* call *disp32(%rbx) */ + .long ap_entry - ap_trampoline + + /* park if it ever returns. */ + + +1: + + hlt + jmp 1b + +/* + * GDT pointer record, read by both LGDTs above. LGDT loads a (limit, base) + * pair, not the descriptor table, so this is kept apart from the table itself. + * the BSP fills both fields ; the base is only known at run time. + */ + .balign 8 +.globl ap_gdt_ptr + + +ap_gdt_ptr: + +.globl ap_gdt_pm_ptr + + +ap_gdt_pm_ptr: + + .word 0 /* patched: GDTR limit */ + .long 0 /* patched: GDTR base */ + .word 0 /* pad */ + +/* + * IDT pointer record, read by the LIDT in ap_lm. limit then base, same as the + * GDT record. both fields are absolute addresses: the table lives in the + * kernel image rather than in this page. + */ + .balign 8 +.globl ap_idt_pm_ptr + + +ap_idt_pm_ptr: + + .word 0 /* patched: IDT limit */ + .long 0 /* patched: IDT base */ + .word 0 /* pad */ + +/* + * BSP-patched slots. each holds an absolute value, never a within-page offset: + * reading the offset where an address belongs installs a small integer as a + * stack pointer or function pointer. ap_cr3_off32 is a slot for the same + * reason, since CR3 names a page table outside this page. + */ + .balign 8 +.globl ap_cr3_off32 + + +ap_cr3_off32: + + .quad 0 /* patched: absolute physical address of the PML4 */ + +.globl ap_percpu + + +ap_percpu: + + .quad 0 /* patched: &percpu[slot] */ + +.globl ap_stack_top + + +ap_stack_top: + + .quad 0 /* patched: top of this AP's stack */ + +.globl ap_entry + + +ap_entry: + + .quad 0 /* patched: ap_main() */ + +.globl ap_slot + + +ap_slot: + + .quad 0 /* patched: dense per-CPU slot for this AP */ + +.globl ap_tss_sel + + +ap_tss_sel: + + .word GDT_SEL_TSS /* selector, read by the LTR above */ + + .quad 0 + +/* + * progress marker, bumped at each step above so the BSP can report how far an + * AP got. the page is reloaded before every AP, so the value always belongs to + * the AP being started. debug aid ; the BSP only reads it. + */ + .globl ap_stage + + +ap_stage: + + .byte 0 + .space 3 + +/* + * minimal flat GDT: null, 32-bit code, 32-bit data, 64-bit code. loaded by the + * real-mode prologue and again before the long-mode far jump, which needs the + * L=1 descriptor at selector 0x18. + * + * a selector is the descriptor's byte offset from the table base, so the .set + * names above and the layout below have to agree. the assembler resolves + * `.if` at build time, so a mismatch fails the build rather than the boot. + * the checks compare label differences because `.if` requires an absolute + * expression, which `label + constant` is not. + */ + .balign 8 +.globl ap_gdt_sym + + +ap_gdt_sym: + + + +ap_gdt: + + .quad 0x0000000000000000 /* null */ + + +ap_gdt_code32: + + .quad 0x00cf9a000000ffff /* 0x08: 32-bit code, base 0, limit 4G */ + + +ap_gdt_data32: + + .quad 0x00cf92000000ffff /* 0x10: 32-bit data, base 0, limit 4G */ + + +ap_gdt_code64: + + .quad 0x00af9a000000ffff /* 0x18: 64-bit code, base 0, limit 4G */ + +.globl ap_gdt_tss + + +ap_gdt_tss: + + /* + * 64-bit available TSS descriptor, 16 bytes, base patched per AP by + * patch_trampoline(): the TSS holds that AP's IST stack pointers, so it + * cannot be a fixed address. + */ + .quad 0 + .quad 0 +.globl ap_gdt_end + + +ap_gdt_end: + +.if (ap_gdt_code32 - ap_gdt) != GDT_SEL_CODE32 + .error "ap_gdt: 32-bit code descriptor is not at GDT_SEL_CODE32" +.endif +.if (ap_gdt_data32 - ap_gdt) != GDT_SEL_DATA32 + .error "ap_gdt: 32-bit data descriptor is not at GDT_SEL_DATA32" +.endif +.if (ap_gdt_code64 - ap_gdt) != GDT_SEL_CODE64 + .error "ap_gdt: 64-bit code descriptor is not at GDT_SEL_CODE64" +.endif +.if (ap_gdt_tss - ap_gdt) != GDT_SEL_TSS + .error "ap_gdt: TSS descriptor is not at GDT_SEL_TSS" +.endif +.if (ap_gdt_end - ap_gdt) != 48 + .error "ap_gdt: table is not 5 descriptors; a selector above is wrong" +.endif + +/* + * top of the 16-bit real-mode stack. SS=0 here, so this is an offset within + * the page ; keep it above all code and patched fields. + */ + .balign 16 + + +ap_rm_stack: + + .space 64 +.globl ap_rm_stack_top + + +ap_rm_stack_top: + + .balign 8 +.globl ap_trampoline_end + + +ap_trampoline_end: + +/* size of the image, for the BSP to copy. */ +.globl ap_trampoline_bytes + + +ap_trampoline_bytes: + + .quad ap_trampoline_end - ap_trampoline
arch/x86_64/ap_boot.c added +739 -0
diff --git a/arch/x86_64/ap_boot.c b/arch/x86_64/ap_boot.c new file mode 100644 index 0000000000000000000000000000000000000000..d9b7bd008d2fffd195992fa2760eb1b62e13f9b5 --- /dev/null +++ b/arch/x86_64/ap_boot.c @@ -0,0 +1,739 @@ +#include <ap_boot.h> +#include <gdt.h> + +#include <idt.h> +#include <io.h> +#include <lapic.h> +#include <percpu.h> +#include <serial.h> + +/* provided by ap_boot.S */ +extern const uint8_t ap_trampoline[]; +extern const uint64_t ap_trampoline_bytes; + +/* patch slots in the trampoline, written by the BSP and read by another core */ +extern volatile uint64_t ap_percpu; +extern volatile uint64_t ap_stack_top; +extern volatile uint64_t ap_entry; +extern volatile uint64_t ap_slot; +extern const uint8_t ap_gdt_sym[]; +extern const uint8_t ap_gdt_end[]; +extern const uint8_t ap_gdt_tss[]; + +/* + * GDT pointer record, read by both LGDTs in the trampoline. it holds the limit + * and base, NOT the descriptor table. pointing LGDT at the descriptors + * themselves leaves GDTR.limit zero and the far jump triple faults. + * + * one record serves both loads: LGDT's memory layout follows the operand size, + * and both loads use a 32-bit operand size. + */ +extern volatile uint8_t ap_gdt_ptr[]; +extern const uint8_t ap_gdt_pm_ptr[]; +extern const uint8_t ap_gdt_end[]; + +/* + * IDT pointer record, read by the LIDT in ap_lm. both fields are absolute + * addresses: the table lives in the kernel image, not in this page. + */ +extern volatile uint8_t ap_idt_pm_ptr[]; + +/* byte offsets within the GDT pointer record, which is limit-first */ +#define AP_GDT_PM_LIMIT 0u +#define AP_GDT_PM_BASE 2u + +#define AP_IDT_PM_LIMIT 0u +#define AP_IDT_PM_BASE 2u + +/* + * far-jump offset slots and the labels they must name. + * + * a far jump computes CS.base + offset, and the flat GDT gives every + * descriptor base 0, so the offset field must hold the ABSOLUTE linear address + * of the target. using the within-page offset alone sends the AP to linear + * 0x23, inside the real-mode IVT, where it executes vector bytes forever. + * + * the 16-bit form is emitted as raw bytes because `ljmp $0x08, sym` in .code16 + * produces an ADDR32 relocation the linker fills with the image address. + */ +extern const uint8_t ap_pm32_off16[]; +extern const uint8_t ap_pm32[]; +extern const uint8_t ap_lm_off32[]; +extern const uint8_t ap_lm[]; + +/* + * the real-mode prologue's idea of where it lives. + * + * the AP cannot work this out for itself: INIT leaves SP=0, so the classic + * `call $+5 ; popw %bx` sequence reads zeros back and every page-relative + * reference then addresses the wrong place. the BSP knows the answer, it is + * the base of the page it just wrote, so it is patched straight in. + * + * two words are needed because the load is a moffs16, i.e. an absolute address + * baked into the instruction rather than a [bx+disp16]: + * + * - ap_rm_base_addr16 is the operand the AP reads FROM. + * - ap_rm_base_slot is the value the AP reads, the page base itself. + * + * both are 16 bits because the page is below 1 MiB. + */ +extern const uint8_t ap_rm_base_addr16[]; +extern const uint8_t ap_rm_base_slot[]; +extern const uint8_t ap_rm_sp_slot[]; +extern const uint8_t ap_rm_stack_top[]; +extern const uint8_t ap_stage[]; + +extern const uint8_t ap_gdt_sym[]; + +/* + * the PML4 base for CR3, a slot rather than a label because CR3 names a page + * table outside this page. patched with an absolute physical address. + */ +extern volatile uint64_t ap_cr3_off32; + +#define AP_PM32_OFF16_OFF ((uint64_t)(&ap_pm32_off16[0] - (uint64_t)ap_trampoline)) +#define AP_LM_OFF32_OFF ((uint64_t)(&ap_lm_off32[0] - (uint64_t)ap_trampoline)) +#define AP_PM32_WITHIN ((uint64_t)(&ap_pm32[0] - (uint64_t)ap_trampoline)) +#define AP_LM_WITHIN ((uint64_t)(&ap_lm[0] - (uint64_t)ap_trampoline)) +#define AP_RM_SP_WITHIN ((uint64_t)(&ap_rm_stack_top[0] - (uint64_t)ap_trampoline)) +/* the AP's TSS descriptor inside its own copy of the flat GDT */ +#define AP_GDT_TSS_WITHIN ((uint64_t)(&ap_gdt_tss[0] - (uint64_t)ap_trampoline)) + +/* the reserved trampoline page, set by ap_boot_reserve() */ +static uint64_t trampoline_base; + +/* + * where the trampoline's progress marker lives inside the page, and the highest + * value any AP has reached there. debug aid only, see delay_and_latch(). + */ +static volatile uint8_t *ap_stage_ptr; +static uint8_t ap_stage_best; + +/* the APs' PML4, built by ap_boot_reserve() */ +static uint64_t ap_pml4_base; + +static void ap_main(void); + +/* + * APs that have come up and initialised their per-CPU area. + * + * incremented with a locked add, not `++`: several APs reach ap_main() at once + * and a plain read-modify-write lets two of them read the same old value, so + * one increment is lost and the BSP waits forever for an AP that is already + * online. the BSP only reads it through atomic_load_u32(). + */ +volatile uint32_t ap_online_count; + +#define PAGE_SIZE 4096u + +/* a real-mode CS:IP cannot reach past 1 MiB */ +#define TRAMPOLINE_MAX_ADDR 0x100000ull + +/* bounded so a lost SIPI surfaces as a failed bringup, never a hang */ +#define AP_START_SPINS 40000000u + +/* + * memory types we may take a page from. EfiLoaderCode/Data is where our own + * image lives and EfiRuntimeServices* is firmware-owned, so both are excluded. + */ +static bool memory_type_usable(uint32_t type) { + return type == EfiBootServicesCode || + type == EfiBootServicesData || + type == EfiConventionalMemory; +} + +/* + * find a free page below 1 MiB from the UEFI memory map. + * + * AllocatePages alone would be simpler but gives no upper bound, and the AP's + * real-mode entry point must be somewhere the CPU can reach. so read the map + * and choose an address, then claim that exact address below. + */ +static bool find_low_page(const EFI_SYSTEM_TABLE *st, uint64_t *out_addr) { + EFI_BOOT_SERVICES *bs = st->BootServices; + EFI_MEMORY_DESCRIPTOR *map = NULL; + UINTN map_size = 0; + UINTN desc_size = 0; + UINTN key = 0; + UINTN entry_size; + UINTN count; + UINTN i; + uint32_t version = 0; + EFI_STATUS status; + bool found = false; + + /* + * the sizing call passes a NULL buffer and is SUPPOSED to fail with + * EFI_BUFFER_TOO_SMALL while reporting the size needed. testing it with + * EFIERR() would reject the only outcome that is not an error, so require + * exactly that status instead. + */ + status = bs->GetMemoryMap(&map_size, NULL, &key, &desc_size, &version); + + if (status != EFI_BUFFER_TOO_SMALL) { + serial_puts("ap: GetMemoryMap size probe failed, status="); + serial_put_hex((uint64_t)status); + serial_puts("\n"); + return false; + } + + /* leave headroom: the map can grow between calls as firmware allocates */ + map_size += desc_size * 8; + + if (EFIERR(bs->AllocatePool(EfiLoaderData, map_size, (void **)&map))) { + serial_puts("ap: AllocatePool failed for the memory map\n"); + return false; + } + + if (EFIERR(bs->GetMemoryMap(&map_size, map, &key, &desc_size, &version))) { + bs->FreePool(map); + serial_puts("ap: GetMemoryMap failed\n"); + return false; + } + + entry_size = desc_size; + count = map_size / entry_size; + + for (i = 0; i < count && !found; i++) { + const EFI_MEMORY_DESCRIPTOR *d = &map[i]; + uint64_t start = d->PhysicalStart; + uint64_t end; + uint64_t first_page; + + if (!memory_type_usable(d->Type)) { + continue; + } + + if (start >= TRAMPOLINE_MAX_ADDR) { + continue; + } + + end = start + d->NumberOfPages * PAGE_SIZE; + + first_page = (start + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1); + + /* + * physical page 0 holds the real-mode IVT and BIOS data area, and the AP + * jumps there on INIT, so never hand out page 0. the region itself may + * legitimately START at 0, so skip the page rather than the region. + */ + if (first_page == 0) { + first_page = PAGE_SIZE; + } + + if (first_page + PAGE_SIZE <= end && first_page + PAGE_SIZE <= TRAMPOLINE_MAX_ADDR) { + *out_addr = first_page; + found = true; + } + } + + bs->FreePool(map); + return found; +} + +/* entry attribute bits for the page tables below */ +#define PTE_PRESENT (1ull << 0) +#define PTE_WRITE (1ull << 1) +#define PTE_LARGE (1ull << 7) + +/* 512 entries per table, each 8 bytes wide */ +#define PAGE_TABLE_ENTRIES 512u + +/* + * identity-map the low 4 GiB with 2 MiB pages, for the APs only. + * + * long mode cannot be entered with paging off, since EFER.LME is not honoured + * until CR0.PG is set. page tables are required to *enter* long mode, not + * merely a later refinement. the AP needs a translation regime before it can + * fetch a single 64-bit instruction. + * + * three levels suffice for 4 GiB of identity mapping: + * + * PML4[0] -> PDPT (covers 512 GiB) + * PDPT[0] -> PD[0] (covers 1 GiB) + * PD[i][j] -> 2 MiB page (512 pages of 2 MiB each) + * + * the BSP does NOT use these tables; it keeps running on the firmware's CR3 + * until it builds its own. only the APs load this PML4. that is temporary and + * a known asymmetry to be resolved with the paging layer. + */ +#define AP_MAPPED_GIB 4u + +/* bytes needed: PML4 + PDPT + AP_MAPPED_GIB page directories */ +#define AP_PT_PAGES (2u + AP_MAPPED_GIB) + +static void zero_page(uint64_t addr) { + uint64_t *p = (uint64_t *)addr; + uint32_t i; + + for (i = 0; i < PAGE_TABLE_ENTRIES; i++) { + p[i] = 0; + } +} + +/* + * build the identity map and return the PML4 physical address, or 0 on failure. + * + * the tables are allocated by the firmware and left as ordinary physical + * memory: they are only ever read by the hardware, never by C code, so a + * virtual mapping is unnecessary. + */ +static uint64_t build_ap_page_tables(const EFI_SYSTEM_TABLE *st) { + EFI_BOOT_SERVICES *bs = st->BootServices; + uint64_t base = 0; + uint64_t pml4; + uint64_t pdpt; + uint64_t *pd; + uint32_t gi; + uint32_t i; + + if (EFIERR(bs->AllocatePages(AllocateAnyPages, EfiBootServicesData, + AP_PT_PAGES, &base))) { + serial_puts("ap: could not allocate page tables\n"); + return 0; + } + + pml4 = base; + pdpt = base + PAGE_SIZE; + + zero_page(pml4); + zero_page(pdpt); + + /* + * entries are physical addresses with the attribute bits ORed in, so the low + * 12 bits, always zero for a page-aligned table, are free to hold flags. + */ + *(volatile uint64_t *)pml4 = pdpt | PTE_PRESENT | PTE_WRITE; + + for (gi = 0; gi < AP_MAPPED_GIB; gi++) { + pd = (uint64_t *)(pdpt + PAGE_SIZE + (uint64_t)gi * PAGE_SIZE); + zero_page(pdpt + PAGE_SIZE + (uint64_t)gi * PAGE_SIZE); + + /* + * every PDPT entry has to be filled in, not just entry 0. + * + * one page directory covers 512 * 2 MiB = 1 GiB, so a PDPT with only + * PDPT[0] populated maps exactly one gibibyte no matter how many + * directories were filled in. the others were built, fully populated, and + * simply unreachable. the first access above 1 GiB is lapic_id(), whose + * #PF lands on the LAPIC's own MMIO window at 0xfee00000 and so reads like + * a broken APIC. + * + * PDPT entries are 8 BYTES APART. `gi * PAGE_SIZE` is wrong here even + * though the page directories themselves really are PAGE_SIZE apart: + * indexing by PAGE_SIZE writes gi=1 over PD[0][0], which chains the + * directories into each other and overwrites the large pages. the walk + * then resolves the trampoline's own address to an unrelated physical + * page, the AP fetches zeros instead of its own instructions, and the + * failure looks like the trampoline page was corrupted. + */ + *(volatile uint64_t *)(pdpt + (uint64_t)gi * sizeof(uint64_t)) = + (uint64_t)pd | PTE_PRESENT | PTE_WRITE; + + for (i = 0; i < PAGE_TABLE_ENTRIES; i++) { + uint64_t phys = ((uint64_t)gi * PAGE_TABLE_ENTRIES + i) * (2u * 1024u * 1024u); + + pd[i] = phys | PTE_PRESENT | PTE_WRITE | PTE_LARGE; + } + } + + return pml4; +} + +/* + * write the page-relative displacements used by the trampoline. + * + * everything here is an offset from the start of the image, because the AP + * adds it to the page base it recovers at run time. the exception is CR3, + * which names a page table outside the page and so holds an absolute address. + */ +static bool patch_page_relatives(volatile uint8_t *page, uint64_t base) { + const uint64_t image = (uint64_t)ap_trampoline; + const uint32_t gdt_limit = (uint32_t)((uint64_t)ap_gdt_end - (uint64_t)ap_gdt_sym) - 1u; + volatile uint8_t *pm = &page[(uint64_t)ap_gdt_pm_ptr - image]; + + /* + * the GDT lives inside the page, so its runtime address is the page base + * plus its within-page offset. this is the one place a true absolute address + * is needed: the CPU reads the base out of the pointer record rather than + * adding a displacement itself. + */ + const uint32_t gdt_base = (uint32_t)(base + ((uint64_t)ap_gdt_sym - image)); + + /* limit:2 then base:4 then 2 reserved */ + *(volatile uint16_t *)&pm[AP_GDT_PM_LIMIT] = (uint16_t)gdt_limit; + *(volatile uint32_t *)&pm[AP_GDT_PM_BASE] = gdt_base; + + /* + * the IDT pointer, unlike the GDT this one is not inside the page: the table + * is the AP's own copy in the kernel image, already built by idt_init(), so + * both fields are the address the BSP published. + */ + { + volatile uint8_t *idt_ptr = &page[(uint64_t)ap_idt_pm_ptr - image]; + + *(volatile uint16_t *)&idt_ptr[AP_IDT_PM_LIMIT] = idt_ap_pointer.limit; + *(volatile uint32_t *)&idt_ptr[AP_IDT_PM_BASE] = + (uint32_t)idt_ap_pointer.base; + } + + /* + * GDTR.limit is the largest usable OFFSET, so it must cover the last byte of + * the descriptor the long-mode far jump selects, selector 0x18, whose + * descriptor ends at offset 0x1f. too small and the far jump raises #GP, + * triple faults and resets the AP, which looks exactly like "it never ran". + */ + if (gdt_limit < 0x2fu) { + serial_puts("ap: gdt limit too small, "); + serial_put_hex(gdt_limit); + serial_puts("\n"); + return false; + } + + /* CR3 gets the PML4's ABSOLUTE physical address */ + *(volatile uint64_t *)&page[(uint64_t)&ap_cr3_off32 - image] = ap_pml4_base; + + /* + * the two far-jump offsets, plus the real-mode page base and stack pointer. + * absolute addresses, not within-page offsets. both targets are inside this + * page, which the BSP places below 1 MiB, so 16 bits suffices for the first + * and 32 for the second. + */ + { + const uint64_t pm32_lin = base + AP_PM32_WITHIN; + const uint64_t lm_lin = base + AP_LM_WITHIN; + const uint64_t rm_sp = base + AP_RM_SP_WITHIN; + const uint64_t base_slot_addr = base + (uint64_t)&ap_rm_base_slot - image; + + if (base > 0xffffu || base_slot_addr > 0xffffu || pm32_lin > 0xffffu || + rm_sp > 0xffffu || lm_lin > 0xffffffffu) { + serial_puts("ap: trampoline page too high for its 16-bit offsets\n"); + return false; + } + + *(volatile uint16_t *)&page[AP_PM32_OFF16_OFF] = (uint16_t)pm32_lin; + *(volatile uint32_t *)&page[AP_LM_OFF32_OFF] = (uint32_t)lm_lin; + + /* both the operand AND the value are patched */ + *(volatile uint16_t *)&page[(uint64_t)ap_rm_base_addr16 - image] = + (uint16_t)base_slot_addr; + *(volatile uint16_t *)&page[(uint64_t)ap_rm_base_slot - image] = + (uint16_t)base; + + /* + * the real-mode stack pointer is an absolute linear address: SS is 0, so + * SP is the physical address itself. + */ + *(volatile uint16_t *)&page[(uint64_t)ap_rm_sp_slot - image] = + (uint16_t)rm_sp; + } + + return true; +} + +/* + * refresh the trampoline page from the image and re-apply every patch. + * + * done once per AP rather than once at reserve time. the page runs below 1 MiB + * with paging off, which is shared space: an AP that faults there has no IDT + * to report through, so it triple faults and QEMU resets it, and a reset AP + * starts executing whatever sits at the reset vector. on an OVMF-only machine + * that is uninitialised low RAM, and the resulting garbage stores land in this + * page. a single failure then leaves the next AP with a trampoline whose + * prologue and far-jump opcode have been overwritten. re-copying makes each AP + * start from a page known to be identical to the image. + */ +static void reload_trampoline(void) { + volatile uint8_t *page = (volatile uint8_t *)trampoline_base; + uint64_t i; + + for (i = 0; i < ap_trampoline_bytes; i++) { + page[i] = ap_trampoline[i]; + } + + if (!patch_page_relatives(page, trampoline_base)) { + serial_puts("ap: could not patch trampoline\n"); + } +} + +bool ap_boot_reserve(const EFI_SYSTEM_TABLE *st, uint64_t *out_base) { + uint64_t base = 0; + volatile uint8_t *page; + uint64_t bytes; + uint64_t i; + + if (!find_low_page(st, &base)) { + serial_puts("ap: no free page below 1 MiB\n"); + return false; + } + + bytes = ap_trampoline_bytes; + + /* + * build the APs' identity map BEFORE patching, so ap_cr3_off32 has something + * real to point at. long mode cannot be entered with paging disabled, so a + * failure here is fatal to AP bringup. + */ + ap_pml4_base = build_ap_page_tables(st); + + if (ap_pml4_base == 0) { + return false; + } + + if (bytes > PAGE_SIZE) { + serial_puts("ap: trampoline too large ("); + serial_put_hex(bytes); + serial_puts(" bytes)\n"); + return false; + } + + /* claim it before writing, so firmware cannot hand it out underneath us */ + if (EFIERR(st->BootServices->AllocatePages(AllocateAddress, EfiBootServicesData, 1, &base))) { + serial_puts("ap: could not claim page at "); + serial_put_hex(base); + serial_puts("\n"); + return false; + } + + page = (volatile uint8_t *)base; + + for (i = 0; i < bytes; i++) { + page[i] = ap_trampoline[i]; + } + + /* fill in every page-relative displacement now that the image is placed */ + if (!patch_page_relatives(page, base)) { + return false; + } + + serial_puts("ap: trampoline page at "); + serial_put_hex(base); + serial_puts(" ("); + serial_put_hex(bytes); + serial_puts(" bytes), sipi vector "); + serial_put_hex(base >> 12); + serial_puts(", pml4 "); + serial_put_hex(ap_pml4_base); + serial_puts("\n"); + + trampoline_base = base; + ap_stage_ptr = &page[(uint64_t)ap_stage - (uint64_t)ap_trampoline]; + *out_base = base; + return true; +} + +/* + * point the trampoline at this AP's per-CPU area, stack and entry point. + * done per AP immediately before its SIPI, since one shared trampoline page + * serves all of them and they are brought up one at a time. + */ +static void patch_trampoline(uint8_t slot) { + /* offsets of the patch slots, relative to the start of the image */ + const uint64_t percpu_off = (uint64_t)&ap_percpu - (uint64_t)ap_trampoline; + const uint64_t stack_off = (uint64_t)&ap_stack_top - (uint64_t)ap_trampoline; + const uint64_t entry_off = (uint64_t)&ap_entry - (uint64_t)ap_trampoline; + const uint64_t slot_off = (uint64_t)&ap_slot - (uint64_t)ap_trampoline; + + /* the page is written here and executed by another core: keep it volatile */ + volatile uint8_t *page = (volatile uint8_t *)trampoline_base; + + /* + * the per-CPU area and stack are chosen by `slot`, a dense index, rather + * than by APIC ID: an x2APIC MADT can report an ID larger than the number of + * per-CPU slots reserved, and using it as an index would run off the array. + */ + *(volatile uint64_t *)&page[percpu_off] = percpu_area(slot); + *(volatile uint64_t *)&page[stack_off] = percpu_stack_top(slot); + *(volatile uint64_t *)&page[entry_off] = (uint64_t)&ap_main; + + /* + * the AP re-reads its own APIC ID from the LAPIC rather than being told, so + * the slot travels with it and is handed to percpu_attach() there. + */ + *(volatile uint64_t *)&page[slot_off] = (uint64_t)slot; + + /* + * point this AP's GDT TSS descriptor at the TSS for its slot. + * + * the descriptor lives inside the copied page, so it is addressed by + * page-base plus its within-page offset, exactly like the pointer record + * patched in patch_page_relatives(). the descriptor is 16 bytes: base low 2, + * base mid 1, access 1, limit/flags 1, base high 1, then the upper base. + * + * access byte 0x89 marks it present, ring0 and available; without it the LTR + * in the trampoline raises #GP, which faults, and the AP triple faults into a + * reset that is indistinguishable from "it never ran". + */ + { + volatile uint8_t *desc = &page[AP_GDT_TSS_WITHIN]; + uint64_t tss = gdt_ap_tss_base(slot); + + desc[2] = (uint8_t)((tss >> 16) & 0xFF); /* base mid */ + desc[3] = 0x89; /* present, ring0, 64-bit TSS */ + desc[5] = (uint8_t)((tss >> 24) & 0xFF); /* base high */ + desc[6] = (uint8_t)(tss & 0xFF); /* base bits 32..39 */ + desc[7] = (uint8_t)((tss >> 40) & 0xFF); /* base bits 40..47 */ + } +} + +/* runs on the AP, in long mode */ +static void ap_main(void) { + uint32_t id = lapic_id(); + + /* + * the BSP patched this AP's slot into the trampoline before the SIPI. read it + * through the same volatile slot the trampoline image declares, so the + * compiler cannot cache it or fold the load. + */ + percpu_attach(id, (uint8_t)ap_slot); + + /* + * tell the BSP we are alive and have our own GS base. this must be the + * locked add and must come before the serial output below, because the BSP + * treats the counter as the completion signal: if it observed the counter + * first it may start patching the trampoline page for the next AP while this + * one is still printing, and the two would interleave on the UART. + */ + (void)atomic_fetch_add_u32(&ap_online_count, 1); + + serial_puts("ap: "); + serial_put_hex(id); + serial_puts(" online\n"); + + /* + * nothing to schedule yet. park with interrupts masked: an unhandled + * interrupt would reach the default dispatcher and halt the whole kernel. + */ + for (;;) { + cpu_cli(); + cpu_hlt(); + } +} + +static void delay_loops(uint32_t n) { + volatile uint32_t i; + + for (i = 0; i < n; i++) { + } +} + +/* + * busy-wait while latching the trampoline's progress marker. + * + * this has to cover the wait AFTER the SIPIs, not just the wait after them in + * the caller: an AP runs its whole trampoline and faults within nanoseconds of + * the SIPI landing, and a fault with no IDT becomes a triple fault and a reset, + * after which the reset processor overwrites the page. by the time the caller + * starts sampling, the evidence is usually already gone. + */ +static void delay_and_latch(uint32_t n) { + uint32_t i; + + for (i = 0; i < n; i++) { + if (ap_stage_ptr != NULL && *ap_stage_ptr > ap_stage_best) { + ap_stage_best = *ap_stage_ptr; + } + } +} + +static uint32_t start_one_ap(uint32_t apic_id, uint8_t sipi_vector) { + uint32_t rc = 0; + /* + * INIT is level-triggered and needs both halves. + * + * the assert (ICR bit 14 set) drives the processor into the reset-hold + * state. only once it is held does the deassert mean anything: it releases + * the AP into real mode at the reset vector, spinning there until a SIPI + * redirects it. sending the deassert alone, with bit 14 clear as this code + * originally did, leaves the AP wherever the firmware had it and + * the subsequent SIPIs go nowhere, so the AP never runs the trampoline. + */ + rc |= lapic_ipi_init_assert(apic_id); + delay_loops(200000); + + rc |= lapic_ipi_init_deassert(apic_id); + delay_loops(200000); + + /* + * two SIPIs are required: the first may be ignored if the processor has not + * finished resetting, the second always lands. the vector selects the page + * to jump to, so it must be the page we copied the trampoline into. + */ + rc |= lapic_ipi_sipi(apic_id, sipi_vector); + delay_and_latch(100000); + + rc |= lapic_ipi_sipi(apic_id, sipi_vector); + delay_and_latch(100000); + + return rc; +} + +uint32_t ap_boot_start(const struct acpi_madt_info *madt) { + uint32_t bsp_id = lapic_id(); + uint8_t sipi_vector = (uint8_t)(trampoline_base >> 12); + uint32_t started = 0; + uint32_t i; + const uint64_t stage_off = (uint64_t)ap_stage - (uint64_t)ap_trampoline; + + ap_online_count = 0; + + serial_puts("ap: starting from apic "); + serial_put_hex(bsp_id); + serial_puts(", vector "); + serial_put_hex(sipi_vector); + serial_puts("\n"); + + for (i = 0; i < madt->cpu_count; i++) { + uint32_t id = madt->apic_ids[i]; + uint8_t slot = madt->slots[i]; + uint32_t before; + uint32_t spins; + uint8_t stage = 0; + + if (id == bsp_id) { + continue; + } + + before = atomic_load_u32(&ap_online_count); + + /* fresh page per AP, see reload_trampoline() */ + reload_trampoline(); + patch_trampoline(slot); + + ap_stage_best = 0; + ap_stage_ptr = &((volatile uint8_t *)trampoline_base)[stage_off]; + + /* + * a non-zero return means the destination ID was outside what the xAPIC + * ICR can name, so no INIT or SIPI was sent at all. report that as a + * failure to start rather than waiting out the spin below for an AP that + * was never going to run. + */ + if (start_one_ap(id, sipi_vector) != 0) { + serial_puts("ap: "); + serial_put_hex(id); + serial_puts(" UNADDRESSABLE (x2apic id needs IA32_X2APIC_ICR)\n"); + continue; + } + + for (spins = 0; spins < AP_START_SPINS; spins++) { + if (atomic_load_u32(&ap_online_count) > before) { + break; + } + } + + stage = ap_stage_best; + + if (atomic_load_u32(&ap_online_count) > before) { + started++; + } else { + serial_puts("ap: "); + serial_put_hex(id); + serial_puts(" TIMEOUT stage="); + serial_put_hex(stage); + serial_puts("\n"); + } + } + + return started; +} \ No newline at end of file
arch/x86_64/asm.S added +363 -0
diff --git a/arch/x86_64/asm.S b/arch/x86_64/asm.S new file mode 100644 index 0000000000000000000000000000000000000000..439989aad7d0dad3a43871d5db9e652e5d225edd --- /dev/null +++ b/arch/x86_64/asm.S @@ -0,0 +1,363 @@ +/* + * low-level x86_64 primitives for furix. + * + * these live in a real assembly file rather than inline asm so that the exact + * instruction encodings are visible, reviewable and assembled by `as` instead of + * the compiler. all functions follow the Microsoft x64 ABI (the kernel is built + * for x86_64-unknown-windows): first argument in RCX, return value in RAX. + */ + +.text + +/* ports */ + +.globl inb + + +inb: + + movl %ecx, %edx + xorb %al, %al + inb %dx, %al + ret + +.globl outb + + +outb: + + movl %edx, %eax /* value -> AL */ + movl %ecx, %edx /* port -> DX (8-bit OUT needs DX) */ + outb %al, %dx + ret + +/* interrupts */ + +.globl cpu_cli + + +cpu_cli: + + cli + ret + +.globl cpu_sti + + +cpu_sti: + + sti + ret + +.globl cpu_hlt + + +cpu_hlt: + + hlt + ret + +/* + * atomic_fetch_add(ptr, val): returns the OLD value, so the result is usable + * as a completion index. + * + * `lock xadd` is a single bus transaction: no other CPU can interleave a read + * and a write of the same location. without the lock prefix the two halves race + * and two CPUs adding to the same counter can both read the same old value and + * one increment is lost. + * + * arg0 = ptr (RCX), arg1 = val (RDX), per the Microsoft x64 ABI. the result + * lands in EAX, which is only the low half ; callers counting events want the + * 32-bit form, so keep the operand and result 32-bit throughout. + */ +.globl atomic_fetch_add_u32 + + +atomic_fetch_add_u32: + + movl %edx, %eax /* EAX = value to add */ + lock xaddl %eax, (%rcx) /* EAX = old, memory += value */ + ret + +/* + * test_and_set(ptr): atomically read *ptr, and write 1 to it. returns the + * value it read, so nonzero means "somebody else got here first". + * + * this is the acquire primitive for the serial lock. `xchg` with a memory + * operand is implicitly locked on x86, so no separate lock prefix is needed. + * the write is a full barrier, which is what makes the caller's own flag check + * after a successful claim unnecessary. + */ +.globl atomic_test_and_set_u8 + + +atomic_test_and_set_u8: + + movzbl (%rcx), %eax + movl $1, %edx + xchgb %dl, (%rcx) + ret + +/* + * atomic_load_u32(ptr): a plain aligned 32-bit load, which x86 already makes + * atomic against concurrent stores. spelled as a function so the volatile + * read cannot be reordered or elided by the compiler at a call site. + */ +.globl atomic_load_u32 + + +atomic_load_u32: + + movl (%rcx), %eax + ret + +/* + * cpu_irq_save(): disable interrupts and return the previous EFLAGS word, so + * the caller can restore the prior IF state exactly rather than assuming it. + * the bit is forced clear in the returned value so that a caller who ignores + * the result still leaves interrupts off. + */ +.globl cpu_irq_save + + +cpu_irq_save: + + pushfq + popq %rax + pushfq + popq %rcx + andl $0xfffffdff, %ecx /* clear IF in the copy we return */ + pushq %rcx + popfq /* interrupts now off */ + ret + +/* cpu_irq_restore(flags = ECX): re-enable interrupts iff the saved word had IF. */ + +.globl cpu_irq_restore + + +cpu_irq_restore: + + pushq %rcx + orl $0x200, %ecx /* normalise IF to 1 */ + testb $1, %cl /* CL holds the saved IF */ + jne 1f + andl $0xfffffdff, %ecx + + +1: + + popfq + ret + +/* read the IF flag on its own: 1 if interrupts are enabled, else 0. */ + +.globl cpu_irqs_enabled + + +cpu_irqs_enabled: + + pushfq + popq %rax + andl $0x200, %eax + xorl %eax, %eax /* normalise to 0 or 1 */ + ret + +/* control registers */ + +.globl read_cr2 + + +read_cr2: + + movq %cr2, %rax + ret + +.globl read_cr3 + + +read_cr3: + + movq %cr3, %rax + ret + +/* + * MOV to/from a control register in long mode uses a 64-bit operand, so these + * must be 64-bit moves. (A 32-bit `mov %eax, %cr4` is rejected: in long mode + * that form does not encode.) + */ +.globl read_cr4 + + +read_cr4: + + movq %cr4, %rax + ret + +.globl write_cr4 + + +write_cr4: + + movq %rcx, %cr4 + ret + +/* MSRs */ + +.globl rdmsr + + +rdmsr: + + rdmsr + shlq $32, %rdx + orq %rdx, %rax + ret + +/* wrmsr(msr = ECX, value = RDX) */ +.globl wrmsr + + +wrmsr: + + movq %rdx, %r8 + movl %r8d, %eax /* EAX = low 32 */ + shrq $32, %r8 /* 64-bit shift: 32-bit would mask to 0 */ + movl %r8d, %edx /* EDX = high 32 */ + wrmsr + ret + +.data +.balign 8 +.globl cpuid_out + + +cpuid_out: + + .quad 0 + .quad 0 + .quad 0 + .quad 0 +.text + +/* CPUID with the leaf in ECX; the four result registers land in cpuid_out. */ +.globl cpuid + + +cpuid: + + movl %ecx, %eax + xorl %ecx, %ecx + cpuid + movq %rax, cpuid_out(%rip) + movq %rbx, cpuid_out+8(%rip) + movq %rcx, cpuid_out+16(%rip) + movq %rdx, cpuid_out+24(%rip) + ret + +/* descriptor table registers */ + +.globl lgdt + + +lgdt: + + lgdt (%rcx) + ret + +.globl lidt + + +lidt: + + lidt (%rcx) + ret + +.globl ltr + + +ltr: + + movl %ecx, %eax + ltr %ax + ret + +/* + * CS cannot be reloaded with mov. push the new selector and a return address, + * then far-return into it. + */ +.globl reload_cs + + +reload_cs: + + leaq 1f(%rip), %rax + pushq %rcx + pushq %rax + lretq + + +1: + + ret + +.globl load_data_segments + + +load_data_segments: + + movl %ecx, %eax + movw %ax, %ds + movw %ax, %es + movw %ax, %ss + ret + +/* per-CPU (GS) access */ + +.globl swapgs + + +swapgs: + + swapgs + ret + +/* + * wrgsbase/rdfsbase require CR4.FSGSBASE, which in turn requires the CPU to + * advertise CPUID.(EAX=7,ECX=0):EBX[0]. qemu's default model (qemu64) leaves + * that bit clear, and setting CR4.FSGSBASE there does not come back as a + * usable #GP, the guest simply stops. prefer IA32_KERNEL_GS_BASE + swapgs + * (see percpu_init), which works everywhere. + */ +.globl wrgsbase + + +wrgsbase: + + wrgsbaseq %rcx + ret + +.globl rdfsbase + + +rdfsbase: + + rdfsbaseq %rax + ret + +/* GS-relative access to the per-CPU area, which starts at GS.base + 0. */ +.globl gs_store + + +gs_store: + + movq %rcx, %gs:0 + ret + +.globl gs_load + + +gs_load: + + movq %gs:0, %rax + ret
arch/x86_64/entry.S deleted +0 -177
diff --git a/arch/x86_64/entry.S b/arch/x86_64/entry.S deleted file mode 100644 index 4c272669b12268577c8d8a563ccd1e8e5a66f853..0000000000000000000000000000000000000000 --- a/arch/x86_64/entry.S +++ /dev/null @@ -1,177 +0,0 @@ -.section .text - -.set PAGE_PRESENT, 1 << 0 -.set PAGE_WRITE, 1 << 1 -.set PAGE_SIZE, 1 << 7 - -.code32 -.global _start -_start: - mov $stack_top, %esp - - mov %ebx, mb_info_ptr - - # clear bss - - mov $__bss_start, %ecx - jmp .bss_zero_check -.bss_zero_loop: - movl $0, (%ecx) - add $4, %ecx -.bss_zero_check: - cmp $__bss_end, %ecx - jb .bss_zero_loop - - # fail if not loaded by multiboot2 - - cmp $MB_BOOTLOADER_MAGIC, %eax - jne .boot_failure - - # map first gb to page_table_l2 - - mov $0, %ecx -.page_loop: - mov %ecx, %eax - shl $21, %eax - - or $(PAGE_PRESENT | PAGE_WRITE | PAGE_SIZE), %eax - mov %eax, page_table_l2(,%ecx,8) - - inc %ecx - cmp $512, %ecx - jne .page_loop - - # the rest of the levels - - mov $page_table_l2, %eax - or $(PAGE_PRESENT | PAGE_WRITE), %eax - mov %eax, page_table_l3 - - mov $page_table_l3, %eax - or $(PAGE_PRESENT | PAGE_WRITE), %eax - mov %eax, page_table_l4 - - # page table addr to CR3 - - mov $page_table_l4, %eax - mov %eax, %cr3 - - # enable PAE (CR4.PAE) - - mov %cr4, %eax - or $(1 << 5), %eax - mov %eax, %cr4 - - # enable long mode (EFER.LME) - - mov $0xc0000080, %ecx - rdmsr - or $(1 << 8), %eax - wrmsr - - # enable paging (CR0.PG) - - mov %cr0, %eax - or $(1 << 31), %eax - mov %eax, %cr0 - - # enable GDT and jump to long mode - - lgdt gdt64_ptr - ljmp $0x08, $long_main - -.boot_failure: - # TODO: more failure cases liek idk cpu doesnt support long mode - - movb $'e', 0xb8000 - movb $' ', 0xb8001 # green - - cli -.halt32: - hlt - jmp .halt32 - -.code64 -long_main: - mov $stack_top, %rsp - - mov $0, %ax - mov %ax, %ds - mov %ax, %es - mov %ax, %fs - mov %ax, %gs - mov %ax, %ss - - call kernel_main - - cli -.halt64: - hlt - jmp .halt64 - -.section .rodata - -mb_info_ptr: - .space 4 - -.set GDT_FL_LONG, 1 << 53 -.set GDT_ACC_RDWR, 1 << 41 -.set GDT_ACC_EXEC, 1 << 43 -.set GDT_ACC_NOTSYS, 1 << 44 -.set GDT_ACC_PRESENT, 1 << 47 - -gdt64: - .quad 0 - .quad GDT_ACC_PRESENT | GDT_ACC_NOTSYS | GDT_ACC_RDWR | GDT_ACC_EXEC | GDT_FL_LONG - .quad GDT_ACC_PRESENT | GDT_ACC_NOTSYS | GDT_ACC_RDWR -gdt64_ptr: - .word . - gdt64 - 1 - .quad gdt64 - -.section .bss - -.align 4096 -page_table_l4: - .space 4096 -page_table_l3: - .space 4096 -page_table_l2: - .space 4096 - -.align 16 -stack_bottom: - .space 1024 * 16 -stack_top: - -.section .multiboot, "a" - -.set MB_BOOTLOADER_MAGIC, 0x36d76289 - -.set MB_MAGIC, 0xE85250D6 -.set MB_ARCH, 0 -.set MB_LENGTH, (multiboot_hdr_end - multiboot_hdr) -.set MB_CHECKSUM, -(MB_LENGTH + MB_ARCH + MB_MAGIC) - -.align 8 -multiboot_hdr: - .long MB_MAGIC - .long MB_ARCH - .long MB_LENGTH - .long MB_CHECKSUM -multiboot_hdr_end: - -.set MB_TAG_END, 0 -.set MB_TAG_INFO_REQ, 1 - -.set MB_INFO_MMAP, 6 - -.align 8 - .word MB_TAG_INFO_REQ - .word 0 - .long 8 + 4 - .long MB_INFO_MMAP - -.align 8 - .word MB_TAG_END - .word 0 - .long 8
arch/x86_64/gdt.c added +133 -0
diff --git a/arch/x86_64/gdt.c b/arch/x86_64/gdt.c new file mode 100644 index 0000000000000000000000000000000000000000..8300773b46158e62de0f01a2c8bdf871215e4f0a --- /dev/null +++ b/arch/x86_64/gdt.c @@ -0,0 +1,133 @@ +#include <gdt.h> + +#include <io.h> +#include <percpu.h> +#include <types.h> + +struct gdt_entry { + uint16_t limit_low; + uint16_t base_low; + uint8_t base_mid; + uint8_t access; + uint8_t granularity; + uint8_t base_high; +} __attribute__((packed)); + +/* system descriptors are 16 bytes */ +struct tss_desc { + uint16_t limit_low; + uint16_t base_low; + uint8_t base_mid; + uint8_t access; + uint8_t granularity; + uint8_t base_high; + uint32_t base_upper; + uint32_t reserved; +} __attribute__((packed)); + +struct gdt_table { + struct gdt_entry null; + struct gdt_entry code; + struct gdt_entry data; + struct tss_desc tss; +} __attribute__((packed)); + +static struct gdt_table gdt; +static struct desc_ptr gdtr; + +static struct tss tss_area __attribute__((aligned(16))); + +/* IST1: #DF, IST2: NMI. dedicated stacks, so a fault that overflowed (or + corrupted) the current stack is still handled */ +static uint8_t ist1_stack[IST_STACK_SIZE] __attribute__((aligned(16))); +static uint8_t ist2_stack[IST_STACK_SIZE] __attribute__((aligned(16))); + +static struct gdt_entry gdt_make_entry(uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) { + struct gdt_entry entry; + + entry.limit_low = (uint16_t)(limit & 0xFFFF); + entry.base_low = (uint16_t)(base & 0xFFFF); + entry.base_mid = (uint8_t)((base >> 16) & 0xFF); + entry.access = access; + entry.granularity = (uint8_t)((flags & 0xF0) | ((limit >> 16) & 0x0F)); + entry.base_high = (uint8_t)((base >> 24) & 0xFF); + + return entry; +} + +static struct tss_desc tss_make_desc(const struct tss *tss) { + uint64_t base = (uint64_t)tss; + uint32_t limit = (uint32_t)(sizeof(struct tss) - 1); + struct tss_desc desc; + + desc.limit_low = (uint16_t)(limit & 0xFFFF); + desc.base_low = (uint16_t)(base & 0xFFFF); + desc.base_mid = (uint8_t)((base >> 16) & 0xFF); + desc.access = 0x89; /* present, ring0, 64-bit TSS available */ + desc.granularity = (uint8_t)(((limit >> 16) & 0x0F) << 4 | 0x01); + desc.base_high = (uint8_t)((base >> 24) & 0xFF); + desc.base_upper = (uint32_t)((base >> 32) & 0xFFFFFFFF); + desc.reserved = 0; + + return desc; +} + +/* + * per-AP TSSes and their IST stacks. + * + * one pair per AP rather than one shared TSS: a TSS holds the IST stack + * POINTERS, so a single shared one would leave every AP using the same #DF + * stack. two processors pushing frames onto one stack is precisely the + * corruption an IST is meant to survive, so this cannot be shared. + * + * the AP's GDT (inside its trampoline page) gets a descriptor pointing at the + * TSS for that AP's slot; see patch_trampoline() in ap_boot.c. + */ +static struct tss ap_tss[PERCPU_MAX_CPUS] __attribute__((aligned(16))); +static uint8_t ap_ist_df[PERCPU_MAX_CPUS][IST_STACK_SIZE] __attribute__((aligned(16))); +static uint8_t ap_ist_nmi[PERCPU_MAX_CPUS][IST_STACK_SIZE] __attribute__((aligned(16))); + +/* zero a TSS and point both IST slots at the stacks for `slot` */ +static void tss_init_ist(struct tss *t, uint8_t slot) { + t->reserved0 = 0; + t->reserved0_pad = 0; + t->reserved1 = 0; + t->reserved2 = 0; + /* stacks grow down, so the usable top of each stack is its end */ + t->ist[IST_DOUBLE_FAULT - 1] = (uint64_t)(&ap_ist_df[slot][0] + IST_STACK_SIZE); + t->ist[IST_NMI - 1] = (uint64_t)(&ap_ist_nmi[slot][0] + IST_STACK_SIZE); + /* no I/O permission bitmap: the IOMap Base field equals the TSS length */ + t->iomap_base = (uint16_t)sizeof(struct tss); +} + +uint64_t gdt_ap_tss_base(uint8_t slot) { + tss_init_ist(&ap_tss[slot], slot); + return (uint64_t)&ap_tss[slot]; +} + +void gdt_init(void) { + tss_area.reserved0 = 0; + tss_area.reserved0_pad = 0; + tss_area.reserved1 = 0; + tss_area.reserved2 = 0; + /* stacks grow down, so point at the top */ + tss_area.ist[IST_DOUBLE_FAULT - 1] = (uint64_t)(ist1_stack + IST_STACK_SIZE); + tss_area.ist[IST_NMI - 1] = (uint64_t)(ist2_stack + IST_STACK_SIZE); + /* no I/O permission bitmap: the IOMap Base field equals the TSS length */ + tss_area.iomap_base = (uint16_t)sizeof(struct tss); + + gdt.null = gdt_make_entry(0, 0, 0x00, 0x00); + gdt.code = gdt_make_entry(0, 0xFFFFF, 0x9A, 0xA0); + gdt.data = gdt_make_entry(0, 0xFFFFF, 0x92, 0xA0); + gdt.tss = tss_make_desc(&tss_area); + + gdtr.limit = (uint16_t)(sizeof(gdt) - 1); + gdtr.base = (uint64_t)&gdt; + lgdt(&gdtr); + + load_data_segments(SEGSEL_KERNEL_DATA); + reload_cs(SEGSEL_KERNEL_CODE); + + /* load the task register only after the new GDT is live */ + ltr(SEGSEL_TSS); +}
arch/x86_64/idt.c added +217 -0
diff --git a/arch/x86_64/idt.c b/arch/x86_64/idt.c new file mode 100644 index 0000000000000000000000000000000000000000..bd0c232edea867404eca7d45d6080e88c03b7279 --- /dev/null +++ b/arch/x86_64/idt.c @@ -0,0 +1,217 @@ +#include <idt.h> + +#include <gdt.h> +#include <io.h> +#include <lapic.h> +#include <serial.h> +#include <types.h> + +/* interrupt gate, present, ring0, 64-bit */ +#define IDT_GATE_INTERRUPT 0x8E + +/* IST indices, 1-based; 0 means "use the current stack" */ +#define IDT_IST_NONE 0 + +struct idt_entry { + uint16_t offset_low; + uint16_t selector; + uint8_t ist; + uint8_t type_attr; + uint16_t offset_mid; + uint32_t offset_high; + uint32_t zero; +} __attribute__((packed)); + +static struct idt_entry idt[256]; +static struct desc_ptr idtr; + +/* + * a second table for application processors, published as idt_ap_pointer. + * + * the BSP's table cannot be reused as-is: its gates select 0x08, which is a + * 32-bit code segment, and a long-mode interrupt gate must name a 64-bit one, + * so that selector raises #GP instead of delivering the vector. + * + * IST indexes used to be a second reason, but are no longer: each AP now loads + * its own TSS from its trampoline GDT (see gdt_ap_tss_base), so the same IST + * stacks the BSP uses are safe here too and the gates carry the same indexes. + * + * 4 KiB, written once at bring-up, so the duplication costs nothing measurable. + */ +static struct idt_entry idt_ap[256]; +struct desc_ptr idt_ap_pointer; + +/* one entry per vector; NULL means "report and halt" */ +static void (*handlers[256])(void); + +/* filled in by isr.S: isr_stub_table[vector] is the stub for that vector */ +extern uint64_t isr_stub_table[]; + +/* the stub table and this struct must agree or every frame is misinterpreted. */ +typedef char assert_frame_vector[(offsetof(struct interrupt_frame, vector) == 15 * 8) ? 1 : -1]; +typedef char assert_frame_error[(offsetof(struct interrupt_frame, error_code) == 16 * 8) ? 1 : -1]; +typedef char assert_frame_rip[(offsetof(struct interrupt_frame, rip) == 17 * 8) ? 1 : -1]; +typedef char assert_frame_rflags[(offsetof(struct interrupt_frame, rflags) == 19 * 8) ? 1 : -1]; +typedef char assert_frame_rsp[(offsetof(struct interrupt_frame, rsp) == 20 * 8) ? 1 : -1]; + +static void idt_set_gate(uint16_t vector, uint64_t handler, uint8_t ist) { + idt[vector].offset_low = (uint16_t)(handler & 0xFFFF); + idt[vector].selector = 0x08; /* kernel code, from gdt.c */ + idt[vector].ist = ist; + idt[vector].type_attr = IDT_GATE_INTERRUPT; + idt[vector].offset_mid = (uint16_t)((handler >> 16) & 0xFFFF); + idt[vector].offset_high = (uint32_t)(handler >> 32); + idt[vector].zero = 0; +} + +/* + * the AP twin of idt_set_gate: no IST, and the trampoline's long-mode code + * selector (0x18) rather than the BSP's 32-bit one. see idt_ap above. + */ +static void idt_ap_set_gate(uint16_t vector, uint64_t handler, uint8_t ist) { + idt_ap[vector].offset_low = (uint16_t)(handler & 0xFFFF); + idt_ap[vector].selector = AP_CODE_SELECTOR_64; + idt_ap[vector].ist = ist; + idt_ap[vector].type_attr = IDT_GATE_INTERRUPT; + idt_ap[vector].offset_mid = (uint16_t)((handler >> 16) & 0xFFFF); + idt_ap[vector].offset_high = (uint32_t)(handler >> 32); + idt_ap[vector].zero = 0; +} + +static void spurious_handler(void); + +void idt_set_handler(uint8_t vector, void (*handler)(void)) { + handlers[vector] = handler; +} + +void idt_init(void) { + uint16_t vector; + + for (vector = 0; vector < 256; vector++) { + idt_set_gate(vector, isr_stub_table[vector], IDT_IST_NONE); + } + + /* + * IST1 for #DF and IST2 for NMI: dedicated stacks, so a fault that + * overflowed (or corrupted) the current stack is still handled. + */ + idt[VECTOR_DOUBLE_FAULT].ist = IST_DOUBLE_FAULT; + idt[VECTOR_NMI].ist = IST_NMI; + + handlers[VECTOR_SPURIOUS] = spurious_handler; + + idtr.limit = (uint16_t)(sizeof(idt) - 1); + idtr.base = (uint64_t)&idt; + lidt(&idtr); + + for (vector = 0; vector < 256; vector++) { + idt_ap_set_gate(vector, isr_stub_table[vector], IDT_IST_NONE); + } + + /* + * same IST indexes as the BSP table above. these are only safe now that each + * AP loads its own TSS before this table (the trampoline LTR precedes its + * LIDT); an IST gate with no TSS loaded raises #GP, and the #GP that reports + * it would itself fault, giving a triple fault and a reset that looks + * exactly like "the AP never started". + */ + idt_ap[VECTOR_DOUBLE_FAULT].ist = IST_DOUBLE_FAULT; + idt_ap[VECTOR_NMI].ist = IST_NMI; + + idt_ap_pointer.limit = (uint16_t)(sizeof(idt_ap) - 1); + idt_ap_pointer.base = (uint64_t)&idt_ap; +} + +/* + * a spurious interrupt was raised with IF already clear and nothing to + * acknowledge, so there is nothing to do but return. the LAPIC has already + * been EOI'd by the dispatcher. + */ +static void spurious_handler(void) { +} + +static void halt(void) { + for (;;) { + cpu_hlt(); + } +} + +static void print_error_bits(const char *label, uint64_t error_code) { + serial_puts(" "); + serial_puts(label); + serial_puts(" = "); + serial_put_hex((error_code >> 16) & 0xFFFF); + serial_puts("\n"); +} + +/* default for exceptions: decode what we can, then stop. */ +static void report_exception(const struct interrupt_frame *frame) { + uint64_t vector = frame->vector; + uint64_t error_code = frame->error_code; + + serial_puts("\nCPU EXCEPTION vector="); + serial_put_hex(vector); + serial_puts("\n err = "); + serial_put_hex(error_code); + serial_puts("\n"); + + if (vector == VECTOR_PAGE_FAULT) { + serial_puts(" page fault at "); + serial_put_hex(read_cr2()); + serial_puts("\n cause: "); + serial_puts((error_code & 0x1) ? "present" : "not-present"); + serial_puts(", "); + serial_puts((error_code & 0x2) ? "write" : "read"); + serial_puts(", "); + serial_puts((error_code & 0x4) ? "user" : "supervisor"); + serial_puts(", "); + serial_puts((error_code & 0x10) ? "instruction fetch" : "data access"); + serial_puts("\n"); + } else { + /* #DF / #GP / #SS style: bit0 IF, bit1 OTS, bit2 GPF, bit3 IDT, + bits 4..15 selector index, bit 15 RPL */ + serial_puts((error_code & 0x1) ? " [IF]\n" : ""); + serial_puts((error_code & 0x2) ? " [OTS]\n" : ""); + serial_puts((error_code & 0x4) ? " [GPF]\n" : ""); + serial_puts((error_code & 0x8) ? " [IDT]\n" : ""); + print_error_bits("selector", error_code); + } + + serial_puts(" rip = "); + serial_put_hex(frame->rip); + serial_puts("\n halted.\n"); +} + +/* + * single entry point for every vector. the LAPIC is acked before dispatch so a + * handler that itself takes a long time cannot block a second interrupt on + * the same vector. + */ +void interrupt_dispatch(const struct interrupt_frame *frame) { + uint8_t vector = (uint8_t)frame->vector; + + if (vector != VECTOR_PAGE_FAULT) { + lapic_eoi(); + } + + if (handlers[vector] != NULL) { + handlers[vector](); + return; + } + + /* exceptions 0..31 are unrecoverable here; no fixup path exists yet */ + if (vector < 32) { + report_exception(frame); + halt(); + } + + serial_puts("\nUNEXPECTED INTERRUPT vector="); + serial_put_hex(vector); + serial_puts(" rip="); + serial_put_hex(frame->rip); + serial_puts(" apic_id="); + serial_put_hex(lapic_id()); + serial_puts("\n"); + + halt(); +} \ No newline at end of file
arch/x86_64/ioapic.c added +66 -0
diff --git a/arch/x86_64/ioapic.c b/arch/x86_64/ioapic.c new file mode 100644 index 0000000000000000000000000000000000000000..53c7edc6334ac4007e9a250b9ae7ebf72f700520 --- /dev/null +++ b/arch/x86_64/ioapic.c @@ -0,0 +1,66 @@ +#include <ioapic.h> + +#include <io.h> +#include <types.h> + +/* IO-APIC indirect register file */ +#define IOAPIC_REGSEL 0x00 +#define IOAPIC_WINDOW 0x10 + +#define REG_ID 0x00 +#define REG_VER 0x01 +#define REG_REDIR 0x10 + +/* bit 16 of a redirection entry masks that input */ +#define REDIR_MASKED (1u << 16) + +/* redirection entry count lives in the version register's bits 23:16 */ +#define IOAPIC_MAX_ENTRIES 24 + +static volatile uint32_t *ioapic(void) { + return (volatile uint32_t *)(uint64_t)IOAPIC_DEFAULT_BASE; +} + +static uint32_t ioapic_read(uint32_t reg) { + /* + * index-then-read: writing the register number to REGSEL selects it, and the + * selected register is then read from WINDOW. + */ + *ioapic() = reg; + return *(volatile uint32_t *)((uint64_t)IOAPIC_DEFAULT_BASE + IOAPIC_WINDOW); +} + +static void ioapic_write(uint32_t reg, uint32_t value) { + *ioapic() = reg; + *(volatile uint32_t *)((uint64_t)IOAPIC_DEFAULT_BASE + IOAPIC_WINDOW) = value; +} + +void ioapic_disarm(void) { + uint32_t entries; + uint32_t i; + + /* + * mask first, before trusting the version register. entries are masked one + * at a time, so if anything is already routed it stops as soon as we reach + * it. + */ + for (i = 0; i < IOAPIC_MAX_ENTRIES; i++) { + uint32_t entry = ioapic_read(REG_REDIR + 2 * i); + + ioapic_write(REG_REDIR + 2 * i, entry | REDIR_MASKED); + } + + /* the version register encodes the entry count in bits 23:16 */ + entries = (ioapic_read(REG_VER) >> 16) & 0xFF; + + if (entries == 0 || entries > IOAPIC_MAX_ENTRIES) { + /* no usable count: mask everything the array can hold */ + entries = IOAPIC_MAX_ENTRIES; + } + + for (i = 0; i < entries; i++) { + uint32_t entry = ioapic_read(REG_REDIR + 2 * i); + + ioapic_write(REG_REDIR + 2 * i, entry | REDIR_MASKED); + } +} \ No newline at end of file
arch/x86_64/isr.S added +632 -0
diff --git a/arch/x86_64/isr.S b/arch/x86_64/isr.S new file mode 100644 index 0000000000000000000000000000000000000000..f310c188e35d5f5f5be2f26754e1005edde53c82 --- /dev/null +++ b/arch/x86_64/isr.S @@ -0,0 +1,632 @@ +/* + * interrupt and exception entry stubs. + * + * the CPU pushes, in order: [error code,] RIP, CS, RFLAGS, RSP [, SS]. vectors + * that have no error code do not push one, so each stub first normalises the + * frame to a single shape by supplying a zero, and pushes the vector number. + * the common path therefore always sees: + * + * [rsp+0] vector + * [rsp+8] error code (0 for vectors that do not push one) + * [rsp+16] RIP + * [rsp+24] CS + * [rsp+32] RFLAGS + * [rsp+40] RSP + * + * the common path then pushes the fifteen general-purpose registers, so the + * frame handed to C (struct interrupt_frame in idt.h) is: + * + * rax rbx rcx rdx rbp rsi rdi r8 r9 r10 r11 r12 r13 r14 r15 + * vector error_code rip cs rflags + * + * everything runs on the interrupted stack: our GDT has ring-0 code and data + * only, so there is no privilege change and therefore no stack switch. + * + * not yet done: the FPU/SSE/AVX state is not saved or restored. that is safe + * today because nothing touches the FPU from interrupt context, but it must be + * fixed before user-space transitions or any FPU use. + * + * the 256 stub bodies below are a mechanical expansion ; there is one per vector + * because only a stub that knows its own number can supply it. + */ + +.text + +.macro ISR_NOERR num + + +isr_stub_\num: + + pushq $0 /* no CPU-pushed error code */ + pushq $\num + jmp isr_common +.endm + +.macro ISR_ERR num + + +isr_stub_\num: + + pushq $\num /* error code is already on the stack */ + jmp isr_common +.endm + +/* vectors that push an error code: #DF 8, #TS 10, #NP 11, #SS 12, #GP 13, + + #PF 14, #AC 17, #CP 21, #VC 29, #SX 30. */ + ISR_NOERR 0 + ISR_NOERR 1 + ISR_NOERR 2 + ISR_NOERR 3 + ISR_NOERR 4 + ISR_NOERR 5 + ISR_NOERR 6 + ISR_NOERR 7 + ISR_ERR 8 + ISR_NOERR 9 + ISR_ERR 10 + ISR_ERR 11 + ISR_ERR 12 + ISR_ERR 13 + ISR_ERR 14 + ISR_NOERR 15 + ISR_NOERR 16 + ISR_ERR 17 + ISR_NOERR 18 + ISR_NOERR 19 + ISR_NOERR 20 + ISR_ERR 21 + ISR_NOERR 22 + ISR_NOERR 23 + ISR_NOERR 24 + ISR_NOERR 25 + ISR_NOERR 26 + ISR_NOERR 27 + ISR_NOERR 28 + ISR_ERR 29 + ISR_ERR 30 + ISR_NOERR 31 + ISR_NOERR 32 + ISR_NOERR 33 + ISR_NOERR 34 + ISR_NOERR 35 + ISR_NOERR 36 + ISR_NOERR 37 + ISR_NOERR 38 + ISR_NOERR 39 + ISR_NOERR 40 + ISR_NOERR 41 + ISR_NOERR 42 + ISR_NOERR 43 + ISR_NOERR 44 + ISR_NOERR 45 + ISR_NOERR 46 + ISR_NOERR 47 + ISR_NOERR 48 + ISR_NOERR 49 + ISR_NOERR 50 + ISR_NOERR 51 + ISR_NOERR 52 + ISR_NOERR 53 + ISR_NOERR 54 + ISR_NOERR 55 + ISR_NOERR 56 + ISR_NOERR 57 + ISR_NOERR 58 + ISR_NOERR 59 + ISR_NOERR 60 + ISR_NOERR 61 + ISR_NOERR 62 + ISR_NOERR 63 + ISR_NOERR 64 + ISR_NOERR 65 + ISR_NOERR 66 + ISR_NOERR 67 + ISR_NOERR 68 + ISR_NOERR 69 + ISR_NOERR 70 + ISR_NOERR 71 + ISR_NOERR 72 + ISR_NOERR 73 + ISR_NOERR 74 + ISR_NOERR 75 + ISR_NOERR 76 + ISR_NOERR 77 + ISR_NOERR 78 + ISR_NOERR 79 + ISR_NOERR 80 + ISR_NOERR 81 + ISR_NOERR 82 + ISR_NOERR 83 + ISR_NOERR 84 + ISR_NOERR 85 + ISR_NOERR 86 + ISR_NOERR 87 + ISR_NOERR 88 + ISR_NOERR 89 + ISR_NOERR 90 + ISR_NOERR 91 + ISR_NOERR 92 + ISR_NOERR 93 + ISR_NOERR 94 + ISR_NOERR 95 + ISR_NOERR 96 + ISR_NOERR 97 + ISR_NOERR 98 + ISR_NOERR 99 + ISR_NOERR 100 + ISR_NOERR 101 + ISR_NOERR 102 + ISR_NOERR 103 + ISR_NOERR 104 + ISR_NOERR 105 + ISR_NOERR 106 + ISR_NOERR 107 + ISR_NOERR 108 + ISR_NOERR 109 + ISR_NOERR 110 + ISR_NOERR 111 + ISR_NOERR 112 + ISR_NOERR 113 + ISR_NOERR 114 + ISR_NOERR 115 + ISR_NOERR 116 + ISR_NOERR 117 + ISR_NOERR 118 + ISR_NOERR 119 + ISR_NOERR 120 + ISR_NOERR 121 + ISR_NOERR 122 + ISR_NOERR 123 + ISR_NOERR 124 + ISR_NOERR 125 + ISR_NOERR 126 + ISR_NOERR 127 + ISR_NOERR 128 + ISR_NOERR 129 + ISR_NOERR 130 + ISR_NOERR 131 + ISR_NOERR 132 + ISR_NOERR 133 + ISR_NOERR 134 + ISR_NOERR 135 + ISR_NOERR 136 + ISR_NOERR 137 + ISR_NOERR 138 + ISR_NOERR 139 + ISR_NOERR 140 + ISR_NOERR 141 + ISR_NOERR 142 + ISR_NOERR 143 + ISR_NOERR 144 + ISR_NOERR 145 + ISR_NOERR 146 + ISR_NOERR 147 + ISR_NOERR 148 + ISR_NOERR 149 + ISR_NOERR 150 + ISR_NOERR 151 + ISR_NOERR 152 + ISR_NOERR 153 + ISR_NOERR 154 + ISR_NOERR 155 + ISR_NOERR 156 + ISR_NOERR 157 + ISR_NOERR 158 + ISR_NOERR 159 + ISR_NOERR 160 + ISR_NOERR 161 + ISR_NOERR 162 + ISR_NOERR 163 + ISR_NOERR 164 + ISR_NOERR 165 + ISR_NOERR 166 + ISR_NOERR 167 + ISR_NOERR 168 + ISR_NOERR 169 + ISR_NOERR 170 + ISR_NOERR 171 + ISR_NOERR 172 + ISR_NOERR 173 + ISR_NOERR 174 + ISR_NOERR 175 + ISR_NOERR 176 + ISR_NOERR 177 + ISR_NOERR 178 + ISR_NOERR 179 + ISR_NOERR 180 + ISR_NOERR 181 + ISR_NOERR 182 + ISR_NOERR 183 + ISR_NOERR 184 + ISR_NOERR 185 + ISR_NOERR 186 + ISR_NOERR 187 + ISR_NOERR 188 + ISR_NOERR 189 + ISR_NOERR 190 + ISR_NOERR 191 + ISR_NOERR 192 + ISR_NOERR 193 + ISR_NOERR 194 + ISR_NOERR 195 + ISR_NOERR 196 + ISR_NOERR 197 + ISR_NOERR 198 + ISR_NOERR 199 + ISR_NOERR 200 + ISR_NOERR 201 + ISR_NOERR 202 + ISR_NOERR 203 + ISR_NOERR 204 + ISR_NOERR 205 + ISR_NOERR 206 + ISR_NOERR 207 + ISR_NOERR 208 + ISR_NOERR 209 + ISR_NOERR 210 + ISR_NOERR 211 + ISR_NOERR 212 + ISR_NOERR 213 + ISR_NOERR 214 + ISR_NOERR 215 + ISR_NOERR 216 + ISR_NOERR 217 + ISR_NOERR 218 + ISR_NOERR 219 + ISR_NOERR 220 + ISR_NOERR 221 + ISR_NOERR 222 + ISR_NOERR 223 + ISR_NOERR 224 + ISR_NOERR 225 + ISR_NOERR 226 + ISR_NOERR 227 + ISR_NOERR 228 + ISR_NOERR 229 + ISR_NOERR 230 + ISR_NOERR 231 + ISR_NOERR 232 + ISR_NOERR 233 + ISR_NOERR 234 + ISR_NOERR 235 + ISR_NOERR 236 + ISR_NOERR 237 + ISR_NOERR 238 + ISR_NOERR 239 + ISR_NOERR 240 + ISR_NOERR 241 + ISR_NOERR 242 + ISR_NOERR 243 + ISR_NOERR 244 + ISR_NOERR 245 + ISR_NOERR 246 + ISR_NOERR 247 + ISR_NOERR 248 + ISR_NOERR 249 + ISR_NOERR 250 + ISR_NOERR 251 + ISR_NOERR 252 + ISR_NOERR 253 + ISR_NOERR 254 + ISR_NOERR 255 + +isr_common: + + cld /* the ABI requires DF = 0 on entry */ + + pushq %r15 + pushq %r14 + pushq %r13 + pushq %r12 + pushq %r11 + pushq %r10 + pushq %r9 + pushq %r8 + pushq %rdi + pushq %rsi + pushq %rbp + pushq %rbx + pushq %rdx + pushq %rcx + pushq %rax + + /* + * RBX was pushed above, so the live register is free scratch. RSP may no + * longer be 16-byte aligned after an odd number of pushes, so align it + * and keep the original in RBX for the restore. + * + * NB: the frame pointer goes in RCX, not RDI. the kernel targets + * x86_64-unknown-windows, so the Microsoft x64 ABI applies and arg0 is + * RCX. the System V convention would silently corrupt the dispatch. + */ + movq %rsp, %rbx + movq %rbx, %rcx /* arg0: const struct interrupt_frame * */ + andq $-16, %rsp + subq $0x20, %rsp /* 32 bytes of shadow space for the callee */ + call interrupt_dispatch + movq %rbx, %rsp + + popq %rax + popq %rcx + popq %rdx + popq %rbx + popq %rbp + popq %rsi + popq %rdi + popq %r8 + popq %r9 + popq %r10 + popq %r11 + popq %r12 + popq %r13 + popq %r14 + popq %r15 + + addq $0x10, %rsp /* drop the vector and the error code */ + iretq /* restores RIP, CS, RFLAGS (including IF) */ + +/* one 64-bit pointer per vector, consumed by idt_init() to fill the IDT. */ +.data +.balign 8 +.globl isr_stub_table + + +isr_stub_table: + + .quad isr_stub_0 + .quad isr_stub_1 + .quad isr_stub_2 + .quad isr_stub_3 + .quad isr_stub_4 + .quad isr_stub_5 + .quad isr_stub_6 + .quad isr_stub_7 + .quad isr_stub_8 + .quad isr_stub_9 + .quad isr_stub_10 + .quad isr_stub_11 + .quad isr_stub_12 + .quad isr_stub_13 + .quad isr_stub_14 + .quad isr_stub_15 + .quad isr_stub_16 + .quad isr_stub_17 + .quad isr_stub_18 + .quad isr_stub_19 + .quad isr_stub_20 + .quad isr_stub_21 + .quad isr_stub_22 + .quad isr_stub_23 + .quad isr_stub_24 + .quad isr_stub_25 + .quad isr_stub_26 + .quad isr_stub_27 + .quad isr_stub_28 + .quad isr_stub_29 + .quad isr_stub_30 + .quad isr_stub_31 + .quad isr_stub_32 + .quad isr_stub_33 + .quad isr_stub_34 + .quad isr_stub_35 + .quad isr_stub_36 + .quad isr_stub_37 + .quad isr_stub_38 + .quad isr_stub_39 + .quad isr_stub_40 + .quad isr_stub_41 + .quad isr_stub_42 + .quad isr_stub_43 + .quad isr_stub_44 + .quad isr_stub_45 + .quad isr_stub_46 + .quad isr_stub_47 + .quad isr_stub_48 + .quad isr_stub_49 + .quad isr_stub_50 + .quad isr_stub_51 + .quad isr_stub_52 + .quad isr_stub_53 + .quad isr_stub_54 + .quad isr_stub_55 + .quad isr_stub_56 + .quad isr_stub_57 + .quad isr_stub_58 + .quad isr_stub_59 + .quad isr_stub_60 + .quad isr_stub_61 + .quad isr_stub_62 + .quad isr_stub_63 + .quad isr_stub_64 + .quad isr_stub_65 + .quad isr_stub_66 + .quad isr_stub_67 + .quad isr_stub_68 + .quad isr_stub_69 + .quad isr_stub_70 + .quad isr_stub_71 + .quad isr_stub_72 + .quad isr_stub_73 + .quad isr_stub_74 + .quad isr_stub_75 + .quad isr_stub_76 + .quad isr_stub_77 + .quad isr_stub_78 + .quad isr_stub_79 + .quad isr_stub_80 + .quad isr_stub_81 + .quad isr_stub_82 + .quad isr_stub_83 + .quad isr_stub_84 + .quad isr_stub_85 + .quad isr_stub_86 + .quad isr_stub_87 + .quad isr_stub_88 + .quad isr_stub_89 + .quad isr_stub_90 + .quad isr_stub_91 + .quad isr_stub_92 + .quad isr_stub_93 + .quad isr_stub_94 + .quad isr_stub_95 + .quad isr_stub_96 + .quad isr_stub_97 + .quad isr_stub_98 + .quad isr_stub_99 + .quad isr_stub_100 + .quad isr_stub_101 + .quad isr_stub_102 + .quad isr_stub_103 + .quad isr_stub_104 + .quad isr_stub_105 + .quad isr_stub_106 + .quad isr_stub_107 + .quad isr_stub_108 + .quad isr_stub_109 + .quad isr_stub_110 + .quad isr_stub_111 + .quad isr_stub_112 + .quad isr_stub_113 + .quad isr_stub_114 + .quad isr_stub_115 + .quad isr_stub_116 + .quad isr_stub_117 + .quad isr_stub_118 + .quad isr_stub_119 + .quad isr_stub_120 + .quad isr_stub_121 + .quad isr_stub_122 + .quad isr_stub_123 + .quad isr_stub_124 + .quad isr_stub_125 + .quad isr_stub_126 + .quad isr_stub_127 + .quad isr_stub_128 + .quad isr_stub_129 + .quad isr_stub_130 + .quad isr_stub_131 + .quad isr_stub_132 + .quad isr_stub_133 + .quad isr_stub_134 + .quad isr_stub_135 + .quad isr_stub_136 + .quad isr_stub_137 + .quad isr_stub_138 + .quad isr_stub_139 + .quad isr_stub_140 + .quad isr_stub_141 + .quad isr_stub_142 + .quad isr_stub_143 + .quad isr_stub_144 + .quad isr_stub_145 + .quad isr_stub_146 + .quad isr_stub_147 + .quad isr_stub_148 + .quad isr_stub_149 + .quad isr_stub_150 + .quad isr_stub_151 + .quad isr_stub_152 + .quad isr_stub_153 + .quad isr_stub_154 + .quad isr_stub_155 + .quad isr_stub_156 + .quad isr_stub_157 + .quad isr_stub_158 + .quad isr_stub_159 + .quad isr_stub_160 + .quad isr_stub_161 + .quad isr_stub_162 + .quad isr_stub_163 + .quad isr_stub_164 + .quad isr_stub_165 + .quad isr_stub_166 + .quad isr_stub_167 + .quad isr_stub_168 + .quad isr_stub_169 + .quad isr_stub_170 + .quad isr_stub_171 + .quad isr_stub_172 + .quad isr_stub_173 + .quad isr_stub_174 + .quad isr_stub_175 + .quad isr_stub_176 + .quad isr_stub_177 + .quad isr_stub_178 + .quad isr_stub_179 + .quad isr_stub_180 + .quad isr_stub_181 + .quad isr_stub_182 + .quad isr_stub_183 + .quad isr_stub_184 + .quad isr_stub_185 + .quad isr_stub_186 + .quad isr_stub_187 + .quad isr_stub_188 + .quad isr_stub_189 + .quad isr_stub_190 + .quad isr_stub_191 + .quad isr_stub_192 + .quad isr_stub_193 + .quad isr_stub_194 + .quad isr_stub_195 + .quad isr_stub_196 + .quad isr_stub_197 + .quad isr_stub_198 + .quad isr_stub_199 + .quad isr_stub_200 + .quad isr_stub_201 + .quad isr_stub_202 + .quad isr_stub_203 + .quad isr_stub_204 + .quad isr_stub_205 + .quad isr_stub_206 + .quad isr_stub_207 + .quad isr_stub_208 + .quad isr_stub_209 + .quad isr_stub_210 + .quad isr_stub_211 + .quad isr_stub_212 + .quad isr_stub_213 + .quad isr_stub_214 + .quad isr_stub_215 + .quad isr_stub_216 + .quad isr_stub_217 + .quad isr_stub_218 + .quad isr_stub_219 + .quad isr_stub_220 + .quad isr_stub_221 + .quad isr_stub_222 + .quad isr_stub_223 + .quad isr_stub_224 + .quad isr_stub_225 + .quad isr_stub_226 + .quad isr_stub_227 + .quad isr_stub_228 + .quad isr_stub_229 + .quad isr_stub_230 + .quad isr_stub_231 + .quad isr_stub_232 + .quad isr_stub_233 + .quad isr_stub_234 + .quad isr_stub_235 + .quad isr_stub_236 + .quad isr_stub_237 + .quad isr_stub_238 + .quad isr_stub_239 + .quad isr_stub_240 + .quad isr_stub_241 + .quad isr_stub_242 + .quad isr_stub_243 + .quad isr_stub_244 + .quad isr_stub_245 + .quad isr_stub_246 + .quad isr_stub_247 + .quad isr_stub_248 + .quad isr_stub_249 + .quad isr_stub_250 + .quad isr_stub_251 + .quad isr_stub_252 + .quad isr_stub_253 + .quad isr_stub_254 + .quad isr_stub_255
arch/x86_64/lapic.c added +159 -0
diff --git a/arch/x86_64/lapic.c b/arch/x86_64/lapic.c new file mode 100644 index 0000000000000000000000000000000000000000..8e73b8cb75a780dae9d494f82804e95f3a5893ad --- /dev/null +++ b/arch/x86_64/lapic.c @@ -0,0 +1,159 @@ +#include <lapic.h> + +#include <io.h> +#include <types.h> + +#define IA32_APIC_BASE 0x1B + +#define APIC_BASE_ENABLE (1ull << 11) +#define APIC_BASE_ADDR_MASK 0xFFFFF000ull +#define APIC_DEFAULT_BASE 0xFEE00000u + +#define APIC_REG_ID 0x20 +#define APIC_REG_VERSION 0x30 +#define APIC_REG_EOI 0xB0 +#define APIC_REG_SPURIOUS 0xF0 +#define APIC_REG_TIMER 0x320 + +/* bit 16 of the timer control register masks it */ +#define APIC_TIMER_MASKED (1u << 16) +#define APIC_REG_ICR 0x300 +#define APIC_REG_ICR_HIGH 0x310 + +#define ICR_DELIVERY_STATUS 0x1000u /* bit 12: set while the IPI is in flight */ +#define ICR_TRIGGER_MODE 0x4000u /* bit 14: level vs edge */ +#define ICR_SHORTHAND_SHIFT 18 + +/* bounded so a lost IPI surfaces as a failed bringup, never a hang */ +#define ICR_WAIT_SPINS 1000000u + +#define SPURIOUS_VECTOR 0xFF + +static volatile uint32_t *apic_reg(uint32_t reg) { + return (volatile uint32_t *)(uint64_t)(lapic_base() + reg); +} + +void lapic_enable(void) { + uint64_t base = rdmsr(IA32_APIC_BASE); + + /* + * ORDER MATTERS: mask the timer BEFORE enabling the LAPIC. + * + * firmware leaves the LAPIC timer running on whatever vector its divide + * config selects, which is 0x20 here. enabling the LAPIC first opens the + * delivery path, so that timer can latch vector 0x20 in the ISR before the + * mask below lands. IF is clear at this point so the request is held pending + * rather than taken, and is delivered at the first `sti` much later, which + * is the self-IPI test. that lands on the default gate and halts the kernel, + * so the failure reads as "unexpected interrupt during the self-IPI test" + * with nothing pointing at the timer. + */ + *apic_reg(APIC_REG_TIMER) = APIC_TIMER_MASKED; + + *apic_reg(APIC_REG_SPURIOUS) = (uint32_t)(SPURIOUS_VECTOR | (1u << 8)); + + if ((base & APIC_BASE_ENABLE) == 0) { + wrmsr(IA32_APIC_BASE, base | APIC_BASE_ENABLE); + } + + /* + * drain anything firmware already had in flight. masking the LVT stops new + * requests but does not retract one already pending, and there is no way to + * read the ISR to find out what it was. acknowledge blindly: an EOI with + * nothing outstanding is a no-op. + */ + lapic_eoi(); +} + +uint32_t lapic_base(void) { + uint32_t base = (uint32_t)(rdmsr(IA32_APIC_BASE) & APIC_BASE_ADDR_MASK); + + if (base == 0) { + base = APIC_DEFAULT_BASE; + } + return base; +} + +uint32_t lapic_id(void) { + /* + * only the low 8 bits of the xAPIC ID register are meaningful; bits above + * that read back as zero on real hardware but are not architecturally + * guaranteed to, so mask rather than trust the read. widened from uint8_t + * because an x2APIC MADT reports 32-bit IDs; see lapic_ipi_send_flags for + * what happens to those. + */ + return (uint32_t)(*apic_reg(APIC_REG_ID) >> 24) & 0xffu; +} + +void lapic_eoi(void) { + *apic_reg(APIC_REG_EOI) = 0; +} + +bool lapic_ipi_wait(void) { + uint32_t spins = ICR_WAIT_SPINS; + + while ((*apic_reg(APIC_REG_ICR) & ICR_DELIVERY_STATUS) != 0) { + if (spins-- == 0) { + return false; + } + } + return true; +} + +/* + * returns 0 if the IPI was sent, non-zero if the destination could not be + * addressed at all (an x2APIC ID beyond the xAPIC ICR's 8-bit field). nothing + * is delivered in that case. + */ +uint32_t lapic_ipi_send_flags(uint32_t delivery, uint32_t shorthand, uint32_t vector, + uint32_t apic_id, uint32_t flags) { + /* + * the delivery-status bit reads from the high half of the 64-bit ICR, and + * the destination field only applies when shorthand is unused, so it is + * harmless to always program ICR_HIGH. + */ + if (shorthand == LAPIC_DEST) { + /* + * the xAPIC destination field is only bits 24..31 of ICR_HIGH, so it can + * name at most 256 processors. an ID above that comes from an x2APIC MADT + * entry and would be silently truncated, sending INIT to processor 0 and + * resetting the BSP instead of the intended AP. + * + * addressing those needs IA32_X2APIC_ICR (MSR 0x8B), which is only + * reachable once x2APIC mode is enabled in IA32_APIC_BASE. that is not done + * yet, so refuse loudly instead of misdirecting the IPI. callers treat a + * non-zero return as "this AP was not started". + */ + if (apic_id > 0xffu) { + return 1; + } + *apic_reg(APIC_REG_ICR_HIGH) = apic_id << 24; + } else { + *apic_reg(APIC_REG_ICR_HIGH) = 0; + } + + *apic_reg(APIC_REG_ICR) = vector | + (delivery << 8) | + (shorthand << ICR_SHORTHAND_SHIFT) | + flags; + + lapic_ipi_wait(); + return 0; +} + +uint32_t lapic_ipi_send(uint32_t delivery, uint32_t shorthand, uint32_t vector, + uint32_t apic_id) { + return lapic_ipi_send_flags(delivery, shorthand, vector, apic_id, 0); +} + +uint32_t lapic_ipi_init_assert(uint32_t apic_id) { + return lapic_ipi_send_flags(LAPIC_IPI_INIT, LAPIC_DEST, 0, apic_id, ICR_TRIGGER_MODE); +} + +uint32_t lapic_ipi_init_deassert(uint32_t apic_id) { + return lapic_ipi_send_flags(LAPIC_IPI_INIT, LAPIC_DEST, 0, apic_id, 0); +} + +uint32_t lapic_ipi_sipi(uint32_t apic_id, uint8_t vector) { + return lapic_ipi_send_flags(LAPIC_IPI_STARTUP, LAPIC_DEST, vector, apic_id, 0); +}
arch/x86_64/linker.ld deleted +0 -44
diff --git a/arch/x86_64/linker.ld b/arch/x86_64/linker.ld deleted file mode 100644 index 073426c50bb4f0e02d088f1d259a1034c9450b93..0000000000000000000000000000000000000000 --- a/arch/x86_64/linker.ld +++ /dev/null @@ -1,44 +0,0 @@ -ENTRY(_start) - -SECTIONS -{ - . = 1M; - - __kernel_start = ALIGN (4K); - - .multiboot ALIGN (4K) : - { - KEEP(*(.multiboot)) - } - - .text ALIGN (4K) : - { - *(.text) - *(.text.*) - } - - .rodata ALIGN (4K) : - { - *(.rodata) - *(.rodata.*) - } - - .data ALIGN (4K) : - { - *(.data) - *(.data.*) - } - - .bss ALIGN (4K) (NOLOAD) : - { - __bss_start = ALIGN(4); - - *(.bss) - *(.bss.*) - *(COMMON) - - __bss_end = ALIGN(4); - } - - __kernel_end = ALIGN(4K); -}
arch/x86_64/percpu.c added +87 -0
diff --git a/arch/x86_64/percpu.c b/arch/x86_64/percpu.c new file mode 100644 index 0000000000000000000000000000000000000000..737a011cdc7a4e5eb6a6532389996e74d8ac24dc --- /dev/null +++ b/arch/x86_64/percpu.c @@ -0,0 +1,87 @@ +#include <percpu.h> + +#include <io.h> +#include <types.h> + +/* + * 64-bit MSR numbers. the low-32 values often quoted for these (0xC5/0xC6) are + * NOT the GS base MSRs, and QEMU silently ignores writes to undefined MSRs, + * which is a confusing way to quietly lose your per-CPU base. + */ +#define MSR_IA32_GS_BASE 0xC0000101u +#define MSR_IA32_KERNEL_GS_BASE 0xC0000102u + +struct per_cpu { + /* must stay first: this is what %gs:0 refers to */ + uint64_t gs_probe; + uint32_t apic_id; + uint32_t online; +}; + +/* 64-byte aligned so a per-CPU struct never straddles a cache line */ +static struct per_cpu percpu[PERCPU_MAX_CPUS] __attribute__((aligned(64))); + +/* + * stacks live in one contiguous pool rather than inside struct per_cpu, so + * growing the per-CPU struct does not silently move every stack. + */ +static uint8_t percpu_stacks[PERCPU_MAX_CPUS][PERCPU_STACK_SIZE] __attribute__((aligned(16))); + +uint64_t percpu_address(void) { + return (uint64_t)&percpu[0]; +} + +uint64_t percpu_area(uint8_t slot) { + return (uint64_t)&percpu[slot]; +} + +uint64_t percpu_stack_top(uint8_t slot) { + return (uint64_t)&percpu_stacks[slot][PERCPU_STACK_SIZE]; +} + +void percpu_attach(uint32_t apic_id, uint8_t slot) { + /* + * deliberately NOT using CR4.FSGSBASE + wrgsbase. firmware leaves + * CR4.FSGSBASE clear and QEMU's default CPU model (qemu64) does not + * advertise CPUID.(7,0):EBX[0], so setting that CR4 bit does not raise a + * catchable #GP, the guest just stops. IA32_KERNEL_GS_BASE + swapgs gets + * the same result with no feature dependency. + * + * swapgs is safe on a freshly INIT'd AP: GS.base is still 0 there, so the + * swap cannot lose a kernel GS base. + */ + wrmsr(MSR_IA32_KERNEL_GS_BASE, (uint64_t)&percpu[slot]); + swapgs(); + + /* + * record the identity AFTER the swap, so it is read and written through the + * GS base just installed. percpu_apic_id() is otherwise only meaningful on + * the BSP, which has no AP to be started by. + */ + percpu[slot].apic_id = apic_id; + percpu[slot].online = 1; +} + +void percpu_init(void) { + percpu_attach(0, 0); +} + +void percpu_set_id(uint32_t apic_id) { + percpu[0].apic_id = apic_id; +} + +uint32_t percpu_apic_id(void) { + return percpu[0].apic_id; +} + +uint64_t percpu_gs_selftest(uint64_t value) { + gs_store(value); + + return gs_load(); +} + +uint64_t percpu_gs_selftest_via_c(uint64_t value) { + gs_store(value); + + return percpu[0].gs_probe; +} \ No newline at end of file
arch/x86_64/pic.c added +59 -0
diff --git a/arch/x86_64/pic.c b/arch/x86_64/pic.c new file mode 100644 index 0000000000000000000000000000000000000000..70e5d5318e6565dcb9fc5d89667fd4d165e4de8c --- /dev/null +++ b/arch/x86_64/pic.c @@ -0,0 +1,59 @@ +#include <pic.h> + +#include <io.h> +#include <types.h> + +/* + * 8259A master/slave. the slave sits on master IRQ2, so it must be masked + * before the master's IRQ2 is dealt with, or a spurious cascade can arrive. + */ +#define PIC1_CMD 0x20 +#define PIC1_DATA 0x21 +#define PIC2_CMD 0xA0 +#define PIC2_DATA 0xA1 + +#define PIC_EOI 0x20 /* non-specific end of interrupt, to the master */ + +#define ICW1_INIT 0x11 /* edge triggered, cascade, expect ICW4 */ +#define ICW4_8086 0x01 /* 8086/88 mode */ + +/* + * remap both PICs out of the way of the CPU exception vectors, then mask every + * line. + * + * the remap matters as much as the mask: firmware may leave the PIC base at + * the default 0x08, which collides with the #DF vector, so even a fully masked + * stray line would be reported as a double fault. 0xF0 sits above our spurious + * vector (0xFF) and is otherwise unassigned. + */ +#define PIC1_BASE 0xF0 +#define PIC2_BASE 0xF8 + +static void pic_init_one(uint16_t cmd, uint16_t data, uint8_t base) { + outb(cmd, ICW1_INIT); + outb(data, base); + outb(data, base + 8); + outb(data, ICW4_8086); + + /* mask every line (bit set = masked) */ + outb(data, 0xFF); +} + +void pic_disarm(void) { + /* mask the slave first, then the master, so no cascade slips through */ + outb(PIC2_DATA, 0xFF); + outb(PIC1_DATA, 0xFF); + + /* acknowledge anything already pending, then remap and mask for real */ + outb(PIC2_CMD, PIC_EOI); + outb(PIC1_CMD, PIC_EOI); + + pic_init_one(PIC1_CMD, PIC1_DATA, PIC1_BASE); + pic_init_one(PIC2_CMD, PIC2_DATA, PIC2_BASE); + + /* + * both PICs are masked now, but the master still needs to know the slave is + * cascaded on IRQ2 or it raises spurious interrupts forever. + */ + outb(PIC1_DATA, 0xFF); +} \ No newline at end of file
arch/x86_64/serial.c added +115 -0
diff --git a/arch/x86_64/serial.c b/arch/x86_64/serial.c new file mode 100644 index 0000000000000000000000000000000000000000..abc28c74b1d07ef751507dfd1ea25be150c0cbda --- /dev/null +++ b/arch/x86_64/serial.c @@ -0,0 +1,115 @@ +#include <serial.h> + +#include <io.h> +#include <types.h> + +#define COM_PORT 0x3F8 + +#define REG_DATA 0 +#define REG_IER 1 +#define REG_FCR 2 +#define REG_LCR 3 +#define REG_MCR 4 +#define REG_LSR 5 + +#define LSR_THR_EMPTY 0x20 + +/* + * serial output lock. + * + * the 16550 has a single transmit holding register and no way to read back what + * is still queued, so two CPUs writing at once interleave mid-line and produce + * garbled output. that is not cosmetic here: the AP bring-up log is the only + * evidence available when a core fails to start, so a scrambled "ap: 3 online" + * is indistinguishable from a core that never came up. + * + * a test-and-set flag is enough rather than a queue: output is strictly + * synchronous (we spin until the holding register drains), so the critical + * section is short and bounded by a few dozen I/O cycles. + * + * the wait is a plain spin, never a halt. an AP that arrived here with + * interrupts off would deadlock against a `hlt`, and the BSP must stay able to + * report the stall. + */ +static volatile uint8_t serial_lock; + +static void serial_lock_acquire(void) { + while (atomic_test_and_set_u8(&serial_lock) != 0) { + /* another CPU owns the port; keep retrying */ + } +} + +static void serial_lock_release(void) { + /* + * plain store rather than a locked release. the xchg in the acquire path is + * a full barrier, so everything written before this store is already visible + * to the next acquirer's locked read of the flag. + */ + serial_lock = 0; +} + +static uint8_t serial_read(uint16_t reg) { + return inb((uint16_t)(COM_PORT + reg)); +} + +static void serial_write(uint16_t reg, uint8_t value) { + outb((uint16_t)(COM_PORT + reg), value); +} + +static void serial_write_byte(uint8_t byte) { + while ((serial_read(REG_LSR) & LSR_THR_EMPTY) == 0) { + /* wait for the transmit holding register to drain */ + } + serial_write(REG_DATA, byte); +} + +void serial_init(void) { + serial_write(REG_IER, 0x00); /* no interrupts */ + serial_write(REG_LCR, 0x80); /* enable DLAB */ + serial_write(REG_DATA, 0x03); /* divisor low (38400 baud) */ + serial_write(REG_IER, 0x00); /* divisor high */ + serial_write(REG_LCR, 0x03); /* 8 bits, no parity, one stop bit */ + serial_write(REG_FCR, 0xC7); /* enable + clear FIFOs, 14-byte threshold */ + serial_write(REG_MCR, 0x0B); /* IRQs enabled, RTS/DSR set */ +} + +void serial_putc(char c) { + serial_lock_acquire(); + + if (c == '\n') { + serial_write_byte((uint8_t)'\r'); + } + serial_write_byte((uint8_t)c); + + serial_lock_release(); +} + +/* + * the lock is taken once per string rather than per character: a 40-character + * line then costs one atomic instead of forty, and the output stays atomic + * across the whole line, which is what the log reader needs. + */ +void serial_puts(const char *s) { + serial_lock_acquire(); + + while (*s != '\0') { + if (*s == '\n') { + serial_write_byte((uint8_t)'\r'); + } + serial_write_byte((uint8_t)*s); + s++; + } + + serial_lock_release(); +} + +void serial_put_hex(uint64_t value) { + int shift; + + serial_puts("0x"); + for (shift = 60; shift >= 0; shift -= 4) { + uint8_t nibble = (uint8_t)((value >> (unsigned)shift) & 0xF); + char c = (nibble < 10) ? (char)('0' + nibble) : (char)('a' + (nibble - 10)); + serial_putc(c); + } +} \ No newline at end of file
boot/uefi/Makefile added +25 -0
diff --git a/boot/uefi/Makefile b/boot/uefi/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..610d5b29d1492e5e91581a18072583d2370cdc81 --- /dev/null +++ b/boot/uefi/Makefile @@ -0,0 +1,25 @@ +BOOT_SRC := boot/uefi/main.c + +# number of vCPUs. anything above 1 is required for SMP work (AP bringup). +SMP ?= 4 + +SRC := $(KERNEL_SRC) $(ARCH_SRC) $(ARCH_ASM_SRC) $(BOOT_SRC) +OBJ := $(patsubst %,$(BUILD)/%.o,$(SRC)) +DEP := $(OBJ:.o=.d) + +.PHONY: run +run: $(EFI) + @mkdir -p $(BUILD_BOOT)/esp/EFI/BOOT + cp $(EFI) $(BUILD_BOOT)/esp/EFI/BOOT/BOOTX64.EFI + @test -f $(BUILD_BOOT)/OVMF_VARS.fd || cp /usr/share/edk2-ovmf/OVMF_VARS.fd $(BUILD_BOOT)/OVMF_VARS.fd + qemu-system-x86_64 -m 128M -smp $(SMP) -no-reboot \ + -drive if=pflash,format=raw,readonly=on,file=/usr/share/edk2-ovmf/OVMF_CODE.fd \ + -drive if=pflash,format=raw,file=$(BUILD_BOOT)/OVMF_VARS.fd \ + -drive format=raw,file=fat:rw:$(BUILD_BOOT)/esp \ + -nographic + +$(EFI): $(OBJ) + @mkdir -p $(dir $@) + lld-link -subsystem:efi_application -entry:efi_main -nodefaultlib -out:$@ $^ + +-include $(DEP)
boot/uefi/main.c added +11 -0
diff --git a/boot/uefi/main.c b/boot/uefi/main.c new file mode 100644 index 0000000000000000000000000000000000000000..5303f9dad1ece874febcd0e3cf12966e6c61731b --- /dev/null +++ b/boot/uefi/main.c @@ -0,0 +1,11 @@ +#include <efi.h> + +#include <kernel.h> + +EFI_STATUS EFIAPI efi_main(EFI_HANDLE image, EFI_SYSTEM_TABLE *st); + +EFI_STATUS EFIAPI efi_main(EFI_HANDLE image, EFI_SYSTEM_TABLE *st) { + kernel_main(image, st); + + return EFI_SUCCESS; +}
boot/x86_64/Makefile deleted +0 -27
diff --git a/boot/x86_64/Makefile b/boot/x86_64/Makefile deleted file mode 100644 index dbca3973b86090475c078a410046d4b60370dd3c..0000000000000000000000000000000000000000 --- a/boot/x86_64/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -ifndef KERNEL_ELF -$(error KERNEL_ELF undefined) -endif - -ifndef BUILD_BOOT -$(error BUILD_BOOT undefined) -endif - -QEMU := qemu-system-x86_64 -QEMUFLAGS := -m 64M -no-reboot -no-shutdown -d int - -GRUB_CFG := boot/x86_64/grub.cfg - -TARGET_ISO := $(BUILD_BOOT)/image.iso -TARGET_DIR := $(BUILD_BOOT)/isodir - -.PHONY: qemu -qemu: clean $(TARGET_ISO) - $(QEMU) $(QEMUFLAGS) -cdrom $(TARGET_ISO) - -$(TARGET_ISO): $(KERNEL_ELF) - @mkdir -p $(TARGET_DIR)/boot/grub/ - - @cp $(GRUB_CFG) $(TARGET_DIR)/boot/grub/grub.cfg - @cp $(KERNEL_ELF) $(TARGET_DIR)/boot/kernel.elf - - grub-mkrescue -o $(TARGET_ISO) $(TARGET_DIR) 2> /dev/null
boot/x86_64/grub.cfg deleted +0 -6
diff --git a/boot/x86_64/grub.cfg b/boot/x86_64/grub.cfg deleted file mode 100644 index 5cf0935ef69f36876586b60f36df304144a3957c..0000000000000000000000000000000000000000 --- a/boot/x86_64/grub.cfg +++ /dev/null @@ -1,6 +0,0 @@ -timeout=0 - -menuentry "furix" { - multiboot2 /boot/kernel.elf - boot -}
include/acpi.h added +43 -0
diff --git a/include/acpi.h b/include/acpi.h new file mode 100644 index 0000000000000000000000000000000000000000..5d31be53f446ce462056752224cfff226467032f --- /dev/null +++ b/include/acpi.h @@ -0,0 +1,43 @@ +#ifndef FURIX_ACPI_H +#define FURIX_ACPI_H + +#include <efi.h> +#include <types.h> + +#define ACPI_MAX_APIC_IDS 256 + +struct acpi_madt_info { + uint32_t cpu_count; + uint32_t ioapic_count; + + /* + * firmware-reported processor IDs, which for a MADT using x2APIC entries are + * 32-bit and need not fit in a byte. kept as uint32_t rather than narrowed, + * because a truncated ID silently retargets an INIT/SIPI at the wrong CPU, + * which presents as an AP that never boots. + */ + uint32_t apic_ids[ACPI_MAX_APIC_IDS]; + + /* + * dense 0..cpu_count-1 index into the per-CPU array, parallel to apic_ids. + * + * needed because x2APIC IDs may exceed the number of per-CPU slots we + * reserve, so the ID itself cannot be used as an array index. the BSP picks + * the slot and the trampoline is patched with it. + */ + uint8_t slots[ACPI_MAX_APIC_IDS]; +}; + +/* locate the RSDP via the UEFI config table and enumerate the MADT */ +bool acpi_madt_parse(const EFI_SYSTEM_TABLE *st, struct acpi_madt_info *out); + +/* + * number of processors the BSP must start, i.e. every MADT CPU except itself. + * + * a helper rather than open-coded `cpu_count - 1`: if the MADT lists no + * enabled local APIC at all, the subtraction wraps to UINT32_MAX and the + * bring-up loop spins looking for four billion processors. + */ +uint32_t acpi_madt_ap_count(const struct acpi_madt_info *madt); + +#endif /* FURIX_ACPI_H */
include/arch/x86_64/ap_boot.h added +21 -0
diff --git a/include/arch/x86_64/ap_boot.h b/include/arch/x86_64/ap_boot.h new file mode 100644 index 0000000000000000000000000000000000000000..c2290c09e5accd541911f037780d9be1a8cc6706 --- /dev/null +++ b/include/arch/x86_64/ap_boot.h @@ -0,0 +1,21 @@ +#ifndef FURIX_ARCH_X86_64_AP_BOOT_H +#define FURIX_ARCH_X86_64_AP_BOOT_H + +#include <acpi.h> +#include <efi.h> +#include <types.h> + +/* + * reserve a free page below 1 MiB for the trampoline and copy the image into + * it. must run BEFORE the kernel installs its own GDT/IDT, because afterwards + * we must not call into UEFI boot services again. false if no page was found. + */ +bool ap_boot_reserve(const EFI_SYSTEM_TABLE *st, uint64_t *out_base); + +/* start every MADT AP via INIT-SIPI-SIPI, waiting with a bounded spin */ +uint32_t ap_boot_start(const struct acpi_madt_info *madt); + +/* set by each AP once running with its per-CPU area */ +extern volatile uint32_t ap_online_count; + +#endif /* FURIX_ARCH_X86_64_AP_BOOT_H */
include/arch/x86_64/gdt.h added +52 -0
diff --git a/include/arch/x86_64/gdt.h b/include/arch/x86_64/gdt.h new file mode 100644 index 0000000000000000000000000000000000000000..e01046a565861e11c2a139b483e28bd328c19df6 --- /dev/null +++ b/include/arch/x86_64/gdt.h @@ -0,0 +1,52 @@ +#ifndef FURIX_ARCH_X86_64_GDT_H +#define FURIX_ARCH_X86_64_GDT_H + +#include <types.h> + +/* long-mode segment selectors, in the kernel GDT */ +#define SEGSEL_KERNEL_CODE 0x08 +#define SEGSEL_KERNEL_DATA 0x10 +#define SEGSEL_TSS 0x18 + +/* + * selector of the TSS descriptor in the AP trampoline's flat GDT. that table is + * built in ap_boot.S and is separate from the kernel GDT above, so it has its + * own numbering; the AP must select the descriptor from ITS table. + */ +#define AP_SEGSEL_TSS 0x20 + +/* IST indices, 1-based; 0 means "use the current stack" */ +#define IST_DOUBLE_FAULT 1 +#define IST_NMI 2 + +#define IST_STACK_SIZE 8192 + +/* + * the 64-bit TSS. declared here because both the kernel GDT and the per-AP + * TSSes are built from it, and ap_boot.S's AP IDT references the IST slots by + * index; a layout change here has to move in lockstep with both. + */ +struct tss { + uint32_t reserved0; + uint32_t reserved0_pad; + uint64_t rsp[3]; + uint64_t reserved1; + uint64_t ist[7]; + uint16_t reserved2; + uint16_t iomap_base; +} __attribute__((packed)); + +/* install the kernel GDT and reload the segment registers, including CS */ +void gdt_init(void); + +/* + * build the TSS for the AP in `slot` and return its base address, for the BSP + * to patch into the AP's copy of the trampoline GDT. + * + * each AP needs its own because the TSS holds the IST stack POINTERS: sharing + * one would put two processors on the same IST stack, which is exactly the + * corruption the IST exists to prevent. + */ +uint64_t gdt_ap_tss_base(uint8_t slot); + +#endif /* FURIX_ARCH_X86_64_GDT_H */
include/arch/x86_64/idt.h added +75 -0
diff --git a/include/arch/x86_64/idt.h b/include/arch/x86_64/idt.h new file mode 100644 index 0000000000000000000000000000000000000000..8ed2173bdf3cff4aa54646d5322b4f5644f3ed6d --- /dev/null +++ b/include/arch/x86_64/idt.h @@ -0,0 +1,75 @@ +#ifndef FURIX_ARCH_X86_64_IDT_H +#define FURIX_ARCH_X86_64_IDT_H + +#include <io.h> +#include <types.h> + +/* install the kernel IDT, with IST stacks wired up */ +void idt_init(void); + +/* + * the IDT the AP trampoline loads: same stubs, but no IST entries (an AP has + * no TSS) and gates selecting the trampoline's 64-bit code segment. written + * by idt_init(); nothing else should touch it. + */ +extern struct desc_ptr idt_ap_pointer; + +/* + * selector of the 64-bit code segment in the AP trampoline's flat GDT. must + * match arch/x86_64/ap_boot.S: index 3, so 0x18. the BSP's IDT uses 0x08 from + * the GDT in gdt.c, which is a 32-bit segment and so unusable from a long-mode + * interrupt gate. + */ +#define AP_CODE_SELECTOR_64 0x18 + +/* register a C handler for one vector; NULL restores the halting default */ +void idt_set_handler(uint8_t vector, void (*handler)(void)); + +/* + * the interrupt frame built by the assembly entry path in isr.S. field order + * and offsets must match the push sequence there; assert_layout() in idt.c + * checks the ones the C code actually reads. + */ +struct interrupt_frame { + /* pushed by isr_common, lowest address first */ + uint64_t rax; + uint64_t rbx; + uint64_t rcx; + uint64_t rdx; + uint64_t rbp; + uint64_t rsi; + uint64_t rdi; + uint64_t r8; + uint64_t r9; + uint64_t r10; + uint64_t r11; + uint64_t r12; + uint64_t r13; + uint64_t r14; + uint64_t r15; + + /* pushed by the stub, then by the CPU */ + uint64_t vector; + uint64_t error_code; + uint64_t rip; + uint64_t cs; + uint64_t rflags; + uint64_t rsp; + uint64_t ss; +}; + +/* called from the assembly stubs with a pointer to the frame above */ +void interrupt_dispatch(const struct interrupt_frame *frame); + +/* vectors used by the kernel itself */ +#define VECTOR_NMI 2 +#define VECTOR_DOUBLE_FAULT 8 +#define VECTOR_GENERAL_PROTECTION 13 +#define VECTOR_PAGE_FAULT 14 +#define VECTOR_SPURIOUS 0xFF + +/* free vector for IPIs and other kernel-generated interrupts */ +#define VECTOR_IPI_BASE 0x30 +#define VECTOR_IPI_TIMER 0x30 + +#endif /* FURIX_ARCH_X86_64_IDT_H */
include/arch/x86_64/io.h added +76 -0
diff --git a/include/arch/x86_64/io.h b/include/arch/x86_64/io.h new file mode 100644 index 0000000000000000000000000000000000000000..8e0792ff4110765df323fd5c4882e69f6fe87030 --- /dev/null +++ b/include/arch/x86_64/io.h @@ -0,0 +1,76 @@ +#ifndef FURIX_ARCH_X86_64_IO_H +#define FURIX_ARCH_X86_64_IO_H + +#include <types.h> + +struct desc_ptr { + uint16_t limit; + uint64_t base; +} __attribute__((packed)); + +/* + * CR4.PAE is required for the 4-level page tables long mode uses. CR4.PSE is + * NOT needed there: in long mode 2 MiB pages come from PAE on its own. + * + * CR4.FSGSBASE is never set on this CPU model, see percpu.c. + */ +#define CR4_PAE (1u << 5) +#define CR4_FSGSBASE (1u << 16) + +/* implemented in arch/x86_64/asm.S, so no inline asm elsewhere */ + +uint8_t inb(uint16_t port); +void outb(uint16_t port, uint8_t value); + +void cpu_cli(void); +void cpu_sti(void); +void cpu_hlt(void); + +/* mask interrupts, returning the prior EFLAGS word to pass back to restore */ +uint64_t cpu_irq_save(void); +void cpu_irq_restore(uint64_t flags); +uint32_t cpu_irqs_enabled(void); + +uint64_t rdmsr(uint32_t msr); +void wrmsr(uint32_t msr, uint64_t value); +void swapgs(void); + +uint32_t read_cr4(void); +void write_cr4(uint32_t value); +void wrgsbase(uint64_t value); +uint64_t rdfsbase(void); + +uint64_t read_cr2(void); +uint64_t read_cr3(void); + +/* result of the last cpuid() call: {eax, ebx, ecx, edx} */ +extern uint64_t cpuid_out[4]; +void cpuid(uint32_t leaf); + +void lgdt(const struct desc_ptr *ptr); +void lidt(const struct desc_ptr *ptr); +void ltr(uint16_t selector); +void reload_cs(uint16_t selector); +void load_data_segments(uint16_t selector); + +void gs_store(uint64_t value); +uint64_t gs_load(void); + +/* + * atomics, implemented in asm.S because they need `lock` prefixes that are + * awkward and error-prone as inline asm (and cannot be written as a plain C + * `+=` without the compiler knowing the access is shared). + * + * fetch_add returns the value BEFORE the addition, so it doubles as a + * completion index. the u32 form is deliberate: a 64-bit add is not atomic on + * a 32-bit target and 32 bits is what the counters here need. + */ +uint32_t atomic_fetch_add_u32(volatile uint32_t *ptr, uint32_t value); + +/* returns the value read, having set *ptr to 1. nonzero means "lost the race" */ +uint8_t atomic_test_and_set_u8(volatile uint8_t *ptr); + +/* a volatile load; x86 makes an aligned 32-bit load atomic anyway */ +uint32_t atomic_load_u32(const volatile uint32_t *ptr); + +#endif /* FURIX_ARCH_X86_64_IO_H */
include/arch/x86_64/ioapic.h added +19 -0
diff --git a/include/arch/x86_64/ioapic.h b/include/arch/x86_64/ioapic.h new file mode 100644 index 0000000000000000000000000000000000000000..47628bbe45ed3e91853acee42dab3549f00072a3 --- /dev/null +++ b/include/arch/x86_64/ioapic.h @@ -0,0 +1,19 @@ +#ifndef FURIX_ARCH_X86_64_IOAPIC_H +#define FURIX_ARCH_X86_64_IOAPIC_H + +#include <types.h> + +/* + * mask every interrupt input on every IO-APIC. + * + * firmware routes the 8259 PIC lines (including the PIT timer) through the + * IO-APIC, so masking only the 8259s is not enough: once we enable the LAPIC + * and execute STI, a leftover PIC line still arrives on whatever vector the + * firmware programmed it for. + */ +void ioapic_disarm(void); + +/* default IO-APIC base when firmware did not leave one mapped */ +#define IOAPIC_DEFAULT_BASE 0xFEC00000u + +#endif /* FURIX_ARCH_X86_64_IOAPIC_H */
include/arch/x86_64/lapic.h added +56 -0
diff --git a/include/arch/x86_64/lapic.h b/include/arch/x86_64/lapic.h new file mode 100644 index 0000000000000000000000000000000000000000..076462e1b70f17cc6e0c3fc772cddcb58b94d910 --- /dev/null +++ b/include/arch/x86_64/lapic.h @@ -0,0 +1,56 @@ +#ifndef FURIX_ARCH_X86_64_LAPIC_H +#define FURIX_ARCH_X86_64_LAPIC_H + +#include <types.h> + +/* delivery modes in ICR bits 8..10 */ +#define LAPIC_IPI_FIXED 0x0 +#define LAPIC_IPI_INIT 0x5 +#define LAPIC_IPI_STARTUP 0x6 + +/* ICR shorthand in bits 18..19 */ +#define LAPIC_DEST 0x0 +#define LAPIC_SHORTHAND_SELF 0x1 +#define LAPIC_SHORTHAND_ALL_EXCL 0x2 +#define LAPIC_SHORTHAND_ALL 0x3 + +void lapic_enable(void); +uint32_t lapic_base(void); +uint32_t lapic_id(void); +void lapic_eoi(void); + +/* wait until the last ICR write has been delivered (bounded) */ +bool lapic_ipi_wait(void); + +/* + * send an IPI and block until the LAPIC reports it delivered. synchronous by + * design, for INIT/SIPI and the self-IPI test; a message-passing API belongs + * on top of this rather than replacing it. + */ +uint32_t lapic_ipi_send(uint32_t delivery, uint32_t shorthand, uint32_t vector, uint32_t apic_id); + +/* + * as lapic_ipi_send but with explicit ICR bits 13..15 (`flags`), used for the + * INIT trigger bit that asserts and deasserts the reset level. + * + * every IPI entry point returns 0 on success and non-zero when the destination + * ID is beyond what the xAPIC ICR can address. callers MUST check it: the + * alternative is an INIT sent to processor 0, which resets the BSP. + */ +uint32_t lapic_ipi_send_flags(uint32_t delivery, uint32_t shorthand, uint32_t vector, + uint32_t apic_id, uint32_t flags); + +/* + * assert and deassert INIT for one processor. both halves are required: INIT is + * level-triggered, so the assert (ICR bit 14) holds the target in reset and + * only the deassert releases it to real mode at the reset vector, where it + * waits for a SIPI. a deassert alone leaves the processor running wherever it + * already was, so the SIPI has no held state to redirect. + */ +uint32_t lapic_ipi_init_assert(uint32_t apic_id); +uint32_t lapic_ipi_init_deassert(uint32_t apic_id); + +/* startup-IPI, which releases a held processor at `vector << 12` */ +uint32_t lapic_ipi_sipi(uint32_t apic_id, uint8_t vector); + +#endif /* FURIX_ARCH_X86_64_LAPIC_H */
include/arch/x86_64/percpu.h added +51 -0
diff --git a/include/arch/x86_64/percpu.h b/include/arch/x86_64/percpu.h new file mode 100644 index 0000000000000000000000000000000000000000..68928f604a88b1ad5ba2d869f52c73bd0a3ba1bd --- /dev/null +++ b/include/arch/x86_64/percpu.h @@ -0,0 +1,51 @@ +#ifndef FURIX_ARCH_X86_64_PERCPU_H +#define FURIX_ARCH_X86_64_PERCPU_H + +#include <types.h> + +/* + * number of per-CPU slots. the array is indexed densely from 0, not by APIC + * ID: an x2APIC MADT can report an ID far larger than the number of + * processors, so the ID itself is not a usable index. 256 is also the largest + * count the trampoline can carry, since it passes the slot in one byte. + */ +#define PERCPU_MAX_CPUS 256 + +/* per-AP stack; statically reserved until the buddy allocator exists */ +#define PERCPU_STACK_SIZE 4096 + +/* bring up GS-based per-CPU data for the BSP */ +void percpu_init(void); + +/* + * point %gs at the per-CPU area for `slot`, recording `apic_id` as this CPU's + * identity. used by the BSP once at boot and by each AP from its trampoline. + * must not be called for APIC ID 0 before percpu_init(), since that is where the + * BSP's swapgs lands. + */ +void percpu_attach(uint32_t apic_id, uint8_t slot); + +void percpu_set_id(uint32_t apic_id); + +/* this CPU's APIC ID, as recorded by percpu_attach() */ +uint32_t percpu_apic_id(void); + +/* address of this CPU's per-CPU area */ +uint64_t percpu_address(void); + +/* per-CPU area for a slot, for the AP trampolines */ +uint64_t percpu_area(uint8_t slot); + +/* top of the stack for a slot, for the AP trampoline */ +uint64_t percpu_stack_top(uint8_t slot); + +/* store a value through %gs:0 and read it back the same way */ +uint64_t percpu_gs_selftest(uint64_t value); + +/* + * same, but the read-back goes through the C variable, proving GS.base and the + * compiler-generated access agree. + */ +uint64_t percpu_gs_selftest_via_c(uint64_t value); + +#endif /* FURIX_ARCH_X86_64_PERCPU_H */
include/arch/x86_64/pic.h added +15 -0
diff --git a/include/arch/x86_64/pic.h b/include/arch/x86_64/pic.h new file mode 100644 index 0000000000000000000000000000000000000000..9b9328aed9c0aa279273ef6c1eb020970fd36275 --- /dev/null +++ b/include/arch/x86_64/pic.h @@ -0,0 +1,15 @@ +#ifndef FURIX_ARCH_X86_64_PIC_H +#define FURIX_ARCH_X86_64_PIC_H + +#include <types.h> + +/* + * mask all legacy 8259 PIC interrupts. + * + * interrupts route through the LAPIC once it is enabled, but the two 8259s + * are still unmapped at whatever vector base firmware left them at, so a stray + * IRQ0 (the PIT timer) arrives as a bogus vector the moment we execute STI. + */ +void pic_disarm(void); + +#endif /* FURIX_ARCH_X86_64_PIC_H */
include/arch/x86_64/serial.h added +11 -0
diff --git a/include/arch/x86_64/serial.h b/include/arch/x86_64/serial.h new file mode 100644 index 0000000000000000000000000000000000000000..e557141b3a7fb84cc84a171fbc6ac922e0cb5ae0 --- /dev/null +++ b/include/arch/x86_64/serial.h @@ -0,0 +1,11 @@ +#ifndef FURIX_ARCH_X86_64_SERIAL_H +#define FURIX_ARCH_X86_64_SERIAL_H + +#include <types.h> + +void serial_init(void); +void serial_putc(char c); +void serial_puts(const char *s); +void serial_put_hex(uint64_t value); + +#endif /* FURIX_ARCH_X86_64_SERIAL_H */
include/efi.h added +235 -0
diff --git a/include/efi.h b/include/efi.h new file mode 100644 index 0000000000000000000000000000000000000000..afad97082e4e30f35d559f143a8a066f8698fbd0 --- /dev/null +++ b/include/efi.h @@ -0,0 +1,235 @@ +#ifndef FURIX_EFI_H +#define FURIX_EFI_H + +#include <types.h> + +#if defined(_MSC_VER) +#define EFIAPI __cdecl +#elif defined(__GNUC__) || defined(__clang__) +#define EFIAPI __attribute__((ms_abi)) +#else +#define EFIAPI +#endif + +typedef uint64_t EFI_STATUS; +typedef uint64_t UINTN; +typedef uint16_t CHAR16; +typedef uint8_t BOOLEAN; +typedef void *EFI_HANDLE; +typedef void *EFI_EVENT; + +#define EFI_SUCCESS ((EFI_STATUS)0) +#define EFI_ERROR_MASK ((EFI_STATUS)1 << 63) +#define EFIERR(status) (((status) & EFI_ERROR_MASK) != 0) + +/* the status codes the kernel actually inspects by value. */ +#define EFI_BUFFER_TOO_SMALL ((EFI_STATUS)(EFI_ERROR_MASK | 5)) + +typedef struct { + uint32_t Data1; + uint16_t Data2; + uint16_t Data3; + uint8_t Data4[8]; +} EFI_GUID; + +typedef struct { + EFI_GUID VendorGuid; + void *VendorTable; +} EFI_CONFIGURATION_TABLE; + +typedef struct { + uint64_t Signature; + uint32_t Revision; + uint32_t HeaderSize; + uint32_t CRC32; + uint32_t Reserved; +} EFI_TABLE_HEADER; + +typedef uint64_t EFI_PHYSICAL_ADDRESS; +typedef uint64_t EFI_VIRTUAL_ADDRESS; +typedef UINTN EFI_TPL; + +typedef enum { + AllocateAnyPages, + AllocateMaxAddress, + AllocateAddress, + MaxAllocateType +} EFI_ALLOCATE_TYPE; + +typedef enum { + EfiReservedMemoryType, + EfiLoaderCode, + EfiLoaderData, + EfiBootServicesCode, + EfiBootServicesData, + EfiRuntimeServicesCode, + EfiRuntimeServicesData, + EfiConventionalMemory, + EfiUnusableMemory, + EfiACPIReclaimMemory, + EfiACPIMemoryNVM, + EfiMemoryMappedIO, + EfiMemoryMappedIOPortSpace, + EfiPalCode, + EfiPersistentMemory, + EfiMaxMemoryType +} EFI_MEMORY_TYPE; + +/* + * NOT packed, and that is deliberate. the UEFI layout puts PhysicalStart at + * offset 8, which needs 4 bytes of padding after the 4-byte Type, so the + * struct is 40 bytes, and marking it packed would move every following field + * down by 4 and silently corrupt every read. + * + * note that sizeof() here is 40 while EDK2 reports a DescriptorSize of 48: the + * firmware struct carries an extra union for versioned descriptors. always + * stride the memory map by the reported DescriptorSize, never by sizeof. + */ +typedef struct { + uint32_t Type; + EFI_PHYSICAL_ADDRESS PhysicalStart; + EFI_VIRTUAL_ADDRESS VirtualStart; + uint64_t NumberOfPages; + uint64_t Attribute; +} EFI_MEMORY_DESCRIPTOR; + +typedef enum { + AllHandles, + ByRegisterNotify, + ByProtocol +} EFI_LOCATE_SEARCH_TYPE; + +typedef enum { + TimerCancel, + TimerPeriodic, + TimerRelative +} EFI_TIMER_DELAY; + +typedef struct EFI_DEVICE_PATH_PROTOCOL EFI_DEVICE_PATH_PROTOCOL; + +// member order and offsets are part of the UEFI ABI; do not reorder. +// only the entries needed so far are typed precisely, but every member that +// precedes them must still be present so their offsets stay correct. +typedef struct EFI_BOOT_SERVICES EFI_BOOT_SERVICES; +struct EFI_BOOT_SERVICES { + EFI_TABLE_HEADER Hdr; + + EFI_STATUS (EFIAPI *RaiseTPL)(EFI_TPL NewTpl); + void (EFIAPI *RestoreTPL)(EFI_TPL OldTpl); + + EFI_STATUS (EFIAPI *AllocatePages)(EFI_ALLOCATE_TYPE Type, EFI_MEMORY_TYPE MemoryType, UINTN Pages, EFI_PHYSICAL_ADDRESS *Memory); + EFI_STATUS (EFIAPI *FreePages)(EFI_PHYSICAL_ADDRESS Memory, UINTN Pages); + EFI_STATUS (EFIAPI *GetMemoryMap)(UINTN *MemoryMapSize, EFI_MEMORY_DESCRIPTOR *MemoryMap, UINTN *MapKey, UINTN *DescriptorSize, uint32_t *DescriptorVersion); + EFI_STATUS (EFIAPI *AllocatePool)(EFI_MEMORY_TYPE PoolType, UINTN Size, void **Buffer); + EFI_STATUS (EFIAPI *FreePool)(void *Buffer); + + EFI_STATUS (EFIAPI *CreateEvent)(uint32_t Type, EFI_TPL NotifyTpl, void (EFIAPI *NotifyFunction)(EFI_EVENT Event, void *Context), void *NotifyContext, EFI_EVENT *Event); + EFI_STATUS (EFIAPI *SetTimer)(EFI_EVENT Event, EFI_TIMER_DELAY Type, uint64_t TriggerTime); + EFI_STATUS (EFIAPI *WaitForEvent)(UINTN NumberOfEvents, EFI_EVENT *Event, UINTN *Index); + EFI_STATUS (EFIAPI *SignalEvent)(EFI_EVENT Event); + EFI_STATUS (EFIAPI *CloseEvent)(EFI_EVENT Event); + EFI_STATUS (EFIAPI *CheckEvent)(EFI_EVENT Event); + + EFI_STATUS (EFIAPI *InstallProtocolInterface)(EFI_HANDLE *Handle, EFI_GUID *Protocol, uint32_t InterfaceType, void *Interface); + EFI_STATUS (EFIAPI *ReinstallProtocolInterface)(EFI_HANDLE Handle, EFI_GUID *Protocol, void *OldInterface, void *NewInterface); + EFI_STATUS (EFIAPI *UninstallProtocolInterface)(EFI_HANDLE Handle, EFI_GUID *Protocol, void *Interface); + EFI_STATUS (EFIAPI *HandleProtocol)(EFI_HANDLE Handle, EFI_GUID *Protocol, void **Interface); + void *Reserved; + EFI_STATUS (EFIAPI *RegisterProtocolNotify)(EFI_GUID *Protocol, void (EFIAPI *NotifyFunction)(EFI_EVENT Event, void *Context), EFI_EVENT *Registration); + EFI_STATUS (EFIAPI *LocateHandle)(EFI_LOCATE_SEARCH_TYPE SearchType, EFI_GUID *Protocol, void *SearchKey, UINTN *BufferSize, EFI_HANDLE *Buffer); + EFI_STATUS (EFIAPI *LocateDevicePath)(EFI_GUID *Protocol, EFI_DEVICE_PATH_PROTOCOL **DevicePath, EFI_HANDLE *Device); + EFI_STATUS (EFIAPI *InstallConfigurationTable)(EFI_GUID *Guid, void *Table); + + EFI_STATUS (EFIAPI *LoadImage)(BOOLEAN BootPolicy, EFI_HANDLE ParentImageHandle, EFI_DEVICE_PATH_PROTOCOL *DevicePath, void *SourceBuffer, UINTN SourceSize, EFI_HANDLE *ImageHandle); + EFI_STATUS (EFIAPI *StartImage)(EFI_HANDLE ImageHandle, UINTN *ExitDataSize, CHAR16 **ExitData); + EFI_STATUS (EFIAPI *Exit)(EFI_HANDLE ImageHandle, EFI_STATUS ExitStatus, UINTN ExitDataSize, CHAR16 *ExitData); + EFI_STATUS (EFIAPI *UnloadImage)(EFI_HANDLE ImageHandle); + EFI_STATUS (EFIAPI *ExitBootServices)(EFI_HANDLE ImageHandle, UINTN MapKey); + EFI_STATUS (EFIAPI *GetNextMonotonicCount)(uint64_t *Count); + EFI_STATUS (EFIAPI *Stall)(UINTN Microseconds); + EFI_STATUS (EFIAPI *SetWatchdogTimer)(UINTN Timeout, uint64_t WatchdogCode, UINTN DataSize, CHAR16 *WatchdogData); + + EFI_STATUS (EFIAPI *ConnectController)(EFI_HANDLE ControllerHandle, EFI_HANDLE *DriverImageHandle, EFI_DEVICE_PATH_PROTOCOL *RemainingDevicePath, BOOLEAN Recursive); + EFI_STATUS (EFIAPI *DisconnectController)(EFI_HANDLE ControllerHandle, EFI_HANDLE DriverImageHandle, EFI_HANDLE ChildHandle); + EFI_STATUS (EFIAPI *OpenProtocol)(EFI_HANDLE Handle, EFI_GUID *Protocol, void **Interface, EFI_HANDLE AgentHandle, EFI_HANDLE ControllerHandle, uint32_t Attributes); + EFI_STATUS (EFIAPI *CloseProtocol)(EFI_HANDLE Handle, EFI_GUID *Protocol, EFI_HANDLE AgentHandle, EFI_HANDLE ControllerHandle); + EFI_STATUS (EFIAPI *OpenProtocolInformation)(EFI_HANDLE Handle, EFI_GUID *Protocol, void **EntryBuffer, UINTN *EntryCount); + EFI_STATUS (EFIAPI *ProtocolsPerHandle)(EFI_HANDLE Handle, EFI_GUID ***ProtocolBuffer, UINTN *ProtocolBufferCount); + EFI_STATUS (EFIAPI *LocateHandleBuffer)(EFI_LOCATE_SEARCH_TYPE SearchType, EFI_GUID *Protocol, void *SearchKey, UINTN *NoHandles, EFI_HANDLE **Buffer); + EFI_STATUS (EFIAPI *LocateProtocol)(EFI_GUID *Protocol, void *Registration, void **Interface); + EFI_STATUS (EFIAPI *InstallMultipleProtocolInterfaces)(EFI_HANDLE *Handle, ...); + EFI_STATUS (EFIAPI *UninstallMultipleProtocolInterfaces)(EFI_HANDLE Handle, ...); + + EFI_STATUS (EFIAPI *CalculateCrc32)(void *Data, UINTN DataSize, uint32_t *Crc32); + void (EFIAPI *CopyMem)(void *Destination, void *Source, UINTN Length); + void (EFIAPI *SetMem)(void *Buffer, UINTN Size, uint8_t Value); + EFI_STATUS (EFIAPI *CreateEventEx)(uint32_t Type, EFI_TPL NotifyTpl, void (EFIAPI *NotifyFunction)(EFI_EVENT Event, void *Context), const void *NotifyContext, const EFI_GUID *EventGroup, EFI_EVENT *Event); +}; + +struct EFI_RUNTIME_SERVICES; + +typedef struct { + uint16_t ScanCode; + CHAR16 UnicodeChar; +} EFI_INPUT_KEY; + +typedef struct EFI_SIMPLE_TEXT_INPUT_PROTOCOL EFI_SIMPLE_TEXT_INPUT_PROTOCOL; +struct EFI_SIMPLE_TEXT_INPUT_PROTOCOL { + EFI_STATUS (EFIAPI *Reset)(EFI_SIMPLE_TEXT_INPUT_PROTOCOL *This, BOOLEAN ExtendedVerification); + EFI_STATUS (EFIAPI *ReadKeyStroke)(EFI_SIMPLE_TEXT_INPUT_PROTOCOL *This, EFI_INPUT_KEY *Key); + EFI_EVENT WaitForKey; +}; + +typedef struct { + int32_t CursorColumn; + int32_t CursorRow; + BOOLEAN CursorVisible; +} EFI_SIMPLE_TEXT_OUTPUT_MODE; + +typedef struct EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL; +struct EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL { + EFI_STATUS (EFIAPI *Reset)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, BOOLEAN ExtendedVerification); + EFI_STATUS (EFIAPI *OutputString)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, CHAR16 *String); + EFI_STATUS (EFIAPI *TestString)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, CHAR16 *String); + EFI_STATUS (EFIAPI *QueryMode)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, uint64_t ModeNumber, uint64_t *Columns, uint64_t *Rows); + EFI_STATUS (EFIAPI *SetMode)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, uint64_t ModeNumber); + EFI_STATUS (EFIAPI *SetAttribute)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, uint64_t Attribute); + EFI_STATUS (EFIAPI *ClearScreen)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This); + EFI_STATUS (EFIAPI *SetCursorPosition)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, uint64_t Column, uint64_t Row); + EFI_STATUS (EFIAPI *EnableCursor)(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, BOOLEAN Visible); + EFI_SIMPLE_TEXT_OUTPUT_MODE *Mode; +}; + +typedef struct EFI_SYSTEM_TABLE EFI_SYSTEM_TABLE; +struct EFI_SYSTEM_TABLE { + EFI_TABLE_HEADER Hdr; + CHAR16 *FirmwareVendor; + uint32_t FirmwareRevision; + EFI_HANDLE ConsoleInHandle; + EFI_SIMPLE_TEXT_INPUT_PROTOCOL *ConIn; + EFI_HANDLE ConsoleOutHandle; + EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *ConOut; + EFI_HANDLE StandardErrorHandle; + EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *StdErr; + struct EFI_RUNTIME_SERVICES *RuntimeServices; + struct EFI_BOOT_SERVICES *BootServices; + UINTN NumberOfTableEntries; + EFI_CONFIGURATION_TABLE *ConfigurationTable; +}; + +typedef char efi_assert_sizeof_guid[(sizeof(EFI_GUID) == 16) ? 1 : -1]; +typedef char efi_assert_sizeof_header[(sizeof(EFI_TABLE_HEADER) == 24) ? 1 : -1]; +typedef char efi_assert_sizeof_system_table[(sizeof(EFI_SYSTEM_TABLE) == 120) ? 1 : -1]; + +typedef char efi_assert_sizeof_boot_services[(sizeof(struct EFI_BOOT_SERVICES) == 376) ? 1 : -1]; +typedef char efi_assert_offset_get_memory_map[(offsetof(struct EFI_BOOT_SERVICES, GetMemoryMap) == 56) ? 1 : -1]; + +/* EFI_MEMORY_DESCRIPTOR layout is part of the UEFI ABI; see the struct comment. */ +typedef char efi_assert_sizeof_memory_descriptor[(sizeof(EFI_MEMORY_DESCRIPTOR) == 40) ? 1 : -1]; +typedef char efi_assert_offset_memory_start[(offsetof(EFI_MEMORY_DESCRIPTOR, PhysicalStart) == 8) ? 1 : -1]; +typedef char efi_assert_offset_memory_pages[(offsetof(EFI_MEMORY_DESCRIPTOR, NumberOfPages) == 24) ? 1 : -1]; +typedef char efi_assert_offset_memory_attr[(offsetof(EFI_MEMORY_DESCRIPTOR, Attribute) == 32) ? 1 : -1]; +typedef char efi_assert_offset_exit_boot_services[(offsetof(struct EFI_BOOT_SERVICES, ExitBootServices) == 232) ? 1 : -1]; + + +#endif /* FURIX_EFI_H */
include/kernel.h added +8 -0
diff --git a/include/kernel.h b/include/kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..b89db9ca23899fb2066ee0b6f4e9260a812b25d5 --- /dev/null +++ b/include/kernel.h @@ -0,0 +1,8 @@ +#ifndef FURIX_KERNEL_H +#define FURIX_KERNEL_H + +#include <efi.h> + +void kernel_main(EFI_HANDLE image, EFI_SYSTEM_TABLE *st); + +#endif /* FURIX_KERNEL_H */
include/types.h modified +23 -9
diff --git a/include/types.h b/include/types.h index f66a7be58d535da4f92799edd4e97fb342f1cdaa..8e3cd561dfd9c981b0d7a14910089464ce330c1d 100644 --- a/include/types.h +++ b/include/types.h @@ -1,15 +1,29 @@ -#pragma once +#ifndef FURIX_TYPES_H +#define FURIX_TYPES_H -typedef __UINT64_TYPE__ uint64_t; -typedef __UINT32_TYPE__ uint32_t; -typedef __UINT16_TYPE__ uint16_t; -typedef __UINT8_TYPE__ uint8_t; +typedef signed char int8_t; +typedef unsigned char uint8_t; -typedef __INT64_TYPE__ int64_t; -typedef __INT32_TYPE__ int32_t; -typedef __INT16_TYPE__ int16_t; -typedef __INT8_TYPE__ int8_t; +typedef signed short int16_t; +typedef unsigned short uint16_t; + +typedef signed int int32_t; +typedef unsigned int uint32_t; + +typedef signed long long int64_t; +typedef unsigned long long uint64_t; + +_Static_assert(sizeof(uint8_t) == 1, "bad uint8_t"); +_Static_assert(sizeof(uint16_t) == 2, "bad uint16_t"); +_Static_assert(sizeof(uint32_t) == 4, "bad uint32_t"); +_Static_assert(sizeof(uint64_t) == 8, "bad uint64_t"); typedef _Bool bool; #define true 1 #define false 0 + +#define offsetof(type, member) __builtin_offsetof(type, member) + +#define NULL ((void *)0) + +#endif /* FURIX_TYPES_H */
kernel/acpi.c added +222 -0
diff --git a/kernel/acpi.c b/kernel/acpi.c new file mode 100644 index 0000000000000000000000000000000000000000..15b0f1070563f570daa150163f2e50955c472c06 --- /dev/null +++ b/kernel/acpi.c @@ -0,0 +1,222 @@ +#include <acpi.h> + +#include <types.h> + +/* ACPI 2.0 RSDP: eb9d2d30-2d88-11d3-9a16-0090273fc14d */ +static const uint8_t guid_rsdp_v2[16] = { + 0x30, 0x2D, 0x9D, 0xEB, 0x88, 0x2D, 0xD3, 0x11, + 0x9A, 0x16, 0x00, 0x90, 0x27, 0x3F, 0xC1, 0x4D +}; + +/* ACPI 1.0 RSDP: 8868e871-e4f1-11d3-bc22-0080c73c8881 */ +static const uint8_t guid_rsdp_v1[16] = { + 0x71, 0xE8, 0x68, 0x88, 0xF1, 0xE4, 0xD3, 0x11, + 0xBC, 0x22, 0x00, 0x80, 0xC7, 0x3C, 0x88, 0x81 +}; + +#define MADT_TYPE_LOCAL_APIC 0 +#define MADT_TYPE_IO_APIC 1 +/* x2APIC: replaces type 0 when the firmware enabled x2APIC mode */ +#define MADT_TYPE_X2APIC 9 + +/* signatures read little-endian via read_u32(): the first four ASCII bytes. */ +#define ACPI_SIG_RSDP 0x20445352u /* "RSD " */ +#define ACPI_SIG_RSDT 0x54445352u /* "RSDT" */ +#define ACPI_SIG_MADT 0x43495041u /* "APIC" */ + +#define ACPI_SDT_HEADER_SIZE 36 + +/* + * the MADT header is 8 bytes longer than a plain SDT header: it appends + * LocalAPICAddress (4), PCATCompatibility (1), PCATCompatCount (1) and + * Reserved (2). Structure entries therefore begin at 44, not 36. + */ +#define ACPI_MADT_HEADER_SIZE 44 + +#define ACPI_RSDT_ENTRY_SIZE 8 + +/* Local APIC structure: Type, Length, ACPI ID, APIC ID, Flags(4). */ +#define MADT_LOCAL_APIC_FLAGS_OFFSET 4 +#define MADT_LOCAL_APIC_ENABLED 0x1 + +/* + * x2APIC structure: Type, Length, Reserved(2), ACPI UID(4), APIC ID(4), + * Flags(4), ... so the APIC ID is a full 32-bit field at offset 8 and the + * enabled flag sits at offset 12. the entry is 16 bytes minimum. + */ +#define MADT_X2APIC_ID_OFFSET 8 +#define MADT_X2APIC_FLAGS_OFFSET 12 +#define MADT_X2APIC_ENABLED 0x1 +#define MADT_X2APIC_MIN_LEN 16 + +static uint32_t read_u32(const uint8_t *p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +/* EFI_GUID is { u32, u16, u16, u8[8] } with no padding, so its memory image is + exactly the 16-byte wire form we compare against. */ +static bool guid_equal(const EFI_GUID *guid, const uint8_t *bytes) { + const uint8_t *p = (const uint8_t *)guid; + int i; + + for (i = 0; i < 16; i++) { + if (p[i] != bytes[i]) { + return false; + } + } + return true; +} + +/* an SDT's checksum is the sum of all its bytes, modulo 256, taken to zero. */ +static bool checksum_ok(const uint8_t *table, uint32_t len) { + uint8_t sum = 0; + uint32_t i; + + for (i = 0; i < len; i++) { + sum = (uint8_t)(sum + table[i]); + } + return sum == 0; +} + +static void madt_parse(const uint8_t *madt, uint32_t madt_len, struct acpi_madt_info *out) { + const uint8_t *entry = madt + ACPI_MADT_HEADER_SIZE; + const uint8_t *end = madt + madt_len; + + while (entry + 2 <= end) { + uint8_t type = entry[0]; + uint8_t len = entry[1]; + + /* length 0 would not advance us and could not be a valid structure. */ + if (len < 2 || entry + len > end) { + break; + } + + if ((type == MADT_TYPE_LOCAL_APIC && len >= 8) || + (type == MADT_TYPE_X2APIC && len >= MADT_X2APIC_MIN_LEN)) { + bool is_x2apic = (type == MADT_TYPE_X2APIC); + uint32_t flags = read_u32(entry + (is_x2apic ? MADT_X2APIC_FLAGS_OFFSET + : MADT_LOCAL_APIC_FLAGS_OFFSET)); + uint32_t enabled_bit = is_x2apic ? MADT_X2APIC_ENABLED : MADT_LOCAL_APIC_ENABLED; + + /* + * bit 0 of flags is "enabled"; skip CPUs the firmware marked unusable. + * + * the two entry types are handled together because a MADT uses one or the + * other, never both, and reading a type 0 entry as type 9 would pick up a + * 32-bit field spanning the flags and the following entry's header. + */ + if ((flags & enabled_bit) != 0 && out->cpu_count < ACPI_MAX_APIC_IDS) { + out->apic_ids[out->cpu_count] = + is_x2apic ? read_u32(entry + MADT_X2APIC_ID_OFFSET) : (uint32_t)entry[3]; + + /* + * the slot is dense rather than the APIC ID, because an x2APIC ID may + * exceed the per-CPU array we reserve. the BSP passes the slot to the + * trampoline, so the AP never has to index anything by ID itself. + */ + out->slots[out->cpu_count] = (uint8_t)out->cpu_count; + out->cpu_count++; + } + } else if (type == MADT_TYPE_IO_APIC && len >= 12) { + out->ioapic_count++; + } + + entry += len; + } +} + +/* + * locate a table by the signature of the table itself rather than by the + * Type byte recorded in the RSDT entry. it costs one extra read and needs no + * trust in the entry metadata, which some firmware images get wrong. + */ +static const uint8_t *rsdt_find_table(const uint8_t *rsdt, uint32_t rsdt_len, uint32_t sig) { + const uint8_t *entry = rsdt + ACPI_SDT_HEADER_SIZE; + const uint8_t *end = rsdt + rsdt_len; + + while (entry + ACPI_RSDT_ENTRY_SIZE <= end) { + const uint8_t *t = (const uint8_t *)(uint64_t)read_u32(entry + 4); + + if (t != NULL && read_u32(t) == sig) { + return t; + } + entry += ACPI_RSDT_ENTRY_SIZE; + } + return NULL; +} + +bool acpi_madt_parse(const EFI_SYSTEM_TABLE *st, struct acpi_madt_info *out) { + const uint8_t *rsdp = NULL; + const uint8_t *rsdt; + const uint8_t *madt; + uint32_t rsdt_len; + uint32_t madt_len; + UINTN i; + + out->cpu_count = 0; + out->ioapic_count = 0; + + /* some firmware installs the MADT directly in the UEFI configuration table. */ + for (i = 0; i < st->NumberOfTableEntries; i++) { + const uint8_t *t = (const uint8_t *)st->ConfigurationTable[i].VendorTable; + + if (t != NULL && read_u32(t) == ACPI_SIG_MADT) { + madt_len = read_u32(t + 4); + if (madt_len >= ACPI_MADT_HEADER_SIZE && checksum_ok(t, madt_len)) { + madt_parse(t, madt_len, out); + return out->cpu_count > 0; + } + } + } + + for (i = 0; i < st->NumberOfTableEntries; i++) { + const EFI_CONFIGURATION_TABLE *ct = &st->ConfigurationTable[i]; + + if (guid_equal(&ct->VendorGuid, guid_rsdp_v2) || guid_equal(&ct->VendorGuid, guid_rsdp_v1)) { + rsdp = (const uint8_t *)ct->VendorTable; + break; + } + } + + if (rsdp == NULL || read_u32(rsdp) != ACPI_SIG_RSDP) { + return false; + } + /* signature is 8 bytes: "RSD PTR " -> R(0) S(1) D(2) ' '(3) P(4) T(5) R(6) ' '(7). */ + if (rsdp[4] != 'P' || rsdp[5] != 'T' || rsdp[6] != 'R' || rsdp[7] != ' ') { + return false; + } + + /* RsdtPhysicalAddress is a 32-bit field at offset 16. */ + rsdt = (const uint8_t *)(uint64_t)read_u32(rsdp + 16); + if (rsdt == NULL || read_u32(rsdt) != ACPI_SIG_RSDT) { + return false; + } + + rsdt_len = read_u32(rsdt + 4); + if (rsdt_len < ACPI_SDT_HEADER_SIZE || !checksum_ok(rsdt, rsdt_len)) { + return false; + } + + madt = rsdt_find_table(rsdt, rsdt_len, ACPI_SIG_MADT); + if (madt == NULL) { + return false; + } + + madt_len = read_u32(madt + 4); + if (madt_len < ACPI_MADT_HEADER_SIZE || !checksum_ok(madt, madt_len)) { + return false; + } + + madt_parse(madt, madt_len, out); + return out->cpu_count > 0; +} + +uint32_t acpi_madt_ap_count(const struct acpi_madt_info *madt) { + /* + * one of the listed processors is the BSP, which is already running. the + * subtraction is guarded rather than written as `cpu_count - 1`: a MADT with + * no enabled processor entries would otherwise wrap to UINT32_MAX and send + * the bring-up loop looking for four billion APs. + */ + return (madt->cpu_count > 0) ? (madt->cpu_count - 1) : 0; +}
kernel/kernel.c modified +211 -10
diff --git a/kernel/kernel.c b/kernel/kernel.c index 57af06c3bf8fb81378a2b946c4c8f73ad6a89a5e..5db5943d3505feacbeda0468994b34afae2603d5 100644 --- a/kernel/kernel.c +++ b/kernel/kernel.c @@ -1,14 +1,215 @@ -#include <types.h> +#include <kernel.h> + +#include <acpi.h> +#include <ap_boot.h> +#include <gdt.h> +#include <idt.h> +#include <io.h> +#include <ioapic.h> +#include <lapic.h> +#include <pic.h> +#include <percpu.h> +#include <serial.h> + +/* addresses spread across different PML4 slots. OVMF maps the first PML4 entry + (0-512 GiB), so low candidates may be present; the high ones are not. we READ + rather than write so a mapped candidate cannot corrupt firmware. */ +static const uint64_t probe_addrs[] = { + 0x0000700000000000ull, /* ~123 TiB (PML4 slot 224, canonical) */ + 0xFFFF800000000000ull, /* kernel half (PML4 slot 256, canonical) */ + 0x0000008000000000ull, /* 512 GiB (PML4 slot 1) */ +}; + +/* + * self-IPI round trip: proves the LAPIC can deliver, the IDT dispatches, and + * iretq returns us cleanly. on real hardware this is also how we would verify + * a vector before trusting it for inter-CPU traffic. + */ +static volatile uint32_t ipi_count; +static volatile uint64_t ipi_sender_apic_id; + +static void on_ipi(void) { + ipi_sender_apic_id = lapic_id(); + ipi_count++; +} + +static bool test_self_ipi(void) { + const uint32_t spins = 2000000; + uint32_t i; + + ipi_count = 0; + idt_set_handler(VECTOR_IPI_TIMER, on_ipi); + + /* + * interrupts must be enabled for the IPI to be delivered: our gates are + * interrupt gates, so IF clear means the request is latched, not taken. + */ + cpu_sti(); + + lapic_ipi_send(LAPIC_IPI_FIXED, LAPIC_SHORTHAND_SELF, VECTOR_IPI_TIMER, 0); + + for (i = 0; i < spins && ipi_count == 0; i++) { + /* wait for the interrupt to arrive */ + } + + cpu_cli(); + + serial_puts("apic: self-IPI on vector "); + serial_put_hex(VECTOR_IPI_TIMER); + serial_puts(" -> count = "); + serial_put_hex(ipi_count); + serial_puts("\n"); -typedef struct { - uint8_t chr; - uint8_t attr; -} vgachar_t; + if (ipi_count == 0) { + serial_puts("apic: FAIL (no interrupt delivered)\n"); + return false; + } + + serial_puts("apic: ipi sender apic id = "); + serial_put_hex(ipi_sender_apic_id); + serial_puts(ipi_sender_apic_id == lapic_id() ? " (ok)\n" : " (WRONG)\n"); + + idt_set_handler(VECTOR_IPI_TIMER, NULL); + return true; +} + +static bool report_madt(const EFI_SYSTEM_TABLE *st, struct acpi_madt_info *madt) { + uint32_t i; + + if (!acpi_madt_parse(st, madt)) { + serial_puts("madt: not found\n"); + return false; + } + + serial_puts("madt: cpus = "); + serial_put_hex(madt->cpu_count); + serial_puts(", ioapics = "); + serial_put_hex(madt->ioapic_count); + serial_puts("\n apic ids:"); -volatile vgachar_t *display = (vgachar_t *)0xb8000; + for (i = 0; i < madt->cpu_count; i++) { + serial_puts(" "); + serial_put_hex(madt->apic_ids[i]); + } + serial_puts("\n"); -void kernel_main(void) { - char meow[] = "meow :3 hello world!"; - for (uint64_t i = 0; i < sizeof(meow); i++) - display[i] = (vgachar_t){.chr = meow[i], .attr = ' '}; + return true; } + +void kernel_main(EFI_HANDLE image, EFI_SYSTEM_TABLE *st) { + uint64_t trampoline_base = 0; + struct acpi_madt_info madt; + uint32_t started = 0; + uint64_t gs_check; + uint32_t count; + uint32_t i; + + (void)image; + + /* firmware console first: once our own GDT/IDT are live we must not call UEFI + boot services again */ + st->ConOut->OutputString(st->ConOut, (CHAR16 *)L"meow :3 hello world!\r\n"); + + cpu_cli(); + serial_init(); + pic_disarm(); + ioapic_disarm(); + serial_puts("furix: phase 0\n"); + + /* + * the LAPIC timer MUST be masked before the first UEFI boot service call + * below, not merely before we enable the LAPIC. boot services re-enable + * interrupts on return, so the firmware's timer is live across the window + * where our IDT is installed but carries no handler for vector 0x20; a tick + * there lands on the default gate and halts the kernel, at a point that + * wanders with boot timing. observed roughly 1 run in 10 on smp=8. + */ + lapic_enable(); + serial_puts("lapic: base = "); + serial_put_hex(lapic_base()); + serial_puts(", id = "); + serial_put_hex(lapic_id()); + serial_puts("\n"); + + /* + * the boot services below can leave IF set on the way out, so re-assert it + * here rather than trusting the state we handed them. + */ + cpu_cli(); + + /* + * must happen before we install our own GDT/IDT: afterwards we must not call + * UEFI boot services again, and we need GetMemoryMap to find a low page. + */ + if (!ap_boot_reserve(st, &trampoline_base)) { + serial_puts("ERROR: ap trampoline reservation failed\n"); + for (;;) { + cpu_hlt(); + } + } + + gdt_init(); /* GDT + TSS + IST stacks */ + idt_init(); /* IDT, IST1 on #DF and IST2 on NMI */ + serial_puts("gdt + tss + ist + idt ok\n"); + + percpu_init(); + serial_puts("percpu: gs load = "); + gs_check = percpu_gs_selftest(0xfeedfacecafebeefull); + serial_put_hex(gs_check); + serial_puts(gs_check == 0xfeedfacecafebeefull ? " (ok)\n" : " (BROKEN)\n"); + serial_puts("percpu: c load = "); + gs_check = percpu_gs_selftest_via_c(0x1111222233334444ull); + serial_put_hex(gs_check); + serial_puts(gs_check == 0x1111222233334444ull ? " (ok)\n" : " (BROKEN)\n"); + + percpu_set_id(lapic_id()); + + test_self_ipi(); + + if (report_madt(st, &madt)) { + /* + * bring the APs up before the deliberate page fault below: that probe halts + * the kernel, so it has to be the last thing that happens. + * + * interrupts are OFF for the duration. AP startup is driven entirely by + * IPIs sent FROM this CPU, so nothing here needs an interrupt serviced, + * and the spin that waits for an AP to report in is long enough (tens of + * millions of iterations) for the timer to fire. nothing routes vector + * 0x20 yet, so it lands on the default gate, which reports and halts, + * which stopped the BSP mid-bringup, making an AP failure and a BSP death + * indistinguishable from the outside. + */ + cpu_cli(); + + started = ap_boot_start(&madt); + + serial_puts("ap: "); + serial_put_hex(started); + serial_puts(" of "); + serial_put_hex(acpi_madt_ap_count(&madt)); + serial_puts(" aps started\n"); + } + + cpu_cli(); + + serial_puts("probing unmapped addresses...\n"); + + count = (uint32_t)(sizeof(probe_addrs) / sizeof(probe_addrs[0])); + for (i = 0; i < count; i++) { + volatile uint64_t sink; + + serial_puts(" probing "); + serial_put_hex(probe_addrs[i]); + serial_puts(" ...\n"); + + sink = *(volatile uint64_t *)probe_addrs[i]; + (void)sink; + } + + serial_puts("ERROR: none of the probe addresses faulted\n"); + + /* do not return to firmware with our GDT/IDT installed */ + for (;;) { + cpu_hlt(); + } +} \ No newline at end of file