λ

furix

cute os

settings

arch/x86_64/ap_boot.c

25002 bytes raw

#include <ap_boot.h>
#include <gdt.h>

#include <idt.h>
#include <io.h>
#include <lapic.h>
#include <percpu.h>
#include <serial.h>

/* provided by ap_boot.S */
extern const uint8_t ap_trampoline[];
extern const uint64_t ap_trampoline_bytes;

/* patch slots in the trampoline, written by the BSP and read by another core */
extern volatile uint64_t ap_percpu;
extern volatile uint64_t ap_stack_top;
extern volatile uint64_t ap_entry;
extern volatile uint64_t ap_slot;
extern const uint8_t ap_gdt_sym[];
extern const uint8_t ap_gdt_end[];
extern const uint8_t ap_gdt_tss[];

/*
 * GDT pointer record, read by both LGDTs in the trampoline. it holds the limit
 * and base, NOT the descriptor table. pointing LGDT at the descriptors
 * themselves leaves GDTR.limit zero and the far jump triple faults.
 *
 * one record serves both loads: LGDT's memory layout follows the operand size,
 * and both loads use a 32-bit operand size.
 */
extern volatile uint8_t ap_gdt_ptr[];
extern const uint8_t ap_gdt_pm_ptr[];
extern const uint8_t ap_gdt_end[];

/*
 * IDT pointer record, read by the LIDT in ap_lm. both fields are absolute
 * addresses: the table lives in the kernel image, not in this page.
 */
extern volatile uint8_t ap_idt_pm_ptr[];

/* byte offsets within the GDT pointer record, which is limit-first */
#define AP_GDT_PM_LIMIT 0u
#define AP_GDT_PM_BASE 2u

#define AP_IDT_PM_LIMIT 0u
#define AP_IDT_PM_BASE 2u

/*
 * far-jump offset slots and the labels they must name.
 *
 * a far jump computes CS.base + offset, and the flat GDT gives every
 * descriptor base 0, so the offset field must hold the ABSOLUTE linear address
 * of the target. using the within-page offset alone sends the AP to linear
 * 0x23, inside the real-mode IVT, where it executes vector bytes forever.
 *
 * the 16-bit form is emitted as raw bytes because `ljmp $0x08, sym` in .code16
 * produces an ADDR32 relocation the linker fills with the image address.
 */
extern const uint8_t ap_pm32_off16[];
extern const uint8_t ap_pm32[];
extern const uint8_t ap_lm_off32[];
extern const uint8_t ap_lm[];

/*
 * the real-mode prologue's idea of where it lives.
 *
 * the AP cannot work this out for itself: INIT leaves SP=0, so the classic
 * `call $+5 ; popw %bx` sequence reads zeros back and every page-relative
 * reference then addresses the wrong place. the BSP knows the answer, it is
 * the base of the page it just wrote, so it is patched straight in.
 *
 * two words are needed because the load is a moffs16, i.e. an absolute address
 * baked into the instruction rather than a [bx+disp16]:
 *
 *   - ap_rm_base_addr16 is the operand the AP reads FROM.
 *   - ap_rm_base_slot is the value the AP reads, the page base itself.
 *
 * both are 16 bits because the page is below 1 MiB.
 */
extern const uint8_t ap_rm_base_addr16[];
extern const uint8_t ap_rm_base_slot[];
extern const uint8_t ap_rm_sp_slot[];
extern const uint8_t ap_rm_stack_top[];
extern const uint8_t ap_stage[];

extern const uint8_t ap_gdt_sym[];

/*
 * the PML4 base for CR3, a slot rather than a label because CR3 names a page
 * table outside this page. patched with an absolute physical address.
 */
extern volatile uint64_t ap_cr3_off32;

#define AP_PM32_OFF16_OFF ((uint64_t)(&ap_pm32_off16[0] - (uint64_t)ap_trampoline))
#define AP_LM_OFF32_OFF ((uint64_t)(&ap_lm_off32[0] - (uint64_t)ap_trampoline))
#define AP_PM32_WITHIN ((uint64_t)(&ap_pm32[0] - (uint64_t)ap_trampoline))
#define AP_LM_WITHIN ((uint64_t)(&ap_lm[0] - (uint64_t)ap_trampoline))
#define AP_RM_SP_WITHIN ((uint64_t)(&ap_rm_stack_top[0] - (uint64_t)ap_trampoline))
/* the AP's TSS descriptor inside its own copy of the flat GDT */
#define AP_GDT_TSS_WITHIN ((uint64_t)(&ap_gdt_tss[0] - (uint64_t)ap_trampoline))

/* the reserved trampoline page, set by ap_boot_reserve() */
static uint64_t trampoline_base;

/*
 * where the trampoline's progress marker lives inside the page, and the highest
 * value any AP has reached there. debug aid only, see delay_and_latch().
 */
static volatile uint8_t *ap_stage_ptr;
static uint8_t ap_stage_best;

/* the APs' PML4, built by ap_boot_reserve() */
static uint64_t ap_pml4_base;

static void ap_main(void);

/*
 * APs that have come up and initialised their per-CPU area.
 *
 * incremented with a locked add, not `++`: several APs reach ap_main() at once
 * and a plain read-modify-write lets two of them read the same old value, so
 * one increment is lost and the BSP waits forever for an AP that is already
 * online. the BSP only reads it through atomic_load_u32().
 */
volatile uint32_t ap_online_count;

#define PAGE_SIZE 4096u

/* a real-mode CS:IP cannot reach past 1 MiB */
#define TRAMPOLINE_MAX_ADDR 0x100000ull

/* bounded so a lost SIPI surfaces as a failed bringup, never a hang */
#define AP_START_SPINS 40000000u

/*
 * memory types we may take a page from. EfiLoaderCode/Data is where our own
 * image lives and EfiRuntimeServices* is firmware-owned, so both are excluded.
 */
static bool memory_type_usable(uint32_t type) {
  return type == EfiBootServicesCode ||
         type == EfiBootServicesData ||
         type == EfiConventionalMemory;
}

/*
 * find a free page below 1 MiB from the UEFI memory map.
 *
 * AllocatePages alone would be simpler but gives no upper bound, and the AP's
 * real-mode entry point must be somewhere the CPU can reach. so read the map
 * and choose an address, then claim that exact address below.
 */
static bool find_low_page(const EFI_SYSTEM_TABLE *st, uint64_t *out_addr) {
  EFI_BOOT_SERVICES *bs = st->BootServices;
  EFI_MEMORY_DESCRIPTOR *map = NULL;
  UINTN map_size = 0;
  UINTN desc_size = 0;
  UINTN key = 0;
  UINTN entry_size;
  UINTN count;
  UINTN i;
  uint32_t version = 0;
  EFI_STATUS status;
  bool found = false;

  /*
   * the sizing call passes a NULL buffer and is SUPPOSED to fail with
   * EFI_BUFFER_TOO_SMALL while reporting the size needed. testing it with
   * EFIERR() would reject the only outcome that is not an error, so require
   * exactly that status instead.
   */
  status = bs->GetMemoryMap(&map_size, NULL, &key, &desc_size, &version);

  if (status != EFI_BUFFER_TOO_SMALL) {
    serial_puts("ap: GetMemoryMap size probe failed, status=");
    serial_put_hex((uint64_t)status);
    serial_puts("\n");
    return false;
  }

  /* leave headroom: the map can grow between calls as firmware allocates */
  map_size += desc_size * 8;

  if (EFIERR(bs->AllocatePool(EfiLoaderData, map_size, (void **)&map))) {
    serial_puts("ap: AllocatePool failed for the memory map\n");
    return false;
  }

  if (EFIERR(bs->GetMemoryMap(&map_size, map, &key, &desc_size, &version))) {
    bs->FreePool(map);
    serial_puts("ap: GetMemoryMap failed\n");
    return false;
  }

  entry_size = desc_size;
  count = map_size / entry_size;

  for (i = 0; i < count && !found; i++) {
    const EFI_MEMORY_DESCRIPTOR *d = &map[i];
    uint64_t start = d->PhysicalStart;
    uint64_t end;
    uint64_t first_page;

    if (!memory_type_usable(d->Type)) {
      continue;
    }

    if (start >= TRAMPOLINE_MAX_ADDR) {
      continue;
    }

    end = start + d->NumberOfPages * PAGE_SIZE;

    first_page = (start + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1);

    /*
     * physical page 0 holds the real-mode IVT and BIOS data area, and the AP
     * jumps there on INIT, so never hand out page 0. the region itself may
     * legitimately START at 0, so skip the page rather than the region.
     */
    if (first_page == 0) {
      first_page = PAGE_SIZE;
    }

    if (first_page + PAGE_SIZE <= end && first_page + PAGE_SIZE <= TRAMPOLINE_MAX_ADDR) {
      *out_addr = first_page;
      found = true;
    }
  }

  bs->FreePool(map);
  return found;
}

/* entry attribute bits for the page tables below */
#define PTE_PRESENT (1ull << 0)
#define PTE_WRITE (1ull << 1)
#define PTE_LARGE (1ull << 7)

/* 512 entries per table, each 8 bytes wide */
#define PAGE_TABLE_ENTRIES 512u

/*
 * identity-map the low 4 GiB with 2 MiB pages, for the APs only.
 *
 * long mode cannot be entered with paging off, since EFER.LME is not honoured
 * until CR0.PG is set. page tables are required to *enter* long mode, not
 * merely a later refinement. the AP needs a translation regime before it can
 * fetch a single 64-bit instruction.
 *
 * three levels suffice for 4 GiB of identity mapping:
 *
 *   PML4[0] -> PDPT   (covers 512 GiB)
 *   PDPT[0] -> PD[0]  (covers   1 GiB)
 *   PD[i][j] -> 2 MiB page (512 pages of 2 MiB each)
 *
 * the BSP does NOT use these tables; it keeps running on the firmware's CR3
 * until it builds its own. only the APs load this PML4. that is temporary and
 * a known asymmetry to be resolved with the paging layer.
 */
#define AP_MAPPED_GIB 4u

/* bytes needed: PML4 + PDPT + AP_MAPPED_GIB page directories */
#define AP_PT_PAGES (2u + AP_MAPPED_GIB)

static void zero_page(uint64_t addr) {
  uint64_t *p = (uint64_t *)addr;
  uint32_t i;

  for (i = 0; i < PAGE_TABLE_ENTRIES; i++) {
    p[i] = 0;
  }
}

/*
 * build the identity map and return the PML4 physical address, or 0 on failure.
 *
 * the tables are allocated by the firmware and left as ordinary physical
 * memory: they are only ever read by the hardware, never by C code, so a
 * virtual mapping is unnecessary.
 */
static uint64_t build_ap_page_tables(const EFI_SYSTEM_TABLE *st) {
  EFI_BOOT_SERVICES *bs = st->BootServices;
  uint64_t base = 0;
  uint64_t pml4;
  uint64_t pdpt;
  uint64_t *pd;
  uint32_t gi;
  uint32_t i;

  if (EFIERR(bs->AllocatePages(AllocateAnyPages, EfiBootServicesData,
                               AP_PT_PAGES, &base))) {
    serial_puts("ap: could not allocate page tables\n");
    return 0;
  }

  pml4 = base;
  pdpt = base + PAGE_SIZE;

  zero_page(pml4);
  zero_page(pdpt);

  /*
   * entries are physical addresses with the attribute bits ORed in, so the low
   * 12 bits, always zero for a page-aligned table, are free to hold flags.
   */
  *(volatile uint64_t *)pml4 = pdpt | PTE_PRESENT | PTE_WRITE;

  for (gi = 0; gi < AP_MAPPED_GIB; gi++) {
    pd = (uint64_t *)(pdpt + PAGE_SIZE + (uint64_t)gi * PAGE_SIZE);
    zero_page(pdpt + PAGE_SIZE + (uint64_t)gi * PAGE_SIZE);

    /*
     * every PDPT entry has to be filled in, not just entry 0.
     *
     * one page directory covers 512 * 2 MiB = 1 GiB, so a PDPT with only
     * PDPT[0] populated maps exactly one gibibyte no matter how many
     * directories were filled in. the others were built, fully populated, and
     * simply unreachable. the first access above 1 GiB is lapic_id(), whose
     * #PF lands on the LAPIC's own MMIO window at 0xfee00000 and so reads like
     * a broken APIC.
     *
     * PDPT entries are 8 BYTES APART. `gi * PAGE_SIZE` is wrong here even
     * though the page directories themselves really are PAGE_SIZE apart:
     * indexing by PAGE_SIZE writes gi=1 over PD[0][0], which chains the
     * directories into each other and overwrites the large pages. the walk
     * then resolves the trampoline's own address to an unrelated physical
     * page, the AP fetches zeros instead of its own instructions, and the
     * failure looks like the trampoline page was corrupted.
     */
    *(volatile uint64_t *)(pdpt + (uint64_t)gi * sizeof(uint64_t)) =
      (uint64_t)pd | PTE_PRESENT | PTE_WRITE;

    for (i = 0; i < PAGE_TABLE_ENTRIES; i++) {
      uint64_t phys = ((uint64_t)gi * PAGE_TABLE_ENTRIES + i) * (2u * 1024u * 1024u);

      pd[i] = phys | PTE_PRESENT | PTE_WRITE | PTE_LARGE;
    }
  }

  return pml4;
}

/*
 * write the page-relative displacements used by the trampoline.
 *
 * everything here is an offset from the start of the image, because the AP
 * adds it to the page base it recovers at run time. the exception is CR3,
 * which names a page table outside the page and so holds an absolute address.
 */
static bool patch_page_relatives(volatile uint8_t *page, uint64_t base) {
  const uint64_t image = (uint64_t)ap_trampoline;
  const uint32_t gdt_limit = (uint32_t)((uint64_t)ap_gdt_end - (uint64_t)ap_gdt_sym) - 1u;
  volatile uint8_t *pm = &page[(uint64_t)ap_gdt_pm_ptr - image];

  /*
   * the GDT lives inside the page, so its runtime address is the page base
   * plus its within-page offset. this is the one place a true absolute address
   * is needed: the CPU reads the base out of the pointer record rather than
   * adding a displacement itself.
   */
  const uint32_t gdt_base = (uint32_t)(base + ((uint64_t)ap_gdt_sym - image));

  /* limit:2 then base:4 then 2 reserved */
  *(volatile uint16_t *)&pm[AP_GDT_PM_LIMIT] = (uint16_t)gdt_limit;
  *(volatile uint32_t *)&pm[AP_GDT_PM_BASE] = gdt_base;

  /*
   * the IDT pointer, unlike the GDT this one is not inside the page: the table
   * is the AP's own copy in the kernel image, already built by idt_init(), so
   * both fields are the address the BSP published.
   */
  {
    volatile uint8_t *idt_ptr = &page[(uint64_t)ap_idt_pm_ptr - image];

    *(volatile uint16_t *)&idt_ptr[AP_IDT_PM_LIMIT] = idt_ap_pointer.limit;
    *(volatile uint32_t *)&idt_ptr[AP_IDT_PM_BASE] =
      (uint32_t)idt_ap_pointer.base;
  }

  /*
   * GDTR.limit is the largest usable OFFSET, so it must cover the last byte of
   * the descriptor the long-mode far jump selects, selector 0x18, whose
   * descriptor ends at offset 0x1f. too small and the far jump raises #GP,
   * triple faults and resets the AP, which looks exactly like "it never ran".
   */
  if (gdt_limit < 0x2fu) {
    serial_puts("ap: gdt limit too small, ");
    serial_put_hex(gdt_limit);
    serial_puts("\n");
    return false;
  }

  /* CR3 gets the PML4's ABSOLUTE physical address */
  *(volatile uint64_t *)&page[(uint64_t)&ap_cr3_off32 - image] = ap_pml4_base;

  /*
   * the two far-jump offsets, plus the real-mode page base and stack pointer.
   * absolute addresses, not within-page offsets. both targets are inside this
   * page, which the BSP places below 1 MiB, so 16 bits suffices for the first
   * and 32 for the second.
   */
  {
    const uint64_t pm32_lin = base + AP_PM32_WITHIN;
    const uint64_t lm_lin = base + AP_LM_WITHIN;
    const uint64_t rm_sp = base + AP_RM_SP_WITHIN;
    const uint64_t base_slot_addr = base + (uint64_t)&ap_rm_base_slot - image;

    if (base > 0xffffu || base_slot_addr > 0xffffu || pm32_lin > 0xffffu ||
        rm_sp > 0xffffu || lm_lin > 0xffffffffu) {
      serial_puts("ap: trampoline page too high for its 16-bit offsets\n");
      return false;
    }

    *(volatile uint16_t *)&page[AP_PM32_OFF16_OFF] = (uint16_t)pm32_lin;
    *(volatile uint32_t *)&page[AP_LM_OFF32_OFF] = (uint32_t)lm_lin;

    /* both the operand AND the value are patched */
    *(volatile uint16_t *)&page[(uint64_t)ap_rm_base_addr16 - image] =
      (uint16_t)base_slot_addr;
    *(volatile uint16_t *)&page[(uint64_t)ap_rm_base_slot - image] =
      (uint16_t)base;

    /*
     * the real-mode stack pointer is an absolute linear address: SS is 0, so
     * SP is the physical address itself.
     */
    *(volatile uint16_t *)&page[(uint64_t)ap_rm_sp_slot - image] =
      (uint16_t)rm_sp;
  }

  return true;
}

/*
 * refresh the trampoline page from the image and re-apply every patch.
 *
 * done once per AP rather than once at reserve time. the page runs below 1 MiB
 * with paging off, which is shared space: an AP that faults there has no IDT
 * to report through, so it triple faults and QEMU resets it, and a reset AP
 * starts executing whatever sits at the reset vector. on an OVMF-only machine
 * that is uninitialised low RAM, and the resulting garbage stores land in this
 * page. a single failure then leaves the next AP with a trampoline whose
 * prologue and far-jump opcode have been overwritten. re-copying makes each AP
 * start from a page known to be identical to the image.
 */
static void reload_trampoline(void) {
  volatile uint8_t *page = (volatile uint8_t *)trampoline_base;
  uint64_t i;

  for (i = 0; i < ap_trampoline_bytes; i++) {
    page[i] = ap_trampoline[i];
  }

  if (!patch_page_relatives(page, trampoline_base)) {
    serial_puts("ap: could not patch trampoline\n");
  }
}

bool ap_boot_reserve(const EFI_SYSTEM_TABLE *st, uint64_t *out_base) {
  uint64_t base = 0;
  volatile uint8_t *page;
  uint64_t bytes;
  uint64_t i;

  if (!find_low_page(st, &base)) {
    serial_puts("ap: no free page below 1 MiB\n");
    return false;
  }

  bytes = ap_trampoline_bytes;

  /*
   * build the APs' identity map BEFORE patching, so ap_cr3_off32 has something
   * real to point at. long mode cannot be entered with paging disabled, so a
   * failure here is fatal to AP bringup.
   */
  ap_pml4_base = build_ap_page_tables(st);

  if (ap_pml4_base == 0) {
    return false;
  }

  if (bytes > PAGE_SIZE) {
    serial_puts("ap: trampoline too large (");
    serial_put_hex(bytes);
    serial_puts(" bytes)\n");
    return false;
  }

  /* claim it before writing, so firmware cannot hand it out underneath us */
  if (EFIERR(st->BootServices->AllocatePages(AllocateAddress, EfiBootServicesData, 1, &base))) {
    serial_puts("ap: could not claim page at ");
    serial_put_hex(base);
    serial_puts("\n");
    return false;
  }

  page = (volatile uint8_t *)base;

  for (i = 0; i < bytes; i++) {
    page[i] = ap_trampoline[i];
  }

  /* fill in every page-relative displacement now that the image is placed */
  if (!patch_page_relatives(page, base)) {
    return false;
  }

  serial_puts("ap: trampoline page at ");
  serial_put_hex(base);
  serial_puts(" (");
  serial_put_hex(bytes);
  serial_puts(" bytes), sipi vector ");
  serial_put_hex(base >> 12);
  serial_puts(", pml4 ");
  serial_put_hex(ap_pml4_base);
  serial_puts("\n");

  trampoline_base = base;
  ap_stage_ptr = &page[(uint64_t)ap_stage - (uint64_t)ap_trampoline];
  *out_base = base;
  return true;
}

/*
 * point the trampoline at this AP's per-CPU area, stack and entry point.
 * done per AP immediately before its SIPI, since one shared trampoline page
 * serves all of them and they are brought up one at a time.
 */
static void patch_trampoline(uint8_t slot) {
  /* offsets of the patch slots, relative to the start of the image */
  const uint64_t percpu_off = (uint64_t)&ap_percpu - (uint64_t)ap_trampoline;
  const uint64_t stack_off = (uint64_t)&ap_stack_top - (uint64_t)ap_trampoline;
  const uint64_t entry_off = (uint64_t)&ap_entry - (uint64_t)ap_trampoline;
  const uint64_t slot_off = (uint64_t)&ap_slot - (uint64_t)ap_trampoline;

  /* the page is written here and executed by another core: keep it volatile */
  volatile uint8_t *page = (volatile uint8_t *)trampoline_base;

  /*
   * the per-CPU area and stack are chosen by `slot`, a dense index, rather
   * than by APIC ID: an x2APIC MADT can report an ID larger than the number of
   * per-CPU slots reserved, and using it as an index would run off the array.
   */
  *(volatile uint64_t *)&page[percpu_off] = percpu_area(slot);
  *(volatile uint64_t *)&page[stack_off] = percpu_stack_top(slot);
  *(volatile uint64_t *)&page[entry_off] = (uint64_t)&ap_main;

  /*
   * the AP re-reads its own APIC ID from the LAPIC rather than being told, so
   * the slot travels with it and is handed to percpu_attach() there.
   */
  *(volatile uint64_t *)&page[slot_off] = (uint64_t)slot;

  /*
   * point this AP's GDT TSS descriptor at the TSS for its slot.
   *
   * the descriptor lives inside the copied page, so it is addressed by
   * page-base plus its within-page offset, exactly like the pointer record
   * patched in patch_page_relatives(). the descriptor is 16 bytes: base low 2,
   * base mid 1, access 1, limit/flags 1, base high 1, then the upper base.
   *
   * access byte 0x89 marks it present, ring0 and available; without it the LTR
   * in the trampoline raises #GP, which faults, and the AP triple faults into a
   * reset that is indistinguishable from "it never ran".
   */
  {
    volatile uint8_t *desc = &page[AP_GDT_TSS_WITHIN];
    uint64_t tss = gdt_ap_tss_base(slot);

    desc[2] = (uint8_t)((tss >> 16) & 0xFF); /* base mid */
    desc[3] = 0x89;                         /* present, ring0, 64-bit TSS */
    desc[5] = (uint8_t)((tss >> 24) & 0xFF); /* base high */
    desc[6] = (uint8_t)(tss & 0xFF);         /* base bits 32..39 */
    desc[7] = (uint8_t)((tss >> 40) & 0xFF); /* base bits 40..47 */
  }
}

/* runs on the AP, in long mode */
static void ap_main(void) {
  uint32_t id = lapic_id();

  /*
   * the BSP patched this AP's slot into the trampoline before the SIPI. read it
   * through the same volatile slot the trampoline image declares, so the
   * compiler cannot cache it or fold the load.
   */
  percpu_attach(id, (uint8_t)ap_slot);

  /*
   * tell the BSP we are alive and have our own GS base. this must be the
   * locked add and must come before the serial output below, because the BSP
   * treats the counter as the completion signal: if it observed the counter
   * first it may start patching the trampoline page for the next AP while this
   * one is still printing, and the two would interleave on the UART.
   */
  (void)atomic_fetch_add_u32(&ap_online_count, 1);

  serial_puts("ap: ");
  serial_put_hex(id);
  serial_puts(" online\n");

  /*
   * nothing to schedule yet. park with interrupts masked: an unhandled
   * interrupt would reach the default dispatcher and halt the whole kernel.
   */
  for (;;) {
    cpu_cli();
    cpu_hlt();
  }
}

static void delay_loops(uint32_t n) {
  volatile uint32_t i;

  for (i = 0; i < n; i++) {
  }
}

/*
 * busy-wait while latching the trampoline's progress marker.
 *
 * this has to cover the wait AFTER the SIPIs, not just the wait after them in
 * the caller: an AP runs its whole trampoline and faults within nanoseconds of
 * the SIPI landing, and a fault with no IDT becomes a triple fault and a reset,
 * after which the reset processor overwrites the page. by the time the caller
 * starts sampling, the evidence is usually already gone.
 */
static void delay_and_latch(uint32_t n) {
  uint32_t i;

  for (i = 0; i < n; i++) {
    if (ap_stage_ptr != NULL && *ap_stage_ptr > ap_stage_best) {
      ap_stage_best = *ap_stage_ptr;
    }
  }
}

static uint32_t start_one_ap(uint32_t apic_id, uint8_t sipi_vector) {
  uint32_t rc = 0;
  /*
   * INIT is level-triggered and needs both halves.
   *
   * the assert (ICR bit 14 set) drives the processor into the reset-hold
   * state. only once it is held does the deassert mean anything: it releases
   * the AP into real mode at the reset vector, spinning there until a SIPI
   * redirects it. sending the deassert alone, with bit 14 clear as this code
   * originally did, leaves the AP wherever the firmware had it and
   * the subsequent SIPIs go nowhere, so the AP never runs the trampoline.
   */
  rc |= lapic_ipi_init_assert(apic_id);
  delay_loops(200000);

  rc |= lapic_ipi_init_deassert(apic_id);
  delay_loops(200000);

  /*
   * two SIPIs are required: the first may be ignored if the processor has not
   * finished resetting, the second always lands. the vector selects the page
   * to jump to, so it must be the page we copied the trampoline into.
   */
  rc |= lapic_ipi_sipi(apic_id, sipi_vector);
  delay_and_latch(100000);

  rc |= lapic_ipi_sipi(apic_id, sipi_vector);
  delay_and_latch(100000);

  return rc;
}

uint32_t ap_boot_start(const struct acpi_madt_info *madt) {
  uint32_t bsp_id = lapic_id();
  uint8_t sipi_vector = (uint8_t)(trampoline_base >> 12);
  uint32_t started = 0;
  uint32_t i;
  const uint64_t stage_off = (uint64_t)ap_stage - (uint64_t)ap_trampoline;

  ap_online_count = 0;

  serial_puts("ap: starting from apic ");
  serial_put_hex(bsp_id);
  serial_puts(", vector ");
  serial_put_hex(sipi_vector);
  serial_puts("\n");

  for (i = 0; i < madt->cpu_count; i++) {
    uint32_t id = madt->apic_ids[i];
    uint8_t slot = madt->slots[i];
    uint32_t before;
    uint32_t spins;
    uint8_t stage = 0;

    if (id == bsp_id) {
      continue;
    }

    before = atomic_load_u32(&ap_online_count);

    /* fresh page per AP, see reload_trampoline() */
    reload_trampoline();
    patch_trampoline(slot);

    ap_stage_best = 0;
    ap_stage_ptr = &((volatile uint8_t *)trampoline_base)[stage_off];

    /*
     * a non-zero return means the destination ID was outside what the xAPIC
     * ICR can name, so no INIT or SIPI was sent at all. report that as a
     * failure to start rather than waiting out the spin below for an AP that
     * was never going to run.
     */
    if (start_one_ap(id, sipi_vector) != 0) {
      serial_puts("ap: ");
      serial_put_hex(id);
      serial_puts(" UNADDRESSABLE (x2apic id needs IA32_X2APIC_ICR)\n");
      continue;
    }

    for (spins = 0; spins < AP_START_SPINS; spins++) {
      if (atomic_load_u32(&ap_online_count) > before) {
        break;
      }
    }

    stage = ap_stage_best;

    if (atomic_load_u32(&ap_online_count) > before) {
      started++;
    } else {
      serial_puts("ap: ");
      serial_put_hex(id);
      serial_puts(" TIMEOUT stage=");
      serial_put_hex(stage);
      serial_puts("\n");
    }
  }

  return started;
}