apic: Improve pending IRQ flushing mechanism

stage23/entry.s3.c

@@ -7,6 +7,7 @@
 #include <lib/trace.h>
 #include <sys/e820.h>
 #include <sys/a20.h>
+#include <sys/idt.h>
 #include <lib/print.h>
 #include <fs/file.h>
 #include <lib/elf.h>
@@ -64,8 +65,6 @@ void uefi_entry(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable) {
 
     disk_create_index();
 
-    init_io_apics();
-
     boot_volume = NULL;
 
     EFI_HANDLE current_handle = ImageHandle;
@@ -126,6 +125,9 @@ __attribute__((section(".stage3_entry")))
 #endif
 __attribute__((noreturn))
 void stage3_common(void) {
+    init_flush_irqs();
+    init_io_apics();
+
     volume_iterate_parts(boot_volume,
         if (!init_config_disk(_PART)) {
             boot_volume = _PART;

stage23/lib/spinup.asm

@@ -5,6 +5,8 @@ invalid_idt:
 
 section .text
 
+extern flush_irqs
+
 global common_spinup
 bits 32
 common_spinup:
@@ -12,6 +14,8 @@ common_spinup:
 
     lidt [invalid_idt]
 
+    call flush_irqs
+
     xor eax, eax
     lldt ax
 

stage23/lib/spinup.asm32

@@ -4,6 +4,8 @@ extern gdt
 
 section .text
 
+extern flush_irqs
+
 global common_spinup
 bits 32
 common_spinup:
@@ -37,6 +39,8 @@ common_spinup:
     mov gs, eax
     mov ss, eax
 
+    call flush_irqs
+
     xor eax, eax
     lldt ax
 

stage23/lib/spinup.asm64

@@ -5,6 +5,8 @@ invalid_idt:
 
 section .text
 
+extern flush_irqs
+
 %macro push32 1
     sub rsp, 4
     mov dword [rsp], %1
@@ -20,6 +22,21 @@ common_spinup:
     lgdt [rel gdt]
     lidt [rel invalid_idt]
 
+    lea rbx, [rel .reload_cs]
+
+    push 0x28
+    push rbx
+    retfq
+.reload_cs:
+    mov eax, 0x30
+    mov ds, eax
+    mov es, eax
+    mov fs, eax
+    mov gs, eax
+    mov ss, eax
+
+    call flush_irqs
+
     mov rbp, rsp
 
     cmp esi, 4

stage23/protos/stivale.c

@@ -16,6 +16,7 @@
 #include <sys/pic.h>
 #include <sys/cpu.h>
 #include <sys/gdt.h>
+#include <sys/idt.h>
 #include <sys/lapic.h>
 #include <fs/file.h>
 #include <mm/vmm.h>
@@ -423,8 +424,6 @@ __attribute__((noreturn)) void stivale_spinup(
     }
 
     pic_mask_all();
-    pic_flush();
-
     io_apic_mask_all();
 
     common_spinup(stivale_spinup_32, 9,

stage23/sys/cpu.h

@@ -160,6 +160,12 @@ inline uint64_t rdtsc(void) {
     return ((uint64_t)edx << 32) | eax;
 }
 
+static inline void delay(uint64_t cycles) {
+    uint64_t next_stop = rdtsc() + cycles;
+
+    while (rdtsc() < next_stop);
+}
+
 #define rdrand(type) ({ \
     type ret; \
     asm volatile ( \

stage23/sys/idt.h

@@ -3,7 +3,7 @@
 
 #include <stdint.h>
 
-#if bios == 1
+#if defined (__i386__)
 
 struct idtr {
     uint16_t limit;
@@ -18,10 +18,34 @@ struct idt_entry {
     uint16_t offset_hi;
 } __attribute__((packed));
 
+#elif defined (__x86_64__)
+
+struct idtr {
+    uint16_t limit;
+    uint64_t ptr;
+} __attribute__((packed));
+
+struct idt_entry {
+    uint16_t offset_lo;
+    uint16_t selector;
+    uint8_t  ist;
+    uint8_t  type_attr;
+    uint16_t offset_mid;
+    uint32_t offset_hi;
+    uint32_t reserved;
+} __attribute__((packed));
+
+#endif
+
+#if bios == 1
+
 extern struct idtr idt;
 
 void init_idt(void);
 
 #endif
 
+void init_flush_irqs(void);
+void flush_irqs(void);
+
 #endif

stage23/sys/idt.s2.c

@@ -1,7 +1,11 @@
 #include <stddef.h>
 #include <stdint.h>
 #include <sys/idt.h>
+#include <sys/cpu.h>
+#include <sys/pic.h>
+#include <sys/lapic.h>
 #include <lib/blib.h>
+#include <mm/pmm.h>
 
 #if bios == 1
 
@@ -33,3 +37,51 @@ void init_idt(void) {
 }
 
 #endif
+
+static struct idt_entry *dummy_idt = NULL;
+
+__attribute__((interrupt))
+static void dummy_isr(void *p) {
+    (void)p;
+    lapic_eoi();
+}
+
+void init_flush_irqs(void) {
+    dummy_idt = ext_mem_alloc(256 * sizeof(struct idt_entry));
+
+    for (size_t i = 0; i < 256; i++) {
+        dummy_idt[i].offset_lo = (uint16_t)(uintptr_t)dummy_isr;
+        dummy_idt[i].type_attr = 0x8e;
+#if defined (__i386__)
+        dummy_idt[i].selector = 0x18;
+        dummy_idt[i].offset_hi = (uint16_t)((uintptr_t)dummy_isr >> 16);
+#elif defined (__x86_64__)
+        dummy_idt[i].selector = 0x28;
+        dummy_idt[i].offset_mid = (uint16_t)((uintptr_t)dummy_isr >> 16);
+        dummy_idt[i].offset_hi = (uint32_t)((uintptr_t)dummy_isr >> 32);
+#endif
+    }
+}
+
+void flush_irqs(void) {
+    struct idtr old_idt;
+    asm volatile ("sidt %0" : "=m"(old_idt) :: "memory");
+
+    struct idtr new_idt = {
+        256 * sizeof(struct idt_entry) - 1,
+        (uintptr_t)dummy_idt
+    };
+    asm volatile ("lidt %0" :: "m"(new_idt) : "memory");
+
+    // Flush the legacy PIC so we know the remaining ints come from the LAPIC
+    pic_flush();
+
+    asm volatile ("sti" ::: "memory");
+
+    // Delay a while to make sure we catch ALL pending IRQs
+    delay(10000000);
+
+    asm volatile ("cli" ::: "memory");
+
+    asm volatile ("lidt %0" :: "m"(old_idt) : "memory");
+}

stage23/sys/lapic.c

@@ -74,6 +74,8 @@ bool x2apic_check(void) {
     return true;
 }
 
+static bool x2apic_mode = false;
+
 bool x2apic_enable(void) {
     if (!x2apic_check())
         return false;
@@ -82,9 +84,19 @@ bool x2apic_enable(void) {
     ia32_apic_base |= (1 << 10);
     wrmsr(0x1b, ia32_apic_base);
 
+    x2apic_mode = true;
+
     return true;
 }
 
+void lapic_eoi(void) {
+    if (!x2apic_mode) {
+        lapic_write(0xb0, 0);
+    } else {
+        x2apic_write(0xb0, 0);
+    }
+}
+
 uint64_t x2apic_read(uint32_t reg) {
     return rdmsr(0x800 + (reg >> 4));
 }
@@ -122,7 +134,6 @@ void init_io_apics(void) {
     io_apics = ext_mem_alloc(max_io_apics * sizeof(struct madt_io_apic *));
     max_io_apics = 0;
 
-    // Try to start all APs
     for (uint8_t *madt_ptr = (uint8_t *)madt->madt_entries_begin;
       (uintptr_t)madt_ptr < (uintptr_t)madt + madt->header.length;
       madt_ptr += *(madt_ptr + 1)) {

stage23/sys/lapic.h

@@ -11,6 +11,7 @@
 #define LAPIC_REG_EOI      0x0b0
 
 bool lapic_check(void);
+void lapic_eoi(void);
 uint32_t lapic_read(uint32_t reg);
 void lapic_write(uint32_t reg, uint32_t data);
 

stage23/sys/smp.c

@@ -39,12 +39,6 @@ struct madt_x2apic {
     uint32_t acpi_processor_uid;
 } __attribute__((packed));
 
-static void delay(uint64_t cycles) {
-    uint64_t next_stop = rdtsc() + cycles;
-
-    while (rdtsc() < next_stop);
-}
-
 extern symbol _binary_smp_trampoline_bin_start;
 extern symbol _binary_smp_trampoline_bin_end;