Aren/haiku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x86[_64]: Support assigning MSI IRQs to arbitrary CPU

Browse Source
This commit is contained in:
Pawel Dziepak 2013-12-20 01:07:08 +01:00
parent caf1b0dffb
commit 6639514437
7 changed files with 152 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
headers/private/kernel/arch/x86/arch_int.h
View File

@ -11,6 +11,13 @@
#define NUM_IO_VECTORS (256 - ARCH_INTERRUPT_BASE)
enum irq_source {
IRQ_SOURCE_INVALID,
IRQ_SOURCE_IOAPIC,
IRQ_SOURCE_MSI,
};
static inline void
arch_int_enable_interrupts_inline(void)
{
@ -72,6 +79,8 @@ typedef struct interrupt_controller_s {
} interrupt_controller;
void x86_set_irq_source(int irq, irq_source source);
void arch_int_set_interrupt_controller(const interrupt_controller &controller);
#endif // __cplusplus

1
headers/private/kernel/arch/x86/msi.h
View File

@ -28,5 +28,6 @@ bool msi_supported();
status_t msi_allocate_vectors(uint8 count, uint8 *startVector,
uint64 *address, uint16 *data);
void msi_free_vectors(uint8 count, uint8 startVector);
void msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu);
#endif // _KERNEL_ARCH_x86_MSI_H

11
headers/private/kernel/int.h
View File

@ -32,13 +32,13 @@ enum interrupt_type {
struct irq_assignment {
list_link link;
uint32 irq;
uint32 count;
int32 handlers_count;
spinlock load_lock;
bigtime_t last_measure_time;
bigtime_t last_measure_active;
int32 load;
int32 cpu;
};
@ -79,7 +79,8 @@ are_interrupts_enabled(void)
status_t reserve_io_interrupt_vectors(long count, long startVector,
enum interrupt_type type);
status_t allocate_io_interrupt_vectors(long count, long *startVector);
status_t allocate_io_interrupt_vectors(long count, long *startVector,
enum interrupt_type type);
void free_io_interrupt_vectors(long count, long startVector);
void assign_io_interrupt_to_cpu(long vector, int32 cpu);

26
src/system/kernel/arch/x86/arch_int.cpp
View File

@ -24,6 +24,7 @@
#include <arch/x86/apic.h>
#include <arch/x86/descriptors.h>
#include <arch/x86/msi.h>
#include <arch/x86/msi_priv.h>
#include <stdio.h>
@ -41,6 +42,8 @@
#endif
static irq_source sVectorSources[NUM_IO_VECTORS];
static const char *kInterruptNames[] = {
/* 0 */ "Divide Error Exception",
/* 1 */ "Debug Exception",
@ -331,6 +334,13 @@ x86_page_fault_exception(struct iframe* frame)
}
void
x86_set_irq_source(int irq, irq_source source)
{
sVectorSources[irq] = source;
}
// #pragma mark -
@ -392,8 +402,20 @@ arch_int_are_interrupts_enabled(void)
void
arch_int_assign_to_cpu(int32 irq, int32 cpu)
{
if (sCurrentPIC->assign_interrupt_to_cpu != NULL)
sCurrentPIC->assign_interrupt_to_cpu(irq, cpu);
dprintf("ASSIGN IRQ TO CPU\n");
switch (sVectorSources[irq]) {
case IRQ_SOURCE_IOAPIC:
if (sCurrentPIC->assign_interrupt_to_cpu != NULL)
sCurrentPIC->assign_interrupt_to_cpu(irq, cpu);
break;
case IRQ_SOURCE_MSI:
msi_assign_interrupt_to_cpu(irq, cpu);
break;
default:
break;
}
}

8
src/system/kernel/arch/x86/ioapic.cpp
View File

@ -280,6 +280,8 @@ ioapic_enable_io_interrupt(int32 gsi)
if (ioapic == NULL)
return;
x86_set_irq_source(gsi, IRQ_SOURCE_IOAPIC);
uint8 pin = gsi - ioapic->global_interrupt_base;
TRACE("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n",
gsi, ioapic->number, pin);
@ -801,6 +803,12 @@ ioapic_init(kernel_args* args)
while (current != NULL) {
reserve_io_interrupt_vectors(current->max_redirection_entry + 1,
current->global_interrupt_base, INTERRUPT_TYPE_IRQ);
for (int32 i = 0; i < current->max_redirection_entry + 1; i++) {
x86_set_irq_source(current->global_interrupt_base + i,
IRQ_SOURCE_IOAPIC);
}
current = current->next;
}

27
src/system/kernel/arch/x86/msi.cpp
View File

@ -5,12 +5,20 @@
#include <arch/x86/apic.h>
#include <arch/x86/msi.h>
#include <arch/x86/arch_smp.h>
#include <debug.h>
#include <int.h>
#include <lock.h>
struct MSIConfiguration {
uint64* fAddress;
uint16* fData;
};
static MSIConfiguration sMSIConfigurations[NUM_IO_VECTORS];
static bool sMSISupported = false;
static uint32 sBootCPUAPICId = 0;
@ -44,7 +52,8 @@ msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address,
return B_UNSUPPORTED;
long vector;
status_t result = allocate_io_interrupt_vectors(count, &vector);
status_t result = allocate_io_interrupt_vectors(count, &vector,
INTERRUPT_TYPE_IRQ);
if (result != B_OK)
return result;
@ -53,6 +62,10 @@ msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address,
return B_NO_MEMORY;
}
sMSIConfigurations[vector].fAddress = address;
sMSIConfigurations[vector].fData = data;
x86_set_irq_source(vector, IRQ_SOURCE_MSI);
*startVector = (uint8)vector;
*address = MSI_ADDRESS_BASE | (sBootCPUAPICId << MSI_DESTINATION_ID_SHIFT)
| MSI_NO_REDIRECTION | MSI_DESTINATION_MODE_PHYSICAL;
@ -78,3 +91,15 @@ msi_free_vectors(uint8 count, uint8 startVector)
free_io_interrupt_vectors(count, startVector);
}
void
msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu)
{
uint32 apic_id = x86_get_cpu_apic_id(cpu);
uint64* address = sMSIConfigurations[irq].fAddress;
*address = MSI_ADDRESS_BASE | (apic_id << MSI_DESTINATION_ID_SHIFT)
| MSI_NO_REDIRECTION | MSI_DESTINATION_MODE_PHYSICAL;
}

125
src/system/kernel/int.cpp
View File

@ -57,7 +57,12 @@ struct io_vector {
bool no_lock_vector;
interrupt_type type;
irq_assignment assigned_cpu;
spinlock load_lock;
bigtime_t last_measure_time;
bigtime_t last_measure_active;
int32 load;
irq_assignment* assigned_cpu;
#if DEBUG_INTERRUPTS
int64 handled_count;
@ -69,8 +74,9 @@ struct io_vector {
static int32 sLastCPU;
static struct io_vector sVectors[NUM_IO_VECTORS];
static io_vector sVectors[NUM_IO_VECTORS];
static bool sAllocatedIOInterruptVectors[NUM_IO_VECTORS];
static irq_assignment sVectorCPUAssignments[NUM_IO_VECTORS];
static mutex sIOInterruptVectorAllocationLock
= MUTEX_INITIALIZER("io_interrupt_vector_allocation");
@ -141,11 +147,15 @@ dump_int_load(int argc, char** argv)
kprintf("int %3d, type %s, enabled %" B_PRId32 ", load %" B_PRId32
"%%", i, typeNames[min_c(sVectors[i].type,
INTERRUPT_TYPE_UNKNOWN)],
sVectors[i].enable_count, sVectors[i].assigned_cpu.load / 10);
sVectors[i].enable_count,
sVectors[i].assigned_cpu != NULL
? sVectors[i].assigned_cpu->load / 10 : 0);
if (sVectors[i].type == INTERRUPT_TYPE_IRQ) {
if (sVectors[i].assigned_cpu.cpu != -1)
kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu.cpu);
ASSERT(sVectors[i].assigned_cpu != NULL);
if (sVectors[i].assigned_cpu->cpu != -1)
kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu->cpu);
else
kprintf(", cpu -");
}
@ -190,15 +200,10 @@ int_init_post_vm(kernel_args* args)
sVectors[i].no_lock_vector = false;
sVectors[i].type = INTERRUPT_TYPE_UNKNOWN;
irq_assignment* assigned_cpu = &sVectors[i].assigned_cpu;
assigned_cpu->irq = i;
B_INITIALIZE_SPINLOCK(&assigned_cpu->load_lock);
assigned_cpu->last_measure_time = 0;
assigned_cpu->last_measure_active = 0;
assigned_cpu->load = 0;
assigned_cpu->cpu = -1;
B_INITIALIZE_SPINLOCK(&sVectors[i].load_lock);
sVectors[i].last_measure_time = 0;
sVectors[i].last_measure_active = 0;
sVectors[i].load = 0;
#if DEBUG_INTERRUPTS
sVectors[i].handled_count = 0;
@ -207,6 +212,12 @@ int_init_post_vm(kernel_args* args)
sVectors[i].ignored_count = 0;
#endif
sVectors[i].handler_list = NULL;
sVectorCPUAssignments[i].irq = i;
sVectorCPUAssignments[i].count = 1;
sVectorCPUAssignments[i].handlers_count = 0;
sVectorCPUAssignments[i].load = 0;
sVectorCPUAssignments[i].cpu = -1;
}
#if DEBUG_INTERRUPTS
@ -240,14 +251,17 @@ int_init_post_device_manager(kernel_args* args)
static void
update_int_load(int i)
{
if (!try_acquire_spinlock(&sVectors[i].assigned_cpu.load_lock))
if (!try_acquire_spinlock(&sVectors[i].load_lock))
return;
compute_load(sVectors[i].assigned_cpu.last_measure_time,
sVectors[i].assigned_cpu.last_measure_active,
sVectors[i].assigned_cpu.load);
int32 oldLoad = sVectors[i].load;
compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active,
sVectors[i].load);
release_spinlock(&sVectors[i].assigned_cpu.load_lock);
if (oldLoad != sVectors[i].load)
atomic_add(&sVectors[i].assigned_cpu->load, sVectors[i].load - oldLoad);
release_spinlock(&sVectors[i].load_lock);
}
@ -345,9 +359,9 @@ int_io_interrupt_handler(int vector, bool levelTriggered)
if (!sVectors[vector].no_lock_vector)
release_spinlock(&sVectors[vector].vector_lock);
SpinLocker locker(sVectors[vector].assigned_cpu.load_lock);
SpinLocker locker(sVectors[vector].load_lock);
bigtime_t deltaTime = system_time() - start;
sVectors[vector].assigned_cpu.last_measure_active += deltaTime;
sVectors[vector].last_measure_active += deltaTime;
locker.Unlock();
atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime);
@ -444,18 +458,17 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data,
// Initial attempt to balance IRQs, the scheduler will correct this
// if some cores end up being overloaded.
if (sVectors[vector].type == INTERRUPT_TYPE_IRQ
&& sVectors[vector].handler_list == NULL) {
&& sVectors[vector].handler_list == NULL
&& sVectors[vector].assigned_cpu->cpu == -1) {
int32 cpuID = assign_cpu();
arch_int_assign_to_cpu(vector, cpuID);
ASSERT(sVectors[vector].assigned_cpu.cpu == -1);
sVectors[vector].assigned_cpu.cpu = cpuID;
sVectors[vector].assigned_cpu->cpu = cpuID;
cpu_ent* cpu = &gCPU[cpuID];
SpinLocker _(cpu->irqs_lock);
list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
atomic_add(&sVectors[vector].assigned_cpu->handlers_count, 1);
list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu);
}
if ((flags & B_NO_HANDLED_INFO) != 0
@ -544,25 +557,32 @@ remove_io_interrupt_handler(long vector, interrupt_handler handler, void *data)
}
if (sVectors[vector].handler_list == NULL
&& sVectors[vector].type == INTERRUPT_TYPE_IRQ) {
&& sVectors[vector].type == INTERRUPT_TYPE_IRQ
&& sVectors[vector].assigned_cpu != NULL
&& sVectors[vector].assigned_cpu->handlers_count > 0) {
int32 oldCPU;
SpinLocker locker;
cpu_ent* cpu;
int32 oldHandlersCount
= atomic_add(&sVectors[vector].assigned_cpu->handlers_count, -1);
do {
locker.Unlock();
if (oldHandlersCount == 1) {
int32 oldCPU;
SpinLocker locker;
cpu_ent* cpu;
oldCPU = sVectors[vector].assigned_cpu.cpu;
do {
locker.Unlock();
ASSERT(oldCPU != -1);
cpu = &gCPU[oldCPU];
oldCPU = sVectors[vector].assigned_cpu->cpu;
locker.SetTo(cpu->irqs_lock, false);
} while (sVectors[vector].assigned_cpu.cpu != oldCPU);
ASSERT(oldCPU != -1);
cpu = &gCPU[oldCPU];
sVectors[vector].assigned_cpu.cpu = -1;
list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
locker.SetTo(cpu->irqs_lock, false);
} while (sVectors[vector].assigned_cpu->cpu != oldCPU);
sVectors[vector].assigned_cpu->cpu = -1;
list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu);
}
}
release_spinlock(&sVectors[vector].vector_lock);
@ -596,6 +616,9 @@ reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type)
}
sVectors[startVector + i].type = type;
sVectors[startVector + i].assigned_cpu
= &sVectorCPUAssignments[startVector + i];
sVectorCPUAssignments[startVector + i].count = 1;
sAllocatedIOInterruptVectors[startVector + i] = true;
}
@ -610,7 +633,8 @@ reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type)
The first vector to be used is returned in \a startVector on success.
*/
status_t
allocate_io_interrupt_vectors(long count, long *startVector)
allocate_io_interrupt_vectors(long count, long *startVector,
interrupt_type type)
{
MutexLocker locker(&sIOInterruptVectorAllocationLock);
@ -639,8 +663,14 @@ allocate_io_interrupt_vectors(long count, long *startVector)
return B_NO_MEMORY;
}
for (long i = 0; i < count; i++)
for (long i = 0; i < count; i++) {
sVectors[vector + i].type = type;
sVectors[vector + i].assigned_cpu = &sVectorCPUAssignments[vector];
sAllocatedIOInterruptVectors[vector + i] = true;
}
sVectorCPUAssignments[vector].irq = vector;
sVectorCPUAssignments[vector].count = count;
*startVector = vector;
dprintf("allocate_io_interrupt_vectors: allocated %ld vectors starting "
@ -672,6 +702,7 @@ free_io_interrupt_vectors(long count, long startVector)
startVector + i);
}
sVectors[startVector + i].assigned_cpu = NULL;
sAllocatedIOInterruptVectors[startVector + i] = false;
}
}
@ -681,7 +712,7 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU)
{
ASSERT(sVectors[vector].type == INTERRUPT_TYPE_IRQ);
int32 oldCPU = sVectors[vector].assigned_cpu.cpu;
int32 oldCPU = sVectors[vector].assigned_cpu->cpu;
if (newCPU == -1)
newCPU = assign_cpu();
@ -693,14 +724,14 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU)
cpu_ent* cpu = &gCPU[oldCPU];
SpinLocker locker(cpu->irqs_lock);
sVectors[vector].assigned_cpu.cpu = -1;
list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
sVectors[vector].assigned_cpu->cpu = -1;
list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu);
locker.Unlock();
cpu = &gCPU[newCPU];
locker.SetTo(cpu->irqs_lock, false);
sVectors[vector].assigned_cpu.cpu = newCPU;
sVectors[vector].assigned_cpu->cpu = newCPU;
arch_int_assign_to_cpu(vector, newCPU);
list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu);
}