Virtio PCI: added support for MSI-X interrupts

* make use of MSI/MSI-X PCI x86 API
* MSI support untested because QEmu only offers MSI-X
* changed a bit the Virtio bus API by adding a queue count parameter
for the setup_interrupt() hook.

headers/private/virtio/virtio.h

@@ -84,7 +84,7 @@ typedef struct {
 	
 	uint16	(*get_queue_ring_size)(void* cookie, uint16 queue);
 	status_t (*setup_queue)(void* cookie, uint16 queue, phys_addr_t phy);
-	status_t (*setup_interrupt)(void* cookie);
+	status_t (*setup_interrupt)(void* cookie, uint16 queueCount);
 	void	(*notify_queue)(void* cookie, uint16 queue);
 } virtio_sim_interface;
 

src/add-ons/kernel/bus_managers/virtio/VirtioDevice.cpp

@@ -177,7 +177,7 @@ VirtioDevice::SetupInterrupt(virtio_intr_func configHandler,
 {
 	fConfigHandler = configHandler;
 	fConfigCookie = configCookie;
-	status_t status = fController->setup_interrupt(fCookie);
+	status_t status = fController->setup_interrupt(fCookie, fQueueCount);
 	if (status != B_OK)
 		return status;
 

src/add-ons/kernel/busses/virtio/virtio_pci.cpp

@@ -9,6 +9,7 @@
 #include <string.h>
 
 #include <bus/PCI.h>
+#include <PCI_x86.h>
 #include <virtio.h>
 
 #include "virtio_pci.h"
@@ -20,8 +21,9 @@
 #else
 #	define TRACE(x...) ;
 #endif
+#define TRACE_ALWAYS(x...)	dprintf("\33[33mvirtio_pci:\33[0m " x)
 #define ERROR(x...)			dprintf("\33[33mvirtio_pci:\33[0m " x)
-#define CALLED() 			TRACE("CALLED %s\n", __PRETTY_FUNCTION__)
+#define CALLED(x...)		TRACE("CALLED %s\n", __PRETTY_FUNCTION__)
 
 
 #define VIRTIO_PCI_DEVICE_MODULE_NAME "busses/virtio/virtio_pci/driver_v1"
@@ -29,21 +31,37 @@
 
 #define VIRTIO_PCI_CONTROLLER_TYPE_NAME "virtio pci controller"
 
+typedef enum {
+	VIRTIO_IRQ_LEGACY,
+	VIRTIO_IRQ_MSI,
+	VIRTIO_IRQ_MSI_X_SHARED,
+	VIRTIO_IRQ_MSI_X,
+} virtio_irq_type;
+
+typedef struct {
+	virtio_sim sim;
+	uint16 queue;
+} virtio_pci_queue_cookie;
 
 typedef struct {
 	pci_device_module_info* pci;
 	pci_device* device;
-	uint16 config_base;
 	addr_t base_addr;
 	uint8 irq;
+	virtio_irq_type irq_type;
 	virtio_sim sim;
+	uint16 queue_count;
 
 	device_node* node;
+	pci_info info;
+
+	virtio_pci_queue_cookie *cookies;
 } virtio_pci_sim_info;
 
 
 device_manager_info* gDeviceManager;
 virtio_for_controller_interface* gVirtio;
+static pci_x86_module_info* sPCIx86Module;
 
 
 int32
@@ -65,6 +83,61 @@ virtio_pci_interrupt(void *data)
 }
 
 
+int32
+virtio_pci_config_interrupt(void *data)
+{
+	virtio_pci_sim_info* bus = (virtio_pci_sim_info*)data;
+	gVirtio->config_interrupt_handler(bus->sim);
+
+	return B_HANDLED_INTERRUPT;
+}
+
+
+int32
+virtio_pci_queue_interrupt(void *data)
+{
+	virtio_pci_queue_cookie* cookie = (virtio_pci_queue_cookie*)data;
+	gVirtio->queue_interrupt_handler(cookie->sim, cookie->queue);
+
+	return B_HANDLED_INTERRUPT;
+}
+
+
+status_t
+virtio_pci_setup_msix_interrupts(virtio_pci_sim_info* bus)
+{
+	CALLED();
+	uint8 irq = 0; // first irq slot
+	bus->pci->write_io_16(bus->device, bus->base_addr
+		+ VIRTIO_MSI_CONFIG_VECTOR, irq);
+	if (bus->pci->read_io_16(bus->device, bus->base_addr
+		+ VIRTIO_MSI_CONFIG_VECTOR) == VIRTIO_MSI_NO_VECTOR) {
+		ERROR("msix config vector incorrect\n");
+		return B_BAD_VALUE;
+	}
+	if (bus->irq_type == VIRTIO_IRQ_MSI_X)
+		irq++;
+
+	for (uint16 queue = 0; queue < bus->queue_count; queue++) {
+		bus->pci->write_io_16(bus->device, bus->base_addr
+			+ VIRTIO_PCI_QUEUE_SEL, queue);
+		bus->pci->write_io_16(bus->device, bus->base_addr
+			+ VIRTIO_MSI_QUEUE_VECTOR, irq);
+
+		if (bus->pci->read_io_16(bus->device, bus->base_addr
+			+ VIRTIO_MSI_QUEUE_VECTOR) == VIRTIO_MSI_NO_VECTOR) {
+			ERROR("msix queue vector incorrect\n");
+			return B_BAD_VALUE;
+		}
+	
+		if (bus->irq_type == VIRTIO_IRQ_MSI_X)
+			irq++;
+	}
+
+	return B_OK;
+}
+
+
 static void
 set_sim(void* cookie, virtio_sim sim)
 {
@@ -127,7 +200,7 @@ read_device_config(void* cookie, uint8 _offset, void* _buffer,
 	CALLED();
 	virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie;
 
-	addr_t offset = bus->base_addr + bus->config_base + _offset;
+	addr_t offset = bus->base_addr + VIRTIO_PCI_CONFIG(bus) + _offset;
 	uint8* buffer = (uint8*)_buffer;
 	while (bufferSize > 0) {
 		uint8 size = 4;
@@ -159,7 +232,7 @@ write_device_config(void* cookie, uint8 _offset, const void* _buffer,
 	CALLED();
 	virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie;
 
-	addr_t offset = bus->base_addr + bus->config_base + _offset;
+	addr_t offset = bus->base_addr + VIRTIO_PCI_CONFIG(bus) + _offset;
 	const uint8* buffer = (const uint8*)_buffer;
 	while (bufferSize > 0) {
 		uint8 size = 4;
@@ -209,17 +282,108 @@ setup_queue(void* cookie, uint16 queue, phys_addr_t phy)
 
 
 status_t 
-setup_interrupt(void* cookie)
+setup_interrupt(void* cookie, uint16 queueCount)
 {
 	CALLED();
 	virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie;
-	
-	// setup interrupt handler
-	status_t status = install_io_interrupt_handler(bus->irq,
-		virtio_pci_interrupt, bus, 0);
-	if (status != B_OK) {
-		ERROR("can't install interrupt handler\n");
-		return status;
+	pci_info *pciInfo = &bus->info;
+
+	bus->queue_count = queueCount;
+
+	if (sPCIx86Module != NULL) {
+		// try MSI-X
+		uint8 msixCount = sPCIx86Module->get_msix_count(
+			pciInfo->bus, pciInfo->device, pciInfo->function);
+		if (msixCount >= 1) {
+			if (msixCount >= (queueCount + 1)) {
+				uint8 vector;
+				bus->cookies = new(std::nothrow)
+					virtio_pci_queue_cookie[queueCount];
+				if (bus->cookies != NULL
+					&& sPCIx86Module->configure_msix(pciInfo->bus,
+						pciInfo->device, pciInfo->function, queueCount + 1,
+						&vector) == B_OK
+					&& sPCIx86Module->enable_msix(pciInfo->bus, pciInfo->device,
+						pciInfo->function) == B_OK) {
+					TRACE_ALWAYS("using MSI-X count %u starting at %d\n",
+						queueCount + 1, vector);
+					bus->irq = vector;
+					bus->irq_type = VIRTIO_IRQ_MSI_X;
+				} else {
+					ERROR("couldn't use MSI-X\n");
+				}
+			} else {
+				uint8 vector;
+				if (sPCIx86Module->configure_msix(pciInfo->bus, pciInfo->device,
+						pciInfo->function, queueCount + 1, &vector) == B_OK
+					&& sPCIx86Module->enable_msix(pciInfo->bus, pciInfo->device,
+						pciInfo->function) == B_OK) {
+					TRACE_ALWAYS("using MSI-X vector shared %u\n", 1);
+					bus->irq = vector;
+					bus->irq_type = VIRTIO_IRQ_MSI_X_SHARED;
+				} else {
+					ERROR("couldn't use MSI-X SHARED\n");
+				}
+			}
+		} else if (sPCIx86Module->get_msi_count(
+			pciInfo->bus, pciInfo->device, pciInfo->function) >= 1) {
+			// try MSI 
+			uint8 vector;
+			if (sPCIx86Module->configure_msi(pciInfo->bus, pciInfo->device,
+					pciInfo->function, 1, &vector) == B_OK
+				&& sPCIx86Module->enable_msi(pciInfo->bus, pciInfo->device,
+					pciInfo->function) == B_OK) {
+				TRACE_ALWAYS("using MSI vector %u\n", vector);
+				bus->irq = vector;
+				bus->irq_type = VIRTIO_IRQ_MSI;
+			} else {
+				ERROR("couldn't use MSI\n");
+			}
+		}
+	}
+	if (bus->irq_type == VIRTIO_IRQ_LEGACY) {
+		bus->irq = pciInfo->u.h0.interrupt_line;
+		TRACE_ALWAYS("using legacy interrupt %u\n", bus->irq);
+	}
+	if (bus->irq == 0 || bus->irq == 0xff) {
+		ERROR("PCI IRQ not assigned\n");
+		if (sPCIx86Module != NULL) {
+			put_module(B_PCI_X86_MODULE_NAME);
+			sPCIx86Module = NULL;
+		}
+		delete bus;
+		return B_ERROR;
+	}
+
+	if (bus->irq_type == VIRTIO_IRQ_MSI_X) {
+		status_t status = install_io_interrupt_handler(bus->irq,
+			virtio_pci_config_interrupt, bus, 0);
+		if (status != B_OK) {
+			ERROR("can't install interrupt handler\n");
+			return status;
+		}
+		int32 irq = bus->irq + 1;
+		for (int32 queue = 0; queue < queueCount; queue++, irq++) {
+			bus->cookies[queue].sim = bus->sim;
+			bus->cookies[queue].queue = queue;
+			status_t status = install_io_interrupt_handler(irq,
+				virtio_pci_queue_interrupt, &bus->cookies[queue], 0);
+			if (status != B_OK) {
+				ERROR("can't install interrupt handler\n");
+				return status;
+			}
+		}
+		TRACE("setup_interrupt() installed MSI-X interrupt handlers\n");
+		virtio_pci_setup_msix_interrupts(bus);
+	} else {
+		// setup interrupt handler
+		status_t status = install_io_interrupt_handler(bus->irq,
+			virtio_pci_interrupt, bus, 0);
+		if (status != B_OK) {
+			ERROR("can't install interrupt handler\n");
+			return status;
+		}
+		TRACE("setup_interrupt() installed legacy interrupt handler\n");
 	}
 
 	return B_OK;
@@ -261,23 +425,22 @@ init_bus(device_node* node, void** bus_cookie)
 		gDeviceManager->put_node(parent);
 	}
 
+	if (get_module(B_PCI_X86_MODULE_NAME, (module_info**)&sPCIx86Module)
+			!= B_OK) {
+		sPCIx86Module = NULL;
+	}
+
 	bus->node = node;
 	bus->pci = pci;
 	bus->device = device;
-	// TODO MSI implies 24
-	bus->config_base = 20;
+	bus->cookies = NULL;
+	bus->irq_type = VIRTIO_IRQ_LEGACY;
 
-	pci_info pciInfo;
-	pci->get_pci_info(device, &pciInfo);
+	pci_info *pciInfo = &bus->info;
+	pci->get_pci_info(device, pciInfo);
 
 	// legacy interrupt
-	bus->base_addr = pciInfo.u.h0.base_registers[0];
-	bus->irq = pciInfo.u.h0.interrupt_line;
-	if (bus->irq == 0 || bus->irq == 0xff) {
-		ERROR("PCI IRQ not assigned\n");
-		delete bus;
-		return B_ERROR;
-	}
+	bus->base_addr = pciInfo->u.h0.base_registers[0];
 
 	// enable bus master and io
 	uint16 pcicmd = pci->read_pci_config(device, PCI_command, 2);
@@ -300,6 +463,33 @@ static void
 uninit_bus(void* bus_cookie)
 {
 	virtio_pci_sim_info* bus = (virtio_pci_sim_info*)bus_cookie;
+	if (bus->irq_type != VIRTIO_IRQ_LEGACY) {
+		if (bus->irq_type == VIRTIO_IRQ_MSI) {
+			remove_io_interrupt_handler(bus->irq, virtio_pci_interrupt, bus);
+			sPCIx86Module->disable_msi(bus->info.bus,
+				bus->info.device, bus->info.function);
+			sPCIx86Module->unconfigure_msi(bus->info.bus,
+				bus->info.device, bus->info.function);
+		} else {
+			int32 irq = bus->irq + 1;
+			for (uint16 queue = 0; queue < bus->queue_count; queue++, irq++) {
+				remove_io_interrupt_handler(irq, virtio_pci_queue_interrupt,
+					&bus->cookies[queue]);
+			}
+			remove_io_interrupt_handler(bus->irq, virtio_pci_config_interrupt,
+					bus);
+			sPCIx86Module->disable_msix(bus->info.bus,
+				bus->info.device, bus->info.function);
+			sPCIx86Module->unconfigure_msix(bus->info.bus,
+				bus->info.device, bus->info.function);
+		}
+	} else
+		remove_io_interrupt_handler(bus->irq, virtio_pci_interrupt, bus);
+	if (sPCIx86Module != NULL) {
+		put_module(B_PCI_X86_MODULE_NAME);
+		sPCIx86Module = NULL;
+	}
+	delete[] bus->cookies;
 	delete bus;
 }
 

src/add-ons/kernel/busses/virtio/virtio_pci.h

@@ -72,8 +72,8 @@
  * The remaining space is defined by each driver as the per-driver
  * configuration space.
  */
-#define VIRTIO_PCI_CONFIG(sc) \
-    (((sc)->vtpci_flags & VTPCI_FLAG_MSIX) ? 24 : 20)
+#define VIRTIO_PCI_CONFIG(bus) \
+    ((bus->irq_type != VIRTIO_IRQ_LEGACY) ? 24 : 20)
 
 /*
  * How many bits to shift physical queue address written to QUEUE_PFN.