/** * @brief xHCI Host Controller Driver * @file modules/xhci.c * @package x86_64 * * @warning This is a stub driver. * * @copyright * This file is part of ToaruOS and is released under the terms * of the NCSA / University of Illinois License - see LICENSE.md * Copyright (C) 2021 K. Lange */ #include #include #include #include #include #include #include #include #include #include static void delay_yield(size_t subticks) { #ifdef __aarch64__ asm volatile ("dsb sy\nisb" ::: "memory"); #endif unsigned long s, ss; relative_time(0, subticks, &s, &ss); sleep_until((process_t *)this_core->current_process, s, ss); switch_task(0); #ifdef __aarch64__ asm volatile ("dmb sy\n" ::: "memory"); #endif } struct xhci_cap_regs { volatile uint32_t cap_caplen_version; volatile uint32_t cap_hcsparams1; volatile uint32_t cap_hcsparams2; volatile uint32_t cap_hcsparams3; volatile uint32_t cap_hccparams1; volatile uint32_t cap_dboff; volatile uint32_t cap_rtsoff; volatile uint32_t cap_hccparams2; } __attribute__((packed)); struct xhci_port_regs { volatile uint32_t port_status; volatile uint32_t port_pm_status; volatile uint32_t port_link_info; volatile uint32_t port_lpm_control; } __attribute__((packed)); struct xhci_op_regs { volatile uint32_t op_usbcmd; /* 0 */ volatile uint32_t op_usbsts; // 4 volatile uint32_t op_pagesize; // 8h volatile uint32_t op__pad1[2]; // ch 10h volatile uint32_t op_dnctrl; // 14h volatile uint32_t op_crcr[2]; // 18h 1ch volatile uint32_t op__pad2[4]; // 20h 24h 28h 2ch volatile uint32_t op_dcbaap[2]; // 30h 34h volatile uint32_t op_config; // 38h volatile uint8_t op_more_padding[964]; // 3ch-400h struct xhci_port_regs op_portregs[256]; } __attribute__((packed)); struct xhci_trb { uint32_t trb_thing_a; uint32_t trb_thing_b; uint32_t trb_status; uint32_t trb_control; } __attribute__((packed)); struct XHCIControllerData { uintptr_t mmio; uint32_t device; uint64_t pcie_offset; struct xhci_cap_regs * cregs; struct xhci_op_regs * oregs; process_t * thread; volatile struct xhci_trb * cr_trbs; volatile struct xhci_trb * er_trbs; spin_lock_t command_queue; uint32_t command_queue_cycle; int command_queue_enq; volatile uint32_t * doorbells; }; static uint64_t pci_addr_map(struct XHCIControllerData * controller, uint64_t addr) { return addr + controller->pcie_offset; } static uintptr_t pci_to_cpu(struct XHCIControllerData * controller, uint64_t addr) { return addr - controller->pcie_offset; } static uintptr_t allocate_page(uint64_t * phys_out) { uint64_t phys = mmu_allocate_a_frame() << 12; uintptr_t virt = (uintptr_t)mmu_map_mmio_region(phys, 4096); memset((void*)virt,0,4096); *phys_out = phys; return virt; } static int xhci_command(struct XHCIControllerData * controller, uint32_t p1, uint32_t p2, uint32_t status, uint32_t control) { spin_lock(controller->command_queue); control &= ~1; control |= controller->command_queue_cycle; controller->cr_trbs[controller->command_queue_enq].trb_thing_a = p1; controller->cr_trbs[controller->command_queue_enq].trb_thing_b = p2; controller->cr_trbs[controller->command_queue_enq].trb_status = status; controller->cr_trbs[controller->command_queue_enq].trb_control = control; controller->command_queue_enq++; if (controller->command_queue_enq == 63) { controller->cr_trbs[controller->command_queue_enq].trb_control ^= 1; if (controller->cr_trbs[controller->command_queue_enq].trb_control & (1 << 1)) { controller->command_queue_cycle ^= 1; } controller->command_queue_enq = 0; } /* ring doorbell */ controller->doorbells[0] = 0; spin_unlock(controller->command_queue); return 0; } static ssize_t xhci_write(fs_node_t * node, off_t offset, size_t size, uint8_t * buffer) { struct XHCIControllerData * controller = node->device; if (size != sizeof(struct xhci_trb)) return -EINVAL; struct xhci_trb * data = (void*)buffer; xhci_command(controller, data->trb_thing_a, data->trb_thing_b, data->trb_status, data->trb_control); return sizeof(struct xhci_trb); } static struct XHCIControllerData * _irq_owner = NULL; #include static int irq_handler(struct regs *r) { int irq = r->int_no - 32; if (_irq_owner) { /* Is it ours? */ uint32_t status = _irq_owner->oregs->op_usbsts; if (status & (1 << 3)) { _irq_owner->oregs->op_usbsts = (1 << 3); dprintf("xhci: irq\n"); uintptr_t rts = (uintptr_t)_irq_owner->cregs + _irq_owner->cregs->cap_rtsoff; volatile uint32_t * irs0_32 = (uint32_t*)(rts + 0x20); irs0_32[0] |= 1; make_process_ready(_irq_owner->thread); irq_ack(irq); return 1; } } return 0; } void xhci_thread(void * arg) { struct XHCIControllerData * controller = arg; controller->thread = (process_t*)this_core->current_process; spin_init(controller->command_queue); /* Begin generic XHCi */ dprintf("xhci: available slots: %d\n", controller->cregs->cap_hcsparams1 & 0xFF); dprintf("xhci: available ports: %d\n", controller->cregs->cap_hcsparams1 >> 24); dprintf("xhci: resetting controller\n"); dprintf("xhci: waiting for controller to stop...\n"); uint32_t cmd = controller->oregs->op_usbcmd; cmd &= ~(1); controller->oregs->op_usbcmd = cmd; while (!(controller->oregs->op_usbsts & (1 << 0))); dprintf("xhci: restarting controller...\n"); cmd = controller->oregs->op_usbcmd; cmd |= (1 << 1); controller->oregs->op_usbcmd = cmd; while ((controller->oregs->op_usbcmd & (1 << 1))); while ((controller->oregs->op_usbsts & (1 << 11))); dprintf("xhci: controller is ready: %#x\n", controller->oregs->op_usbsts); dprintf("xhci: slot config %#x -> %#x\n", controller->oregs->op_config, controller->cregs->cap_hcsparams1 & 0xFF); controller->oregs->op_config = controller->cregs->cap_hcsparams1 & 0xFF; /* TODO We may need to clear interrupts here by writing status back */ uint32_t sts = controller->oregs->op_usbsts; (void)sts; dprintf("xhci: context size is %d\n", (controller->cregs->cap_hccparams1 & (1 << 2)) ? 64 : 32); uintptr_t ext_off = (controller->cregs->cap_hccparams1 >> 16) << 2; volatile uint32_t * ext_caps = (void*)((uintptr_t)controller->cregs + ext_off); /** * Verify port configurations; * should be port 1 is usb 2.0 * port 2, 3, 4, 5 are 3.0 * port 1 has a hub with 4 ports? */ while (1) { uint32_t cap_val = *ext_caps; dprintf("xhci: ecap = %#x\n", cap_val); /* Bottom byte is type */ if ((cap_val & 0xFF) == 2) { uint8_t rev_minor = ext_caps[0] >> 16; uint8_t rev_major = ext_caps[0] >> 24; //uint32_t name_str = ext_caps[1]; uint8_t port_offset = ext_caps[2]; uint8_t port_count = ext_caps[2] >> 8; uint8_t psic = ext_caps[2] >> 28; dprintf("xhci: protocol %d.%d %d port%s starting from port %d has %d speed%s\n", rev_major, rev_minor, port_count, &"s"[port_count==1], port_offset, psic, &"s"[psic==1]); } if (cap_val == 0xFFFFffff) break; if ((cap_val & 0xFF00) == 0) break; ext_caps = (void*)((uintptr_t)ext_caps + ((cap_val & 0xFF00) >> 6)); } /* Device Context Base Address Array */ uint64_t dcbaap; uint64_t * baseCtx = (void*)allocate_page(&dcbaap); dprintf("xhci: DCBAAP at %#zx (phys=%#zx)\n", (uintptr_t)baseCtx, dcbaap); controller->oregs->op_dcbaap[0] = pci_addr_map(controller, dcbaap); controller->oregs->op_dcbaap[1] = pci_addr_map(controller, dcbaap) >> 32; /* Enable slots */ uint32_t cfg = controller->oregs->op_config; cfg &= ~0xFF; cfg |= 32; dprintf("xhci: set cfg = %#x\n", cfg); controller->oregs->op_config = cfg; /* trbs for event ring */ uint64_t er_trbs_phys; void * er_trbs_virt = (void*)allocate_page(&er_trbs_phys); dprintf("xhci: er trbs = %#zx (phys=%#zx)\n", (uintptr_t)er_trbs_virt, er_trbs_phys); /* erst */ uint64_t er_erst_phys; void * er_erst_virt = (void*)allocate_page(&er_erst_phys); dprintf("xhci: er erst = %#zx (phys=%#zx)\n", (uintptr_t)er_erst_virt, er_erst_phys); ((volatile uint64_t*)er_erst_virt)[0] = pci_addr_map(controller, er_trbs_phys); ((volatile uint64_t*)er_erst_virt)[1] = 64; dprintf("xhci: rtsoff = %#x\n", controller->cregs->cap_rtsoff); uintptr_t rts = (uintptr_t)controller->cregs + controller->cregs->cap_rtsoff; /* Interrupter points to event ring */ volatile uint32_t * irs0_32 = (uint32_t*)(rts + 0x20); irs0_32[2] = 1; /* Size = 1 */ irs0_32[6] = pci_addr_map(controller, er_trbs_phys) | (1 << 3); irs0_32[7] = (pci_addr_map(controller, er_trbs_phys) | (1 << 3)) >> 32; irs0_32[1] = 500; /* IMOD */ irs0_32[0] = 2; /* enable interrupts */ irs0_32[4] = pci_addr_map(controller, er_erst_phys); irs0_32[5] = pci_addr_map(controller, er_erst_phys) >> 32; /* trbs for control ring */ uint64_t cr_trbs_phys; void * cr_trbs_virt = (void*)allocate_page(&cr_trbs_phys); ((volatile uint64_t*)cr_trbs_virt)[63*2] = pci_addr_map(controller, cr_trbs_phys); ((volatile uint64_t*)cr_trbs_virt)[63*2+1] = ((0x2UL | (6UL << 10)) << 32); controller->oregs->op_crcr[0] = (pci_addr_map(controller, cr_trbs_phys) | 1); controller->oregs->op_crcr[1] = (pci_addr_map(controller, cr_trbs_phys) | 1) >> 32; /* Scratchpad buffers, if needed */ uint32_t hcs2 = controller->cregs->cap_hcsparams2; uint32_t sb_hi = (hcs2 >> 21) & 0x1f; uint32_t sb_lo = (hcs2 >> 27) & 0x1f; uint32_t sb_max = (sb_hi << 5) | sb_lo; /* should be 31 */ if (sb_max) { dprintf("num scratchpad buffers = %u\n", sb_max); /* Allocate buffer for array */ uint64_t scratch_phys; uint64_t *scratch_virt = (uint64_t*)allocate_page(&scratch_phys); dprintf("xhci: scratch at %#zx (phys=%#zx)\n", (uintptr_t)scratch_virt, scratch_phys); /* Our DMA mapping should be 1:1, so, uh, yolo */ for (unsigned int i = 0; i < sb_max; ++i) { uint64_t sb_phys; allocate_page(&sb_phys); scratch_virt[i] = pci_addr_map(controller, sb_phys); } baseCtx[0] = pci_addr_map(controller, scratch_phys); dprintf("xhci: assigned scratchpad buffer array\n"); } /* TODO This irq API sucks */ int irq_number = pci_get_interrupt(controller->device); irq_install_handler(irq_number, irq_handler, "xhci"); _irq_owner = controller; dprintf("xhci: Starting command ring...\n"); { uint32_t cmd = controller->oregs->op_usbcmd; dprintf("cmd before = %#x\n", cmd); cmd |= (1 << 0) | (1 << 2); controller->oregs->op_usbcmd = cmd; } dprintf("xhci: status = %#x\n", controller->oregs->op_usbsts); delay_yield(50000); dprintf("xhci: status = %#x\n", controller->oregs->op_usbsts); if (controller->oregs->op_usbsts & (1 << 2)) goto _bail; dprintf("xhci: doorbells at %#x\n", controller->cregs->cap_dboff); controller->doorbells = (void*)((uintptr_t)controller->cregs + controller->cregs->cap_dboff); /* Just want to enable the hub for now, see if we can id it */ controller->cr_trbs = cr_trbs_virt; controller->er_trbs = er_trbs_virt; controller->command_queue_cycle = 1; controller->command_queue_enq = 0; dprintf("xhci: status before ring = %#x\n", controller->oregs->op_usbsts); xhci_command(controller, 0, 0, 0, (23 << 10)); char devName[20] = "/dev/xhciN"; snprintf(devName, 19, "/dev/xhci%d", 0); fs_node_t * fnode = calloc(sizeof(fs_node_t), 1); snprintf(fnode->name, 100, "xhci%d", 0); fnode->flags = FS_BLOCKDEVICE; fnode->mask = 0660; /* Only accessible to root user/group */ fnode->read = NULL; fnode->write = xhci_write; fnode->device = controller; vfs_mount(devName, fnode); int event_deq = 0; uint32_t event_cycle_state = 1; while (1) { while ((controller->er_trbs[event_deq].trb_control & 1) != event_cycle_state) { switch_task(0); } uint32_t thing_a = controller->er_trbs[event_deq].trb_thing_a; uint32_t thing_b = controller->er_trbs[event_deq].trb_thing_a; uint32_t status = controller->er_trbs[event_deq].trb_status; uint32_t control = controller->er_trbs[event_deq].trb_control; dprintf("xhci: event %d [%#x %#x %#x %#x]\n", event_deq, thing_a, thing_b, status, control); event_deq++; if (event_deq == 64) { event_deq = 0; event_cycle_state = !event_cycle_state; } /* Write new event dequeue pointer */ uint64_t new_deq_phys = pci_addr_map(controller, er_trbs_phys + sizeof(struct xhci_trb) * event_deq) | (1 << 3); irs0_32[6] = new_deq_phys; irs0_32[7] = new_deq_phys >> 32; } _bail: task_exit(1); __builtin_unreachable(); } static void find_xhci(uint32_t device, uint16_t v, uint16_t d, void * extra) { if (pci_find_type(device) != 0x0C03) return; if (pci_read_field(device, PCI_PROG_IF, 1) != 0x30) return; fs_node_t * stderr = extra; uint16_t command_reg = pci_read_field(device, PCI_COMMAND, 2); command_reg |= (1 << 2); command_reg |= (1 << 1); pci_write_field(device, PCI_COMMAND, 2, command_reg); /* The mmio address is 64 bits and combines BAR0 and BAR1... */ uint64_t addr_low = pci_read_field(device, PCI_BAR0, 4) & 0xFFFFFFF0; uint64_t addr_high = pci_read_field(device, PCI_BAR1, 4) & 0xFFFFFFFF; /* I think this is right? */ uint64_t mmio_addr = (addr_high << 32) | addr_low; if (mmio_addr == 0) { /* Need to map... */ fprintf(stderr, "xhci: Device is unmapped. TODO: Check if this is behind a PCI bridge...\n"); return; #if 0 mmio_addr = mmu_allocate_n_frames(2) << 12; pci_write_field(device, PCI_BAR0, 4, (mmio_addr & 0xFFFFFFF0) | (1 << 2)); pci_write_field(device, PCI_BAR1, 4, (mmio_addr >> 32)); #endif } fprintf(stderr, "xhci: controller found\n"); struct XHCIControllerData * controller = calloc(sizeof(struct XHCIControllerData), 1); controller->device = device; /* Map mmio space... */ uintptr_t xhci_regs = (uintptr_t)mmu_map_mmio_region(mmio_addr, 0x1000 * 4); /* I don't know. */ controller->mmio = mmio_addr; controller->cregs = (struct xhci_cap_regs*)xhci_regs; controller->oregs = (struct xhci_op_regs*)(xhci_regs + (controller->cregs->cap_caplen_version & 0xFF)); controller->pcie_offset = 0; spawn_worker_thread(xhci_thread, "[xhci]", controller); } static int init(int argc, char * argv[]) { fs_node_t * node = FD_ENTRY(1); /* Get the stdout for the process that loaded the module */ pci_scan(find_xhci, -1, node); return 0; } static int fini(void) { return 0; } struct Module metadata = { .name = "xhci", .init = init, .fini = fini, };