freebsd_iflib: Synchronize with FreeBSD trunk.
Up through Sep. 20th (yesterday.) Includes some pretty substantial fixes around MSI interrupts, among other things.
This commit is contained in:
Augustin Cavalier
parent
453027c1c3
commit
6ad99cdddf
|
|
@ -78,7 +78,7 @@ typedef struct if_rxd_info {
|
|||
/* XXX redundant with the new irf_len field */
|
||||
uint16_t iri_len; /* packet length */
|
||||
qidx_t iri_cidx; /* consumer index of cq */
|
||||
struct ifnet *iri_ifp; /* some drivers >1 interface per softc */
|
||||
if_t iri_ifp; /* driver may have >1 iface per softc */
|
||||
|
||||
/* updated by driver */
|
||||
if_rxd_frag_t iri_frags;
|
||||
|
|
@ -131,12 +131,12 @@ typedef struct if_pkt_info {
|
|||
uint8_t ipi_mflags; /* packet mbuf flags */
|
||||
|
||||
uint32_t ipi_tcp_seq; /* tcp seqno */
|
||||
uint32_t ipi_tcp_sum; /* tcp csum */
|
||||
uint32_t __spare0__;
|
||||
} *if_pkt_info_t;
|
||||
|
||||
typedef struct if_irq {
|
||||
struct resource *ii_res;
|
||||
int ii_rid;
|
||||
int __spare0__;
|
||||
void *ii_tag;
|
||||
} *if_irq_t;
|
||||
|
||||
|
|
@ -165,7 +165,7 @@ typedef struct pci_vendor_info {
|
|||
uint32_t pvi_subdevice_id;
|
||||
uint32_t pvi_rev_id;
|
||||
uint32_t pvi_class_mask;
|
||||
caddr_t pvi_name;
|
||||
const char *pvi_name;
|
||||
} pci_vendor_info_t;
|
||||
|
||||
#define PVID(vendor, devid, name) {vendor, devid, 0, 0, 0, 0, name}
|
||||
|
|
@ -193,9 +193,8 @@ typedef struct if_softc_ctx {
|
|||
int isc_vectors;
|
||||
int isc_nrxqsets;
|
||||
int isc_ntxqsets;
|
||||
uint8_t isc_min_tx_latency; /* disable doorbell update batching */
|
||||
uint8_t isc_rx_mvec_enable; /* generate mvecs on rx */
|
||||
uint32_t isc_txrx_budget_bytes_max;
|
||||
uint16_t __spare0__;
|
||||
uint32_t __spare1__;
|
||||
int isc_msix_bar; /* can be model specific - initialize in attach_pre */
|
||||
int isc_tx_nsegments; /* can be model specific - initialize in attach_pre */
|
||||
int isc_ntxd[8];
|
||||
|
|
@ -217,16 +216,23 @@ typedef struct if_softc_ctx {
|
|||
int isc_rss_table_mask;
|
||||
int isc_nrxqsets_max;
|
||||
int isc_ntxqsets_max;
|
||||
uint32_t isc_tx_qdepth;
|
||||
uint32_t __spare2__;
|
||||
|
||||
iflib_intr_mode_t isc_intr;
|
||||
uint16_t isc_max_frame_size; /* set at init time by driver */
|
||||
uint16_t isc_min_frame_size; /* set at init time by driver, only used if
|
||||
IFLIB_NEED_ETHER_PAD is set. */
|
||||
uint32_t isc_pause_frames; /* set by driver for iflib_timer to detect */
|
||||
pci_vendor_info_t isc_vendor_info; /* set by iflib prior to attach_pre */
|
||||
uint32_t __spare3__;
|
||||
uint32_t __spare4__;
|
||||
uint32_t __spare5__;
|
||||
uint32_t __spare6__;
|
||||
uint32_t __spare7__;
|
||||
uint32_t __spare8__;
|
||||
caddr_t __spare9__;
|
||||
int isc_disable_msix;
|
||||
if_txrx_t isc_txrx;
|
||||
struct ifmedia *isc_media;
|
||||
} *if_softc_ctx_t;
|
||||
|
||||
/*
|
||||
|
|
@ -246,7 +252,7 @@ struct if_shared_ctx {
|
|||
int isc_admin_intrcnt; /* # of admin/link interrupts */
|
||||
|
||||
/* fields necessary for probe */
|
||||
pci_vendor_info_t *isc_vendor_info;
|
||||
const pci_vendor_info_t *isc_vendor_info;
|
||||
const char *isc_driver_version;
|
||||
/* optional function to transform the read values to match the table*/
|
||||
void (*isc_parse_devinfo) (uint16_t *device_id, uint16_t *subvendor_id,
|
||||
|
|
@ -262,7 +268,7 @@ struct if_shared_ctx {
|
|||
int isc_nfl __aligned(CACHE_LINE_SIZE);
|
||||
int isc_ntxqs; /* # of tx queues per tx qset - usually 1 */
|
||||
int isc_nrxqs; /* # of rx queues per rx qset - intel 1, chelsio 2, broadcom 3 */
|
||||
int isc_rx_process_limit;
|
||||
int __spare0__;
|
||||
int isc_tx_reclaim_thresh;
|
||||
int isc_flags;
|
||||
const char *isc_name;
|
||||
|
|
@ -286,11 +292,6 @@ typedef enum {
|
|||
IFLIB_INTR_IOV,
|
||||
} iflib_intr_type_t;
|
||||
|
||||
#ifndef ETH_ADDR_LEN
|
||||
#define ETH_ADDR_LEN 6
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
* Interface has a separate command queue for RX
|
||||
*/
|
||||
|
|
@ -360,7 +361,10 @@ typedef enum {
|
|||
* Interface needs admin task to ignore interface up/down status
|
||||
*/
|
||||
#define IFLIB_ADMIN_ALWAYS_RUN 0x10000
|
||||
|
||||
/*
|
||||
* Driver will pass the media
|
||||
*/
|
||||
#define IFLIB_DRIVER_MEDIA 0x20000
|
||||
|
||||
/*
|
||||
* field accessors
|
||||
|
|
@ -392,6 +396,12 @@ int iflib_device_suspend(device_t);
|
|||
int iflib_device_resume(device_t);
|
||||
int iflib_device_shutdown(device_t);
|
||||
|
||||
/*
|
||||
* Use this instead of iflib_device_probe if the driver should report
|
||||
* BUS_PROBE_VENDOR instead of BUS_PROBE_DEFAULT. (For example, an out-of-tree
|
||||
* driver based on iflib).
|
||||
*/
|
||||
int iflib_device_probe_vendor(device_t);
|
||||
|
||||
int iflib_device_iov_init(device_t, uint16_t, const nvlist_t *);
|
||||
void iflib_device_iov_uninit(device_t);
|
||||
|
|
@ -404,8 +414,6 @@ int iflib_device_iov_add_vf(device_t, uint16_t, const nvlist_t *);
|
|||
int iflib_device_register(device_t dev, void *softc, if_shared_ctx_t sctx, if_ctx_t *ctxp);
|
||||
int iflib_device_deregister(if_ctx_t);
|
||||
|
||||
|
||||
|
||||
int iflib_irq_alloc(if_ctx_t, if_irq_t, int, driver_filter_t, void *filter_arg, driver_intr_t, void *arg, const char *name);
|
||||
int iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
|
||||
iflib_intr_type_t type, driver_filter_t *filter,
|
||||
|
|
@ -414,34 +422,28 @@ void iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t t
|
|||
|
||||
void iflib_irq_free(if_ctx_t ctx, if_irq_t irq);
|
||||
|
||||
void iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name);
|
||||
void iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu,
|
||||
const char *name);
|
||||
|
||||
void iflib_config_gtask_init(void *ctx, struct grouptask *gtask,
|
||||
gtask_fn_t *fn, const char *name);
|
||||
|
||||
void iflib_config_gtask_deinit(struct grouptask *gtask);
|
||||
|
||||
|
||||
|
||||
void iflib_tx_intr_deferred(if_ctx_t ctx, int txqid);
|
||||
void iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid);
|
||||
void iflib_admin_intr_deferred(if_ctx_t ctx);
|
||||
void iflib_iov_intr_deferred(if_ctx_t ctx);
|
||||
|
||||
|
||||
void iflib_link_state_change(if_ctx_t ctx, int linkstate, uint64_t baudrate);
|
||||
|
||||
int iflib_dma_alloc(if_ctx_t ctx, int size, iflib_dma_info_t dma, int mapflags);
|
||||
int iflib_dma_alloc_align(if_ctx_t ctx, int size, int align, iflib_dma_info_t dma, int mapflags);
|
||||
void iflib_dma_free(iflib_dma_info_t dma);
|
||||
|
||||
int iflib_dma_alloc_multi(if_ctx_t ctx, int *sizes, iflib_dma_info_t *dmalist, int mapflags, int count);
|
||||
|
||||
void iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count);
|
||||
|
||||
|
||||
struct sx *iflib_ctx_lock_get(if_ctx_t);
|
||||
struct mtx *iflib_qset_lock_get(if_ctx_t, uint16_t);
|
||||
|
||||
void iflib_led_create(if_ctx_t ctx);
|
||||
|
||||
|
|
@ -453,4 +455,5 @@ void iflib_add_int_delay_sysctl(if_ctx_t, const char *, const char *,
|
|||
*/
|
||||
if_pseudo_t iflib_clone_register(if_shared_ctx_t);
|
||||
void iflib_clone_deregister(if_pseudo_t);
|
||||
|
||||
#endif /* __IFLIB_H_ */
|
||||
|
|
|
|||
|
|
@ -28,8 +28,7 @@
|
|||
*/
|
||||
|
||||
#ifndef __NET_IFLIB_PRIVATE_H_
|
||||
#define __NET_IFLIB_PRIVATE_H_
|
||||
|
||||
#define __NET_IFLIB_PRIVATE_H_
|
||||
|
||||
#define IFC_LEGACY 0x001
|
||||
#define IFC_QFLUSH 0x002
|
||||
|
|
@ -40,19 +39,14 @@
|
|||
#define IFC_PREFETCH 0x040
|
||||
#define IFC_DO_RESET 0x080
|
||||
#define IFC_DO_WATCHDOG 0x100
|
||||
#define IFC_CHECK_HUNG 0x200
|
||||
#define IFC_SPARE0 0x200
|
||||
#define IFC_PSEUDO 0x400
|
||||
#define IFC_IN_DETACH 0x800
|
||||
|
||||
#define IFC_NETMAP_TX_IRQ 0x80000000
|
||||
#define IFC_NETMAP_TX_IRQ 0x80000000
|
||||
|
||||
MALLOC_DECLARE(M_IFLIB);
|
||||
|
||||
#define IFLIB_MAX_TX_BYTES (2*1024*1024)
|
||||
#define IFLIB_MIN_TX_BYTES (8*1024)
|
||||
#define IFLIB_DEFAULT_TX_QDEPTH 2048
|
||||
|
||||
|
||||
struct iflib_cloneattach_ctx {
|
||||
struct if_clone *cc_ifc;
|
||||
caddr_t cc_params;
|
||||
|
|
|
|||
|
|
@ -35,15 +35,15 @@
|
|||
#error "no user-serviceable parts inside"
|
||||
#endif
|
||||
|
||||
#if defined(__powerpc__) || defined(__mips__) || defined(__i386__) || defined(__HAIKU__)
|
||||
#define NO_64BIT_ATOMICS
|
||||
#endif
|
||||
|
||||
struct ifmp_ring;
|
||||
typedef u_int (*mp_ring_drain_t)(struct ifmp_ring *, u_int, u_int);
|
||||
typedef u_int (*mp_ring_can_drain_t)(struct ifmp_ring *);
|
||||
typedef void (*mp_ring_serial_t)(struct ifmp_ring *);
|
||||
|
||||
#if defined(__powerpc__) || defined(__mips__) || defined(__i386__) || defined(__HAIKU__)
|
||||
#define MP_RING_NO_64BIT_ATOMICS
|
||||
#endif
|
||||
|
||||
struct ifmp_ring {
|
||||
volatile uint64_t state __aligned(CACHE_LINE_SIZE);
|
||||
|
||||
|
|
@ -58,7 +58,7 @@ struct ifmp_ring {
|
|||
counter_u64_t stalls;
|
||||
counter_u64_t restarts; /* recovered after stalling */
|
||||
counter_u64_t abdications;
|
||||
#ifdef NO_64BIT_ATOMICS
|
||||
#ifdef MP_RING_NO_64BIT_ATOMICS
|
||||
struct mtx lock;
|
||||
#endif
|
||||
void * volatile items[
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -36,12 +36,6 @@ __FBSDID("$FreeBSD$");
|
|||
#include <sys/mutex.h>
|
||||
#include <sys/malloc.h>
|
||||
#include <machine/cpu.h>
|
||||
|
||||
#if defined(__i386__)
|
||||
#define atomic_cmpset_acq_64 atomic_cmpset_64
|
||||
#define atomic_cmpset_rel_64 atomic_cmpset_64
|
||||
#endif
|
||||
|
||||
#include <net/mp_ring.h>
|
||||
|
||||
union ring_state {
|
||||
|
|
@ -97,7 +91,7 @@ state_to_flags(union ring_state s, int abdicate)
|
|||
return (BUSY);
|
||||
}
|
||||
|
||||
#ifdef NO_64BIT_ATOMICS
|
||||
#ifdef MP_RING_NO_64BIT_ATOMICS
|
||||
static void
|
||||
drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget)
|
||||
{
|
||||
|
|
@ -195,11 +189,12 @@ drain_ring_lockless(struct ifmp_ring *r, union ring_state os, uint16_t prev, int
|
|||
n = r->drain(r, cidx, pidx);
|
||||
if (n == 0) {
|
||||
critical_enter();
|
||||
os.state = r->state;
|
||||
do {
|
||||
os.state = ns.state = r->state;
|
||||
ns.state = os.state;
|
||||
ns.cidx = cidx;
|
||||
ns.flags = STALLED;
|
||||
} while (atomic_cmpset_64(&r->state, os.state,
|
||||
} while (atomic_fcmpset_64(&r->state, &os.state,
|
||||
ns.state) == 0);
|
||||
critical_exit();
|
||||
if (prev != STALLED)
|
||||
|
|
@ -222,11 +217,13 @@ drain_ring_lockless(struct ifmp_ring *r, union ring_state os, uint16_t prev, int
|
|||
if (cidx != pidx && pending < 64 && total < budget)
|
||||
continue;
|
||||
critical_enter();
|
||||
os.state = r->state;
|
||||
do {
|
||||
os.state = ns.state = r->state;
|
||||
ns.state = os.state;
|
||||
ns.cidx = cidx;
|
||||
ns.flags = state_to_flags(ns, total >= budget);
|
||||
} while (atomic_cmpset_acq_64(&r->state, os.state, ns.state) == 0);
|
||||
} while (atomic_fcmpset_acq_64(&r->state, &os.state,
|
||||
ns.state) == 0);
|
||||
critical_exit();
|
||||
|
||||
if (ns.flags == ABDICATED)
|
||||
|
|
@ -287,7 +284,7 @@ ifmp_ring_alloc(struct ifmp_ring **pr, int size, void *cookie, mp_ring_drain_t d
|
|||
}
|
||||
|
||||
*pr = r;
|
||||
#ifdef NO_64BIT_ATOMICS
|
||||
#ifdef MP_RING_NO_64BIT_ATOMICS
|
||||
mtx_init(&r->lock, "mp_ring lock", NULL, MTX_DEF);
|
||||
#endif
|
||||
return (0);
|
||||
|
|
@ -321,7 +318,7 @@ ifmp_ring_free(struct ifmp_ring *r)
|
|||
*
|
||||
* Returns an errno.
|
||||
*/
|
||||
#ifdef NO_64BIT_ATOMICS
|
||||
#ifdef MP_RING_NO_64BIT_ATOMICS
|
||||
int
|
||||
ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdicate)
|
||||
{
|
||||
|
|
@ -366,7 +363,6 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdi
|
|||
i = pidx_start;
|
||||
do {
|
||||
r->items[i] = *items++;
|
||||
/*HAIKU*/KASSERT((r->items[i] == NULL) || ((uintptr_t)(r->items[i]) > 1024UL), ("is %p", r->items[i]));
|
||||
if (__predict_false(++i == r->size))
|
||||
i = 0;
|
||||
} while (i != pidx_stop);
|
||||
|
|
@ -380,10 +376,8 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdi
|
|||
if (abdicate) {
|
||||
if (os.flags == IDLE)
|
||||
ns.flags = ABDICATED;
|
||||
}
|
||||
else {
|
||||
} else
|
||||
ns.flags = BUSY;
|
||||
}
|
||||
r->state = ns.state;
|
||||
counter_u64_add(r->enqueues, n);
|
||||
|
||||
|
|
@ -399,7 +393,6 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdi
|
|||
mtx_unlock(&r->lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
#else
|
||||
int
|
||||
ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdicate)
|
||||
|
|
@ -415,8 +408,8 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdi
|
|||
* Reserve room for the new items. Our reservation, if successful, is
|
||||
* from 'pidx_start' to 'pidx_stop'.
|
||||
*/
|
||||
os.state = r->state;
|
||||
for (;;) {
|
||||
os.state = r->state;
|
||||
if (n >= space_available(r, os)) {
|
||||
counter_u64_add(r->drops, n);
|
||||
MPASS(os.flags != IDLE);
|
||||
|
|
@ -427,7 +420,7 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdi
|
|||
ns.state = os.state;
|
||||
ns.pidx_head = increment_idx(r, os.pidx_head, n);
|
||||
critical_enter();
|
||||
if (atomic_cmpset_64(&r->state, os.state, ns.state))
|
||||
if (atomic_fcmpset_64(&r->state, &os.state, ns.state))
|
||||
break;
|
||||
critical_exit();
|
||||
cpu_spinwait();
|
||||
|
|
@ -457,17 +450,16 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdi
|
|||
* Update the ring's pidx_tail. The release style atomic guarantees
|
||||
* that the items are visible to any thread that sees the updated pidx.
|
||||
*/
|
||||
os.state = r->state;
|
||||
do {
|
||||
os.state = ns.state = r->state;
|
||||
ns.state = os.state;
|
||||
ns.pidx_tail = pidx_stop;
|
||||
if (abdicate) {
|
||||
if (os.flags == IDLE)
|
||||
ns.flags = ABDICATED;
|
||||
}
|
||||
else {
|
||||
} else
|
||||
ns.flags = BUSY;
|
||||
}
|
||||
} while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0);
|
||||
} while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0);
|
||||
critical_exit();
|
||||
counter_u64_add(r->enqueues, n);
|
||||
|
||||
|
|
@ -500,7 +492,7 @@ ifmp_ring_check_drainage(struct ifmp_ring *r, int budget)
|
|||
ns.flags = BUSY;
|
||||
|
||||
|
||||
#ifdef NO_64BIT_ATOMICS
|
||||
#ifdef MP_RING_NO_64BIT_ATOMICS
|
||||
mtx_lock(&r->lock);
|
||||
if (r->state != os.state) {
|
||||
mtx_unlock(&r->lock);
|
||||
|
|
|
|||
Loading…
Reference in New Issue