2010-03-17 14:08:17 +03:00
|
|
|
/*
|
|
|
|
* vhost support
|
|
|
|
*
|
|
|
|
* Copyright Red Hat, Inc. 2010
|
|
|
|
*
|
|
|
|
* Authors:
|
|
|
|
* Michael S. Tsirkin <mst@redhat.com>
|
|
|
|
*
|
|
|
|
* This work is licensed under the terms of the GNU GPL, version 2. See
|
|
|
|
* the COPYING file in the top-level directory.
|
2012-01-13 20:44:23 +04:00
|
|
|
*
|
|
|
|
* Contributions after 2012-01-13 are licensed under the terms of the
|
|
|
|
* GNU GPL, version 2 or (at your option) any later version.
|
2010-03-17 14:08:17 +03:00
|
|
|
*/
|
|
|
|
|
2016-01-26 21:17:07 +03:00
|
|
|
#include "qemu/osdep.h"
|
include/qemu/osdep.h: Don't include qapi/error.h
Commit 57cb38b included qapi/error.h into qemu/osdep.h to get the
Error typedef. Since then, we've moved to include qemu/osdep.h
everywhere. Its file comment explains: "To avoid getting into
possible circular include dependencies, this file should not include
any other QEMU headers, with the exceptions of config-host.h,
compiler.h, os-posix.h and os-win32.h, all of which are doing a
similar job to this file and are under similar constraints."
qapi/error.h doesn't do a similar job, and it doesn't adhere to
similar constraints: it includes qapi-types.h. That's in excess of
100KiB of crap most .c files don't actually need.
Add the typedef to qemu/typedefs.h, and include that instead of
qapi/error.h. Include qapi/error.h in .c files that need it and don't
get it now. Include qapi-types.h in qom/object.h for uint16List.
Update scripts/clean-includes accordingly. Update it further to match
reality: replace config.h by config-target.h, add sysemu/os-posix.h,
sysemu/os-win32.h. Update the list of includes in the qemu/osdep.h
comment quoted above similarly.
This reduces the number of objects depending on qapi/error.h from "all
of them" to less than a third. Unfortunately, the number depending on
qapi-types.h shrinks only a little. More work is needed for that one.
Signed-off-by: Markus Armbruster <armbru@redhat.com>
[Fix compilation without the spice devel packages. - Paolo]
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2016-03-14 11:01:28 +03:00
|
|
|
#include "qapi/error.h"
|
2013-02-05 20:06:20 +04:00
|
|
|
#include "hw/virtio/vhost.h"
|
2013-05-13 15:29:47 +04:00
|
|
|
#include "qemu/atomic.h"
|
2012-12-17 21:20:00 +04:00
|
|
|
#include "qemu/range.h"
|
2015-06-17 16:23:39 +03:00
|
|
|
#include "qemu/error-report.h"
|
2015-10-09 18:17:25 +03:00
|
|
|
#include "qemu/memfd.h"
|
2019-02-14 20:35:50 +03:00
|
|
|
#include "standard-headers/linux/vhost_types.h"
|
2013-04-24 12:21:21 +04:00
|
|
|
#include "hw/virtio/virtio-bus.h"
|
2015-06-17 16:23:39 +03:00
|
|
|
#include "hw/virtio/virtio-access.h"
|
2017-04-06 13:00:28 +03:00
|
|
|
#include "migration/blocker.h"
|
2019-08-12 08:23:39 +03:00
|
|
|
#include "migration/qemu-file-types.h"
|
2017-01-11 07:32:12 +03:00
|
|
|
#include "sysemu/dma.h"
|
2020-06-05 18:49:25 +03:00
|
|
|
#include "sysemu/tcg.h"
|
2018-01-19 13:39:24 +03:00
|
|
|
#include "trace.h"
|
2010-03-17 14:08:17 +03:00
|
|
|
|
2016-07-27 00:15:05 +03:00
|
|
|
/* enabled until disconnected backend stabilizes */
|
|
|
|
#define _VHOST_DEBUG 1
|
|
|
|
|
|
|
|
#ifdef _VHOST_DEBUG
|
|
|
|
#define VHOST_OPS_DEBUG(fmt, ...) \
|
|
|
|
do { error_report(fmt ": %s (%d)", ## __VA_ARGS__, \
|
|
|
|
strerror(errno), errno); } while (0)
|
|
|
|
#else
|
|
|
|
#define VHOST_OPS_DEBUG(fmt, ...) \
|
|
|
|
do { } while (0)
|
|
|
|
#endif
|
|
|
|
|
2015-06-04 12:28:46 +03:00
|
|
|
static struct vhost_log *vhost_log;
|
2015-10-09 18:17:25 +03:00
|
|
|
static struct vhost_log *vhost_log_shm;
|
2015-06-04 12:28:46 +03:00
|
|
|
|
2015-10-06 11:37:27 +03:00
|
|
|
static unsigned int used_memslots;
|
|
|
|
static QLIST_HEAD(, vhost_dev) vhost_devices =
|
|
|
|
QLIST_HEAD_INITIALIZER(vhost_devices);
|
|
|
|
|
|
|
|
bool vhost_has_free_slot(void)
|
|
|
|
{
|
|
|
|
unsigned int slots_limit = ~0U;
|
|
|
|
struct vhost_dev *hdev;
|
|
|
|
|
|
|
|
QLIST_FOREACH(hdev, &vhost_devices, entry) {
|
|
|
|
unsigned int r = hdev->vhost_ops->vhost_backend_memslots_limit(hdev);
|
|
|
|
slots_limit = MIN(slots_limit, r);
|
|
|
|
}
|
|
|
|
return slots_limit > used_memslots;
|
|
|
|
}
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
static void vhost_dev_sync_region(struct vhost_dev *dev,
|
2011-12-19 15:18:13 +04:00
|
|
|
MemoryRegionSection *section,
|
2010-03-17 14:08:17 +03:00
|
|
|
uint64_t mfirst, uint64_t mlast,
|
|
|
|
uint64_t rfirst, uint64_t rlast)
|
|
|
|
{
|
2015-06-04 12:28:46 +03:00
|
|
|
vhost_log_chunk_t *log = dev->log->log;
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
uint64_t start = MAX(mfirst, rfirst);
|
|
|
|
uint64_t end = MIN(mlast, rlast);
|
2015-06-04 12:28:46 +03:00
|
|
|
vhost_log_chunk_t *from = log + start / VHOST_LOG_CHUNK;
|
|
|
|
vhost_log_chunk_t *to = log + end / VHOST_LOG_CHUNK + 1;
|
2017-06-22 14:04:16 +03:00
|
|
|
uint64_t addr = QEMU_ALIGN_DOWN(start, VHOST_LOG_CHUNK);
|
2010-03-17 14:08:17 +03:00
|
|
|
|
|
|
|
if (end < start) {
|
|
|
|
return;
|
|
|
|
}
|
2010-08-13 17:54:52 +04:00
|
|
|
assert(end / VHOST_LOG_CHUNK < dev->log_size);
|
2012-04-01 12:39:43 +04:00
|
|
|
assert(start / VHOST_LOG_CHUNK < dev->log_size);
|
2010-08-13 17:54:52 +04:00
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
for (;from < to; ++from) {
|
|
|
|
vhost_log_chunk_t log;
|
|
|
|
/* We first check with non-atomic: much cheaper,
|
|
|
|
* and we expect non-dirty to be the common case. */
|
|
|
|
if (!*from) {
|
2010-11-27 17:05:07 +03:00
|
|
|
addr += VHOST_LOG_CHUNK;
|
2010-03-17 14:08:17 +03:00
|
|
|
continue;
|
|
|
|
}
|
2013-05-13 15:29:47 +04:00
|
|
|
/* Data must be read atomically. We don't really need barrier semantics
|
|
|
|
* but it's easier to use atomic_* than roll our own. */
|
2020-09-23 13:56:46 +03:00
|
|
|
log = qatomic_xchg(from, 0);
|
2014-04-29 18:17:29 +04:00
|
|
|
while (log) {
|
|
|
|
int bit = ctzl(log);
|
2013-02-21 15:16:06 +04:00
|
|
|
hwaddr page_addr;
|
|
|
|
hwaddr section_offset;
|
|
|
|
hwaddr mr_offset;
|
|
|
|
page_addr = addr + bit * VHOST_LOG_PAGE;
|
|
|
|
section_offset = page_addr - section->offset_within_address_space;
|
|
|
|
mr_offset = section_offset + section->offset_within_region;
|
|
|
|
memory_region_set_dirty(section->mr, mr_offset, VHOST_LOG_PAGE);
|
2010-03-17 14:08:17 +03:00
|
|
|
log &= ~(0x1ull << bit);
|
|
|
|
}
|
|
|
|
addr += VHOST_LOG_CHUNK;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-12-18 16:06:05 +04:00
|
|
|
static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
|
2011-12-19 15:18:13 +04:00
|
|
|
MemoryRegionSection *section,
|
2013-02-21 15:16:06 +04:00
|
|
|
hwaddr first,
|
|
|
|
hwaddr last)
|
2010-03-17 14:08:17 +03:00
|
|
|
{
|
|
|
|
int i;
|
2013-02-21 15:16:06 +04:00
|
|
|
hwaddr start_addr;
|
|
|
|
hwaddr end_addr;
|
2011-12-18 16:06:05 +04:00
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
if (!dev->log_enabled || !dev->started) {
|
|
|
|
return 0;
|
|
|
|
}
|
2013-02-21 15:16:06 +04:00
|
|
|
start_addr = section->offset_within_address_space;
|
2013-05-27 12:08:27 +04:00
|
|
|
end_addr = range_get_last(start_addr, int128_get64(section->size));
|
2013-02-21 15:16:06 +04:00
|
|
|
start_addr = MAX(first, start_addr);
|
|
|
|
end_addr = MIN(last, end_addr);
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
for (i = 0; i < dev->mem->nregions; ++i) {
|
|
|
|
struct vhost_memory_region *reg = dev->mem->regions + i;
|
2011-12-19 15:18:13 +04:00
|
|
|
vhost_dev_sync_region(dev, section, start_addr, end_addr,
|
2010-03-17 14:08:17 +03:00
|
|
|
reg->guest_phys_addr,
|
|
|
|
range_get_last(reg->guest_phys_addr,
|
|
|
|
reg->memory_size));
|
|
|
|
}
|
|
|
|
for (i = 0; i < dev->nvqs; ++i) {
|
|
|
|
struct vhost_virtqueue *vq = dev->vqs + i;
|
vhost: fix vhost_log size overflow during migration
When a guest which doesn't support multiqueue is migrated with a multi queues
vhost-user-blk deivce, a crash will occur like:
0 qemu_memfd_alloc (name=<value optimized out>, size=562949953421312, seals=<value optimized out>, fd=0x7f87171fe8b4, errp=0x7f87171fe8a8) at util/memfd.c:153
1 0x00007f883559d7cf in vhost_log_alloc (size=70368744177664, share=true) at hw/virtio/vhost.c:186
2 0x00007f88355a0758 in vhost_log_get (listener=0x7f8838bd7940, enable=1) at qemu-2-12/hw/virtio/vhost.c:211
3 vhost_dev_log_resize (listener=0x7f8838bd7940, enable=1) at hw/virtio/vhost.c:263
4 vhost_migration_log (listener=0x7f8838bd7940, enable=1) at hw/virtio/vhost.c:787
5 0x00007f88355463d6 in memory_global_dirty_log_start () at memory.c:2503
6 0x00007f8835550577 in ram_init_bitmaps (f=0x7f88384ce600, opaque=0x7f8836024098) at migration/ram.c:2173
7 ram_init_all (f=0x7f88384ce600, opaque=0x7f8836024098) at migration/ram.c:2192
8 ram_save_setup (f=0x7f88384ce600, opaque=0x7f8836024098) at migration/ram.c:2219
9 0x00007f88357a419d in qemu_savevm_state_setup (f=0x7f88384ce600) at migration/savevm.c:1002
10 0x00007f883579fc3e in migration_thread (opaque=0x7f8837530400) at migration/migration.c:2382
11 0x00007f8832447893 in start_thread () from /lib64/libpthread.so.0
12 0x00007f8832178bfd in clone () from /lib64/libc.so.6
This is because vhost_get_log_size() returns a overflowed vhost-log size.
In this function, it uses the uninitialized variable vqs->used_phys and
vqs->used_size to get the vhost-log size.
Signed-off-by: Li Hangjing <lihangjing@baidu.com>
Reviewed-by: Xie Yongji <xieyongji@baidu.com>
Reviewed-by: Chai Wen <chaiwen@baidu.com>
Message-Id: <20190603061524.24076-1-lihangjing@baidu.com>
Cc: qemu-stable@nongnu.org
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2019-06-03 09:15:24 +03:00
|
|
|
|
|
|
|
if (!vq->used_phys && !vq->used_size) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2011-12-19 15:18:13 +04:00
|
|
|
vhost_dev_sync_region(dev, section, start_addr, end_addr, vq->used_phys,
|
2010-03-17 14:08:17 +03:00
|
|
|
range_get_last(vq->used_phys, vq->used_size));
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-12-18 16:06:05 +04:00
|
|
|
static void vhost_log_sync(MemoryListener *listener,
|
|
|
|
MemoryRegionSection *section)
|
|
|
|
{
|
|
|
|
struct vhost_dev *dev = container_of(listener, struct vhost_dev,
|
|
|
|
memory_listener);
|
2013-02-21 15:16:06 +04:00
|
|
|
vhost_sync_dirty_bitmap(dev, section, 0x0, ~0x0ULL);
|
|
|
|
}
|
2011-12-18 16:06:05 +04:00
|
|
|
|
2013-02-21 15:16:06 +04:00
|
|
|
static void vhost_log_sync_range(struct vhost_dev *dev,
|
|
|
|
hwaddr first, hwaddr last)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
/* FIXME: this is N^2 in number of sections */
|
|
|
|
for (i = 0; i < dev->n_mem_sections; ++i) {
|
|
|
|
MemoryRegionSection *section = &dev->mem_sections[i];
|
|
|
|
vhost_sync_dirty_bitmap(dev, section, first, last);
|
|
|
|
}
|
2011-12-18 16:06:05 +04:00
|
|
|
}
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
static uint64_t vhost_get_log_size(struct vhost_dev *dev)
|
|
|
|
{
|
|
|
|
uint64_t log_size = 0;
|
|
|
|
int i;
|
|
|
|
for (i = 0; i < dev->mem->nregions; ++i) {
|
|
|
|
struct vhost_memory_region *reg = dev->mem->regions + i;
|
|
|
|
uint64_t last = range_get_last(reg->guest_phys_addr,
|
|
|
|
reg->memory_size);
|
|
|
|
log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
|
|
|
|
}
|
|
|
|
return log_size;
|
|
|
|
}
|
2015-10-09 18:17:25 +03:00
|
|
|
|
2021-08-09 16:40:15 +03:00
|
|
|
static int vhost_set_backend_type(struct vhost_dev *dev,
|
|
|
|
VhostBackendType backend_type)
|
|
|
|
{
|
|
|
|
int r = 0;
|
|
|
|
|
|
|
|
switch (backend_type) {
|
|
|
|
#ifdef CONFIG_VHOST_KERNEL
|
|
|
|
case VHOST_BACKEND_TYPE_KERNEL:
|
|
|
|
dev->vhost_ops = &kernel_ops;
|
|
|
|
break;
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_VHOST_USER
|
|
|
|
case VHOST_BACKEND_TYPE_USER:
|
|
|
|
dev->vhost_ops = &user_ops;
|
|
|
|
break;
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_VHOST_VDPA
|
|
|
|
case VHOST_BACKEND_TYPE_VDPA:
|
|
|
|
dev->vhost_ops = &vdpa_ops;
|
|
|
|
break;
|
|
|
|
#endif
|
|
|
|
default:
|
|
|
|
error_report("Unknown vhost backend type");
|
|
|
|
r = -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2015-10-09 18:17:25 +03:00
|
|
|
static struct vhost_log *vhost_log_alloc(uint64_t size, bool share)
|
2015-06-04 12:28:46 +03:00
|
|
|
{
|
2018-02-01 16:27:51 +03:00
|
|
|
Error *err = NULL;
|
2015-10-09 18:17:25 +03:00
|
|
|
struct vhost_log *log;
|
|
|
|
uint64_t logsize = size * sizeof(*(log->log));
|
|
|
|
int fd = -1;
|
|
|
|
|
|
|
|
log = g_new0(struct vhost_log, 1);
|
|
|
|
if (share) {
|
|
|
|
log->log = qemu_memfd_alloc("vhost-log", logsize,
|
|
|
|
F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
|
2018-02-01 16:27:51 +03:00
|
|
|
&fd, &err);
|
|
|
|
if (err) {
|
|
|
|
error_report_err(err);
|
|
|
|
g_free(log);
|
|
|
|
return NULL;
|
|
|
|
}
|
2015-10-09 18:17:25 +03:00
|
|
|
memset(log->log, 0, logsize);
|
|
|
|
} else {
|
|
|
|
log->log = g_malloc0(logsize);
|
|
|
|
}
|
2015-06-04 12:28:46 +03:00
|
|
|
|
|
|
|
log->size = size;
|
|
|
|
log->refcnt = 1;
|
2015-10-09 18:17:25 +03:00
|
|
|
log->fd = fd;
|
2015-06-04 12:28:46 +03:00
|
|
|
|
|
|
|
return log;
|
|
|
|
}
|
|
|
|
|
2015-10-09 18:17:25 +03:00
|
|
|
static struct vhost_log *vhost_log_get(uint64_t size, bool share)
|
2015-06-04 12:28:46 +03:00
|
|
|
{
|
2015-10-09 18:17:25 +03:00
|
|
|
struct vhost_log *log = share ? vhost_log_shm : vhost_log;
|
|
|
|
|
|
|
|
if (!log || log->size != size) {
|
|
|
|
log = vhost_log_alloc(size, share);
|
|
|
|
if (share) {
|
|
|
|
vhost_log_shm = log;
|
|
|
|
} else {
|
|
|
|
vhost_log = log;
|
|
|
|
}
|
2015-06-04 12:28:46 +03:00
|
|
|
} else {
|
2015-10-09 18:17:25 +03:00
|
|
|
++log->refcnt;
|
2015-06-04 12:28:46 +03:00
|
|
|
}
|
|
|
|
|
2015-10-09 18:17:25 +03:00
|
|
|
return log;
|
2015-06-04 12:28:46 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_log_put(struct vhost_dev *dev, bool sync)
|
|
|
|
{
|
|
|
|
struct vhost_log *log = dev->log;
|
|
|
|
|
|
|
|
if (!log) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
--log->refcnt;
|
|
|
|
if (log->refcnt == 0) {
|
|
|
|
/* Sync only the range covered by the old log */
|
|
|
|
if (dev->log_size && sync) {
|
|
|
|
vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
|
|
|
|
}
|
2015-10-09 18:17:25 +03:00
|
|
|
|
2015-06-04 12:28:46 +03:00
|
|
|
if (vhost_log == log) {
|
2015-10-09 18:17:25 +03:00
|
|
|
g_free(log->log);
|
2015-06-04 12:28:46 +03:00
|
|
|
vhost_log = NULL;
|
2015-10-09 18:17:25 +03:00
|
|
|
} else if (vhost_log_shm == log) {
|
|
|
|
qemu_memfd_free(log->log, log->size * sizeof(*(log->log)),
|
|
|
|
log->fd);
|
|
|
|
vhost_log_shm = NULL;
|
2015-06-04 12:28:46 +03:00
|
|
|
}
|
2015-10-09 18:17:25 +03:00
|
|
|
|
2015-06-04 12:28:46 +03:00
|
|
|
g_free(log);
|
|
|
|
}
|
2017-09-20 21:53:06 +03:00
|
|
|
|
|
|
|
dev->log = NULL;
|
|
|
|
dev->log_size = 0;
|
2015-06-04 12:28:46 +03:00
|
|
|
}
|
2010-03-17 14:08:17 +03:00
|
|
|
|
2015-10-09 18:17:25 +03:00
|
|
|
static bool vhost_dev_log_is_shared(struct vhost_dev *dev)
|
|
|
|
{
|
|
|
|
return dev->vhost_ops->vhost_requires_shm_log &&
|
|
|
|
dev->vhost_ops->vhost_requires_shm_log(dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size)
|
2010-03-17 14:08:17 +03:00
|
|
|
{
|
2015-10-09 18:17:25 +03:00
|
|
|
struct vhost_log *log = vhost_log_get(size, vhost_dev_log_is_shared(dev));
|
2015-06-04 12:28:46 +03:00
|
|
|
uint64_t log_base = (uintptr_t)log->log;
|
2013-02-21 15:16:06 +04:00
|
|
|
int r;
|
2013-01-22 14:07:56 +04:00
|
|
|
|
2015-10-09 18:17:22 +03:00
|
|
|
/* inform backend of log switching, this must be done before
|
|
|
|
releasing the current log, to ensure no logging is lost */
|
2015-10-09 18:17:26 +03:00
|
|
|
r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log);
|
2016-07-27 00:15:05 +03:00
|
|
|
if (r < 0) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_set_log_base failed");
|
|
|
|
}
|
|
|
|
|
2015-06-04 12:28:46 +03:00
|
|
|
vhost_log_put(dev, true);
|
2010-03-17 14:08:17 +03:00
|
|
|
dev->log = log;
|
|
|
|
dev->log_size = size;
|
|
|
|
}
|
|
|
|
|
2017-01-11 07:32:12 +03:00
|
|
|
static int vhost_dev_has_iommu(struct vhost_dev *dev)
|
|
|
|
{
|
|
|
|
VirtIODevice *vdev = dev->vdev;
|
|
|
|
|
2020-03-02 07:24:54 +03:00
|
|
|
/*
|
|
|
|
* For vhost, VIRTIO_F_IOMMU_PLATFORM means the backend support
|
|
|
|
* incremental memory mapping API via IOTLB API. For platform that
|
|
|
|
* does not have IOMMU, there's no need to enable this feature
|
|
|
|
* which may cause unnecessary IOTLB miss/update trnasactions.
|
|
|
|
*/
|
2021-08-04 06:48:03 +03:00
|
|
|
return virtio_bus_device_iommu_enabled(vdev) &&
|
2020-03-02 07:24:54 +03:00
|
|
|
virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM);
|
2017-01-11 07:32:12 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static void *vhost_memory_map(struct vhost_dev *dev, hwaddr addr,
|
2020-02-19 21:18:45 +03:00
|
|
|
hwaddr *plen, bool is_write)
|
2017-01-11 07:32:12 +03:00
|
|
|
{
|
|
|
|
if (!vhost_dev_has_iommu(dev)) {
|
|
|
|
return cpu_physical_memory_map(addr, plen, is_write);
|
|
|
|
} else {
|
|
|
|
return (void *)(uintptr_t)addr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_memory_unmap(struct vhost_dev *dev, void *buffer,
|
|
|
|
hwaddr len, int is_write,
|
|
|
|
hwaddr access_len)
|
|
|
|
{
|
|
|
|
if (!vhost_dev_has_iommu(dev)) {
|
|
|
|
cpu_physical_memory_unmap(buffer, len, is_write, access_len);
|
|
|
|
}
|
|
|
|
}
|
2016-11-04 11:39:15 +03:00
|
|
|
|
2018-01-19 13:39:19 +03:00
|
|
|
static int vhost_verify_ring_part_mapping(void *ring_hva,
|
|
|
|
uint64_t ring_gpa,
|
|
|
|
uint64_t ring_size,
|
|
|
|
void *reg_hva,
|
|
|
|
uint64_t reg_gpa,
|
|
|
|
uint64_t reg_size)
|
2016-11-04 11:39:15 +03:00
|
|
|
{
|
2018-01-19 13:39:19 +03:00
|
|
|
uint64_t hva_ring_offset;
|
|
|
|
uint64_t ring_last = range_get_last(ring_gpa, ring_size);
|
|
|
|
uint64_t reg_last = range_get_last(reg_gpa, reg_size);
|
2016-11-04 11:39:15 +03:00
|
|
|
|
2018-01-19 13:39:19 +03:00
|
|
|
if (ring_last < reg_gpa || ring_gpa > reg_last) {
|
2016-11-04 11:39:15 +03:00
|
|
|
return 0;
|
|
|
|
}
|
2018-01-19 13:39:19 +03:00
|
|
|
/* check that whole ring's is mapped */
|
|
|
|
if (ring_last > reg_last) {
|
|
|
|
return -ENOMEM;
|
2016-11-04 11:39:15 +03:00
|
|
|
}
|
2018-01-19 13:39:19 +03:00
|
|
|
/* check that ring's MemoryRegion wasn't replaced */
|
|
|
|
hva_ring_offset = ring_gpa - reg_gpa;
|
|
|
|
if (ring_hva != reg_hva + hva_ring_offset) {
|
|
|
|
return -EBUSY;
|
2016-11-04 11:39:15 +03:00
|
|
|
}
|
2018-01-19 13:39:19 +03:00
|
|
|
|
|
|
|
return 0;
|
2016-11-04 11:39:15 +03:00
|
|
|
}
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
static int vhost_verify_ring_mappings(struct vhost_dev *dev,
|
2018-01-19 13:39:19 +03:00
|
|
|
void *reg_hva,
|
|
|
|
uint64_t reg_gpa,
|
|
|
|
uint64_t reg_size)
|
2010-03-17 14:08:17 +03:00
|
|
|
{
|
2016-11-04 11:39:15 +03:00
|
|
|
int i, j;
|
2014-06-18 19:55:22 +04:00
|
|
|
int r = 0;
|
2016-11-04 11:39:15 +03:00
|
|
|
const char *part_name[] = {
|
|
|
|
"descriptor table",
|
|
|
|
"available ring",
|
|
|
|
"used ring"
|
|
|
|
};
|
2014-06-18 19:55:22 +04:00
|
|
|
|
2018-04-13 06:01:49 +03:00
|
|
|
if (vhost_dev_has_iommu(dev)) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-11-04 11:39:15 +03:00
|
|
|
for (i = 0; i < dev->nvqs; ++i) {
|
2010-03-17 14:08:17 +03:00
|
|
|
struct vhost_virtqueue *vq = dev->vqs + i;
|
|
|
|
|
2018-02-28 12:35:28 +03:00
|
|
|
if (vq->desc_phys == 0) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2016-11-04 11:39:15 +03:00
|
|
|
j = 0;
|
2018-01-19 13:39:19 +03:00
|
|
|
r = vhost_verify_ring_part_mapping(
|
|
|
|
vq->desc, vq->desc_phys, vq->desc_size,
|
|
|
|
reg_hva, reg_gpa, reg_size);
|
2017-12-01 00:39:59 +03:00
|
|
|
if (r) {
|
2016-11-04 11:39:15 +03:00
|
|
|
break;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2016-11-04 11:39:15 +03:00
|
|
|
|
|
|
|
j++;
|
2018-01-19 13:39:19 +03:00
|
|
|
r = vhost_verify_ring_part_mapping(
|
2018-02-28 12:35:29 +03:00
|
|
|
vq->avail, vq->avail_phys, vq->avail_size,
|
2018-01-19 13:39:19 +03:00
|
|
|
reg_hva, reg_gpa, reg_size);
|
2017-12-01 00:39:59 +03:00
|
|
|
if (r) {
|
2016-11-04 11:39:15 +03:00
|
|
|
break;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2016-11-04 11:39:15 +03:00
|
|
|
|
|
|
|
j++;
|
2018-01-19 13:39:19 +03:00
|
|
|
r = vhost_verify_ring_part_mapping(
|
2018-02-28 12:35:29 +03:00
|
|
|
vq->used, vq->used_phys, vq->used_size,
|
2018-01-19 13:39:19 +03:00
|
|
|
reg_hva, reg_gpa, reg_size);
|
2017-12-01 00:39:59 +03:00
|
|
|
if (r) {
|
2016-11-04 11:39:15 +03:00
|
|
|
break;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2016-11-04 11:39:15 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
if (r == -ENOMEM) {
|
|
|
|
error_report("Unable to map %s for ring %d", part_name[j], i);
|
|
|
|
} else if (r == -EBUSY) {
|
|
|
|
error_report("%s relocated for ring %d", part_name[j], i);
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2014-06-18 19:55:22 +04:00
|
|
|
return r;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
|
|
|
|
2020-06-05 18:49:25 +03:00
|
|
|
/*
|
|
|
|
* vhost_section: identify sections needed for vhost access
|
|
|
|
*
|
|
|
|
* We only care about RAM sections here (where virtqueue and guest
|
|
|
|
* internals accessed by virtio might live). If we find one we still
|
|
|
|
* allow the backend to potentially filter it out of our list.
|
|
|
|
*/
|
2018-05-24 13:33:31 +03:00
|
|
|
static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section)
|
2013-04-03 13:15:11 +04:00
|
|
|
{
|
2020-06-05 18:49:25 +03:00
|
|
|
MemoryRegion *mr = section->mr;
|
|
|
|
|
|
|
|
if (memory_region_is_ram(mr) && !memory_region_is_rom(mr)) {
|
|
|
|
uint8_t dirty_mask = memory_region_get_dirty_log_mask(mr);
|
|
|
|
uint8_t handled_dirty;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Kernel based vhost doesn't handle any block which is doing
|
|
|
|
* dirty-tracking other than migration for which it has
|
|
|
|
* specific logging support. However for TCG the kernel never
|
|
|
|
* gets involved anyway so we can also ignore it's
|
|
|
|
* self-modiying code detection flags. However a vhost-user
|
|
|
|
* client could still confuse a TCG guest if it re-writes
|
|
|
|
* executable memory that has already been translated.
|
|
|
|
*/
|
|
|
|
handled_dirty = (1 << DIRTY_MEMORY_MIGRATION) |
|
|
|
|
(1 << DIRTY_MEMORY_CODE);
|
|
|
|
|
|
|
|
if (dirty_mask & ~handled_dirty) {
|
|
|
|
trace_vhost_reject_section(mr->name, 1);
|
|
|
|
return false;
|
|
|
|
}
|
2018-01-19 13:39:24 +03:00
|
|
|
|
2020-06-05 18:49:25 +03:00
|
|
|
if (dev->vhost_ops->vhost_backend_mem_section_filter &&
|
|
|
|
!dev->vhost_ops->vhost_backend_mem_section_filter(dev, section)) {
|
|
|
|
trace_vhost_reject_section(mr->name, 2);
|
|
|
|
return false;
|
|
|
|
}
|
2018-05-24 13:33:31 +03:00
|
|
|
|
2020-06-05 18:49:25 +03:00
|
|
|
trace_vhost_section(mr->name);
|
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
trace_vhost_reject_section(mr->name, 3);
|
|
|
|
return false;
|
|
|
|
}
|
2013-04-03 13:15:11 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_begin(MemoryListener *listener)
|
|
|
|
{
|
|
|
|
struct vhost_dev *dev = container_of(listener, struct vhost_dev,
|
|
|
|
memory_listener);
|
2018-01-19 13:39:18 +03:00
|
|
|
dev->tmp_sections = NULL;
|
|
|
|
dev->n_tmp_sections = 0;
|
2013-04-03 13:15:11 +04:00
|
|
|
}
|
2010-03-17 14:08:17 +03:00
|
|
|
|
2013-04-03 13:15:11 +04:00
|
|
|
static void vhost_commit(MemoryListener *listener)
|
|
|
|
{
|
|
|
|
struct vhost_dev *dev = container_of(listener, struct vhost_dev,
|
|
|
|
memory_listener);
|
2018-01-19 13:39:18 +03:00
|
|
|
MemoryRegionSection *old_sections;
|
|
|
|
int n_old_sections;
|
2013-04-03 13:15:11 +04:00
|
|
|
uint64_t log_size;
|
2018-01-19 13:39:21 +03:00
|
|
|
size_t regions_size;
|
2013-04-03 13:15:11 +04:00
|
|
|
int r;
|
2018-01-19 13:39:19 +03:00
|
|
|
int i;
|
2018-01-19 13:39:21 +03:00
|
|
|
bool changed = false;
|
2013-04-03 13:15:11 +04:00
|
|
|
|
2018-01-19 13:39:21 +03:00
|
|
|
/* Note we can be called before the device is started, but then
|
|
|
|
* starting the device calls set_mem_table, so we need to have
|
|
|
|
* built the data structures.
|
|
|
|
*/
|
2018-01-19 13:39:18 +03:00
|
|
|
old_sections = dev->mem_sections;
|
|
|
|
n_old_sections = dev->n_mem_sections;
|
|
|
|
dev->mem_sections = dev->tmp_sections;
|
|
|
|
dev->n_mem_sections = dev->n_tmp_sections;
|
|
|
|
|
2018-01-19 13:39:21 +03:00
|
|
|
if (dev->n_mem_sections != n_old_sections) {
|
|
|
|
changed = true;
|
|
|
|
} else {
|
|
|
|
/* Same size, lets check the contents */
|
2019-08-14 20:55:35 +03:00
|
|
|
for (int i = 0; i < n_old_sections; i++) {
|
|
|
|
if (!MemoryRegionSection_eq(&old_sections[i],
|
|
|
|
&dev->mem_sections[i])) {
|
|
|
|
changed = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2013-04-03 13:15:11 +04:00
|
|
|
}
|
2018-01-19 13:39:21 +03:00
|
|
|
|
|
|
|
trace_vhost_commit(dev->started, changed);
|
|
|
|
if (!changed) {
|
2018-01-19 13:39:18 +03:00
|
|
|
goto out;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2018-01-19 13:39:21 +03:00
|
|
|
|
|
|
|
/* Rebuild the regions list from the new sections list */
|
|
|
|
regions_size = offsetof(struct vhost_memory, regions) +
|
|
|
|
dev->n_mem_sections * sizeof dev->mem->regions[0];
|
|
|
|
dev->mem = g_realloc(dev->mem, regions_size);
|
|
|
|
dev->mem->nregions = dev->n_mem_sections;
|
|
|
|
used_memslots = dev->mem->nregions;
|
|
|
|
for (i = 0; i < dev->n_mem_sections; i++) {
|
|
|
|
struct vhost_memory_region *cur_vmr = dev->mem->regions + i;
|
|
|
|
struct MemoryRegionSection *mrs = dev->mem_sections + i;
|
|
|
|
|
|
|
|
cur_vmr->guest_phys_addr = mrs->offset_within_address_space;
|
|
|
|
cur_vmr->memory_size = int128_get64(mrs->size);
|
|
|
|
cur_vmr->userspace_addr =
|
|
|
|
(uintptr_t)memory_region_get_ram_ptr(mrs->mr) +
|
|
|
|
mrs->offset_within_region;
|
|
|
|
cur_vmr->flags_padding = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!dev->started) {
|
2018-01-19 13:39:18 +03:00
|
|
|
goto out;
|
2013-04-03 13:15:11 +04:00
|
|
|
}
|
2010-03-17 14:08:17 +03:00
|
|
|
|
2018-01-19 13:39:19 +03:00
|
|
|
for (i = 0; i < dev->mem->nregions; i++) {
|
|
|
|
if (vhost_verify_ring_mappings(dev,
|
|
|
|
(void *)(uintptr_t)dev->mem->regions[i].userspace_addr,
|
|
|
|
dev->mem->regions[i].guest_phys_addr,
|
|
|
|
dev->mem->regions[i].memory_size)) {
|
|
|
|
error_report("Verify ring failure on region %d", i);
|
|
|
|
abort();
|
|
|
|
}
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!dev->log_enabled) {
|
2015-10-09 18:17:28 +03:00
|
|
|
r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
|
2016-07-27 00:15:05 +03:00
|
|
|
if (r < 0) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_set_mem_table failed");
|
|
|
|
}
|
2018-01-19 13:39:18 +03:00
|
|
|
goto out;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
|
|
|
log_size = vhost_get_log_size(dev);
|
|
|
|
/* We allocate an extra 4K bytes to log,
|
|
|
|
* to reduce the * number of reallocations. */
|
|
|
|
#define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log)
|
|
|
|
/* To log more, must increase log size before table update. */
|
|
|
|
if (dev->log_size < log_size) {
|
|
|
|
vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
|
|
|
|
}
|
2015-10-09 18:17:28 +03:00
|
|
|
r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
|
2016-07-27 00:15:05 +03:00
|
|
|
if (r < 0) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_set_mem_table failed");
|
|
|
|
}
|
2010-03-17 14:08:17 +03:00
|
|
|
/* To log less, can only decrease log size after table update. */
|
|
|
|
if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
|
|
|
|
vhost_dev_log_resize(dev, log_size);
|
|
|
|
}
|
2018-01-19 13:39:18 +03:00
|
|
|
|
|
|
|
out:
|
|
|
|
/* Deref the old list of sections, this must happen _after_ the
|
|
|
|
* vhost_set_mem_table to ensure the client isn't still using the
|
|
|
|
* section we're about to unref.
|
|
|
|
*/
|
|
|
|
while (n_old_sections--) {
|
|
|
|
memory_region_unref(old_sections[n_old_sections].mr);
|
|
|
|
}
|
|
|
|
g_free(old_sections);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-01-19 13:39:20 +03:00
|
|
|
/* Adds the section data to the tmp_section structure.
|
|
|
|
* It relies on the listener calling us in memory address order
|
|
|
|
* and for each region (via the _add and _nop methods) to
|
|
|
|
* join neighbours.
|
|
|
|
*/
|
|
|
|
static void vhost_region_add_section(struct vhost_dev *dev,
|
|
|
|
MemoryRegionSection *section)
|
2018-01-19 13:39:18 +03:00
|
|
|
{
|
2018-01-19 13:39:20 +03:00
|
|
|
bool need_add = true;
|
|
|
|
uint64_t mrs_size = int128_get64(section->size);
|
|
|
|
uint64_t mrs_gpa = section->offset_within_address_space;
|
|
|
|
uintptr_t mrs_host = (uintptr_t)memory_region_get_ram_ptr(section->mr) +
|
|
|
|
section->offset_within_region;
|
2018-03-12 20:21:21 +03:00
|
|
|
RAMBlock *mrs_rb = section->mr->ram_block;
|
2018-01-19 13:39:20 +03:00
|
|
|
|
|
|
|
trace_vhost_region_add_section(section->mr->name, mrs_gpa, mrs_size,
|
|
|
|
mrs_host);
|
|
|
|
|
2020-01-22 11:06:47 +03:00
|
|
|
if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER) {
|
2020-01-16 23:24:14 +03:00
|
|
|
/* Round the section to it's page size */
|
|
|
|
/* First align the start down to a page boundary */
|
|
|
|
size_t mrs_page = qemu_ram_pagesize(mrs_rb);
|
|
|
|
uint64_t alignage = mrs_host & (mrs_page - 1);
|
|
|
|
if (alignage) {
|
|
|
|
mrs_host -= alignage;
|
|
|
|
mrs_size += alignage;
|
|
|
|
mrs_gpa -= alignage;
|
|
|
|
}
|
|
|
|
/* Now align the size up to a page boundary */
|
|
|
|
alignage = mrs_size & (mrs_page - 1);
|
|
|
|
if (alignage) {
|
|
|
|
mrs_size += mrs_page - alignage;
|
|
|
|
}
|
2020-01-22 11:06:47 +03:00
|
|
|
trace_vhost_region_add_section_aligned(section->mr->name, mrs_gpa,
|
|
|
|
mrs_size, mrs_host);
|
2020-01-16 23:24:14 +03:00
|
|
|
}
|
2018-03-12 20:21:21 +03:00
|
|
|
|
2018-01-19 13:39:20 +03:00
|
|
|
if (dev->n_tmp_sections) {
|
|
|
|
/* Since we already have at least one section, lets see if
|
|
|
|
* this extends it; since we're scanning in order, we only
|
|
|
|
* have to look at the last one, and the FlatView that calls
|
|
|
|
* us shouldn't have overlaps.
|
|
|
|
*/
|
|
|
|
MemoryRegionSection *prev_sec = dev->tmp_sections +
|
|
|
|
(dev->n_tmp_sections - 1);
|
|
|
|
uint64_t prev_gpa_start = prev_sec->offset_within_address_space;
|
|
|
|
uint64_t prev_size = int128_get64(prev_sec->size);
|
|
|
|
uint64_t prev_gpa_end = range_get_last(prev_gpa_start, prev_size);
|
|
|
|
uint64_t prev_host_start =
|
|
|
|
(uintptr_t)memory_region_get_ram_ptr(prev_sec->mr) +
|
|
|
|
prev_sec->offset_within_region;
|
|
|
|
uint64_t prev_host_end = range_get_last(prev_host_start, prev_size);
|
|
|
|
|
2018-03-12 20:21:21 +03:00
|
|
|
if (mrs_gpa <= (prev_gpa_end + 1)) {
|
|
|
|
/* OK, looks like overlapping/intersecting - it's possible that
|
|
|
|
* the rounding to page sizes has made them overlap, but they should
|
|
|
|
* match up in the same RAMBlock if they do.
|
|
|
|
*/
|
|
|
|
if (mrs_gpa < prev_gpa_start) {
|
2020-01-16 23:24:13 +03:00
|
|
|
error_report("%s:Section '%s' rounded to %"PRIx64
|
|
|
|
" prior to previous '%s' %"PRIx64,
|
|
|
|
__func__, section->mr->name, mrs_gpa,
|
|
|
|
prev_sec->mr->name, prev_gpa_start);
|
2018-03-12 20:21:21 +03:00
|
|
|
/* A way to cleanly fail here would be better */
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
/* Offset from the start of the previous GPA to this GPA */
|
|
|
|
size_t offset = mrs_gpa - prev_gpa_start;
|
|
|
|
|
|
|
|
if (prev_host_start + offset == mrs_host &&
|
|
|
|
section->mr == prev_sec->mr &&
|
|
|
|
(!dev->vhost_ops->vhost_backend_can_merge ||
|
|
|
|
dev->vhost_ops->vhost_backend_can_merge(dev,
|
2018-01-19 13:39:20 +03:00
|
|
|
mrs_host, mrs_size,
|
|
|
|
prev_host_start, prev_size))) {
|
2018-03-12 20:21:21 +03:00
|
|
|
uint64_t max_end = MAX(prev_host_end, mrs_host + mrs_size);
|
|
|
|
need_add = false;
|
|
|
|
prev_sec->offset_within_address_space =
|
|
|
|
MIN(prev_gpa_start, mrs_gpa);
|
|
|
|
prev_sec->offset_within_region =
|
|
|
|
MIN(prev_host_start, mrs_host) -
|
|
|
|
(uintptr_t)memory_region_get_ram_ptr(prev_sec->mr);
|
|
|
|
prev_sec->size = int128_make64(max_end - MIN(prev_host_start,
|
|
|
|
mrs_host));
|
|
|
|
trace_vhost_region_add_section_merge(section->mr->name,
|
|
|
|
int128_get64(prev_sec->size),
|
|
|
|
prev_sec->offset_within_address_space,
|
|
|
|
prev_sec->offset_within_region);
|
|
|
|
} else {
|
2018-03-23 18:39:39 +03:00
|
|
|
/* adjoining regions are fine, but overlapping ones with
|
|
|
|
* different blocks/offsets shouldn't happen
|
|
|
|
*/
|
|
|
|
if (mrs_gpa != prev_gpa_end + 1) {
|
|
|
|
error_report("%s: Overlapping but not coherent sections "
|
|
|
|
"at %"PRIx64,
|
|
|
|
__func__, mrs_gpa);
|
|
|
|
return;
|
|
|
|
}
|
2018-03-12 20:21:21 +03:00
|
|
|
}
|
2018-01-19 13:39:20 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (need_add) {
|
|
|
|
++dev->n_tmp_sections;
|
|
|
|
dev->tmp_sections = g_renew(MemoryRegionSection, dev->tmp_sections,
|
|
|
|
dev->n_tmp_sections);
|
|
|
|
dev->tmp_sections[dev->n_tmp_sections - 1] = *section;
|
|
|
|
/* The flatview isn't stable and we don't use it, making it NULL
|
|
|
|
* means we can memcmp the list.
|
|
|
|
*/
|
|
|
|
dev->tmp_sections[dev->n_tmp_sections - 1].fv = NULL;
|
|
|
|
memory_region_ref(section->mr);
|
|
|
|
}
|
2012-02-08 23:36:02 +04:00
|
|
|
}
|
|
|
|
|
2018-01-19 13:39:23 +03:00
|
|
|
/* Used for both add and nop callbacks */
|
|
|
|
static void vhost_region_addnop(MemoryListener *listener,
|
|
|
|
MemoryRegionSection *section)
|
2011-12-18 16:06:05 +04:00
|
|
|
{
|
2011-12-19 15:18:13 +04:00
|
|
|
struct vhost_dev *dev = container_of(listener, struct vhost_dev,
|
|
|
|
memory_listener);
|
|
|
|
|
2018-05-24 13:33:31 +03:00
|
|
|
if (!vhost_section(dev, section)) {
|
2012-01-09 16:01:39 +04:00
|
|
|
return;
|
|
|
|
}
|
2018-01-19 13:39:20 +03:00
|
|
|
vhost_region_add_section(dev, section);
|
2011-12-18 16:06:05 +04:00
|
|
|
}
|
|
|
|
|
2017-03-29 07:10:04 +03:00
|
|
|
static void vhost_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
|
|
|
|
{
|
|
|
|
struct vhost_iommu *iommu = container_of(n, struct vhost_iommu, n);
|
|
|
|
struct vhost_dev *hdev = iommu->hdev;
|
|
|
|
hwaddr iova = iotlb->iova + iommu->iommu_offset;
|
|
|
|
|
2017-06-02 13:18:28 +03:00
|
|
|
if (vhost_backend_invalidate_device_iotlb(hdev, iova,
|
|
|
|
iotlb->addr_mask + 1)) {
|
2017-03-29 07:10:04 +03:00
|
|
|
error_report("Fail to invalidate device iotlb");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_iommu_region_add(MemoryListener *listener,
|
|
|
|
MemoryRegionSection *section)
|
|
|
|
{
|
|
|
|
struct vhost_dev *dev = container_of(listener, struct vhost_dev,
|
|
|
|
iommu_listener);
|
|
|
|
struct vhost_iommu *iommu;
|
memory: add section range info for IOMMU notifier
In this patch, IOMMUNotifier.{start|end} are introduced to store section
information for a specific notifier. When notification occurs, we not
only check the notification type (MAP|UNMAP), but also check whether the
notified iova range overlaps with the range of specific IOMMU notifier,
and skip those notifiers if not in the listened range.
When removing an region, we need to make sure we removed the correct
VFIOGuestIOMMU by checking the IOMMUNotifier.start address as well.
This patch is solving the problem that vfio-pci devices receive
duplicated UNMAP notification on x86 platform when vIOMMU is there. The
issue is that x86 IOMMU has a (0, 2^64-1) IOMMU region, which is
splitted by the (0xfee00000, 0xfeefffff) IRQ region. AFAIK
this (splitted IOMMU region) is only happening on x86.
This patch also helps vhost to leverage the new interface as well, so
that vhost won't get duplicated cache flushes. In that sense, it's an
slight performance improvement.
Suggested-by: David Gibson <david@gibson.dropbear.id.au>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <1491562755-23867-2-git-send-email-peterx@redhat.com>
[ehabkost: included extra vhost_iommu_region_del() change from Peter Xu]
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2017-04-07 13:59:07 +03:00
|
|
|
Int128 end;
|
2020-07-22 11:40:48 +03:00
|
|
|
int iommu_idx;
|
2018-07-20 11:36:44 +03:00
|
|
|
IOMMUMemoryRegion *iommu_mr;
|
2021-02-04 22:12:28 +03:00
|
|
|
int ret;
|
2017-03-29 07:10:04 +03:00
|
|
|
|
|
|
|
if (!memory_region_is_iommu(section->mr)) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-07-20 11:36:44 +03:00
|
|
|
iommu_mr = IOMMU_MEMORY_REGION(section->mr);
|
|
|
|
|
2017-03-29 07:10:04 +03:00
|
|
|
iommu = g_malloc0(sizeof(*iommu));
|
memory: add section range info for IOMMU notifier
In this patch, IOMMUNotifier.{start|end} are introduced to store section
information for a specific notifier. When notification occurs, we not
only check the notification type (MAP|UNMAP), but also check whether the
notified iova range overlaps with the range of specific IOMMU notifier,
and skip those notifiers if not in the listened range.
When removing an region, we need to make sure we removed the correct
VFIOGuestIOMMU by checking the IOMMUNotifier.start address as well.
This patch is solving the problem that vfio-pci devices receive
duplicated UNMAP notification on x86 platform when vIOMMU is there. The
issue is that x86 IOMMU has a (0, 2^64-1) IOMMU region, which is
splitted by the (0xfee00000, 0xfeefffff) IRQ region. AFAIK
this (splitted IOMMU region) is only happening on x86.
This patch also helps vhost to leverage the new interface as well, so
that vhost won't get duplicated cache flushes. In that sense, it's an
slight performance improvement.
Suggested-by: David Gibson <david@gibson.dropbear.id.au>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <1491562755-23867-2-git-send-email-peterx@redhat.com>
[ehabkost: included extra vhost_iommu_region_del() change from Peter Xu]
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2017-04-07 13:59:07 +03:00
|
|
|
end = int128_add(int128_make64(section->offset_within_region),
|
|
|
|
section->size);
|
|
|
|
end = int128_sub(end, int128_one());
|
2018-06-15 16:57:16 +03:00
|
|
|
iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
|
|
|
|
MEMTXATTRS_UNSPECIFIED);
|
memory: add section range info for IOMMU notifier
In this patch, IOMMUNotifier.{start|end} are introduced to store section
information for a specific notifier. When notification occurs, we not
only check the notification type (MAP|UNMAP), but also check whether the
notified iova range overlaps with the range of specific IOMMU notifier,
and skip those notifiers if not in the listened range.
When removing an region, we need to make sure we removed the correct
VFIOGuestIOMMU by checking the IOMMUNotifier.start address as well.
This patch is solving the problem that vfio-pci devices receive
duplicated UNMAP notification on x86 platform when vIOMMU is there. The
issue is that x86 IOMMU has a (0, 2^64-1) IOMMU region, which is
splitted by the (0xfee00000, 0xfeefffff) IRQ region. AFAIK
this (splitted IOMMU region) is only happening on x86.
This patch also helps vhost to leverage the new interface as well, so
that vhost won't get duplicated cache flushes. In that sense, it's an
slight performance improvement.
Suggested-by: David Gibson <david@gibson.dropbear.id.au>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <1491562755-23867-2-git-send-email-peterx@redhat.com>
[ehabkost: included extra vhost_iommu_region_del() change from Peter Xu]
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2017-04-07 13:59:07 +03:00
|
|
|
iommu_notifier_init(&iommu->n, vhost_iommu_unmap_notify,
|
2020-11-16 19:55:04 +03:00
|
|
|
IOMMU_NOTIFIER_DEVIOTLB_UNMAP,
|
memory: add section range info for IOMMU notifier
In this patch, IOMMUNotifier.{start|end} are introduced to store section
information for a specific notifier. When notification occurs, we not
only check the notification type (MAP|UNMAP), but also check whether the
notified iova range overlaps with the range of specific IOMMU notifier,
and skip those notifiers if not in the listened range.
When removing an region, we need to make sure we removed the correct
VFIOGuestIOMMU by checking the IOMMUNotifier.start address as well.
This patch is solving the problem that vfio-pci devices receive
duplicated UNMAP notification on x86 platform when vIOMMU is there. The
issue is that x86 IOMMU has a (0, 2^64-1) IOMMU region, which is
splitted by the (0xfee00000, 0xfeefffff) IRQ region. AFAIK
this (splitted IOMMU region) is only happening on x86.
This patch also helps vhost to leverage the new interface as well, so
that vhost won't get duplicated cache flushes. In that sense, it's an
slight performance improvement.
Suggested-by: David Gibson <david@gibson.dropbear.id.au>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <1491562755-23867-2-git-send-email-peterx@redhat.com>
[ehabkost: included extra vhost_iommu_region_del() change from Peter Xu]
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2017-04-07 13:59:07 +03:00
|
|
|
section->offset_within_region,
|
2018-06-15 16:57:16 +03:00
|
|
|
int128_get64(end),
|
|
|
|
iommu_idx);
|
2017-03-29 07:10:04 +03:00
|
|
|
iommu->mr = section->mr;
|
|
|
|
iommu->iommu_offset = section->offset_within_address_space -
|
|
|
|
section->offset_within_region;
|
|
|
|
iommu->hdev = dev;
|
2021-02-04 22:12:28 +03:00
|
|
|
ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL);
|
|
|
|
if (ret) {
|
|
|
|
/*
|
|
|
|
* Some vIOMMUs do not support dev-iotlb yet. If so, try to use the
|
|
|
|
* UNMAP legacy message
|
|
|
|
*/
|
|
|
|
iommu->n.notifier_flags = IOMMU_NOTIFIER_UNMAP;
|
|
|
|
memory_region_register_iommu_notifier(section->mr, &iommu->n,
|
|
|
|
&error_fatal);
|
|
|
|
}
|
2017-03-29 07:10:04 +03:00
|
|
|
QLIST_INSERT_HEAD(&dev->iommu_list, iommu, iommu_next);
|
|
|
|
/* TODO: can replay help performance here? */
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_iommu_region_del(MemoryListener *listener,
|
|
|
|
MemoryRegionSection *section)
|
|
|
|
{
|
|
|
|
struct vhost_dev *dev = container_of(listener, struct vhost_dev,
|
|
|
|
iommu_listener);
|
|
|
|
struct vhost_iommu *iommu;
|
|
|
|
|
|
|
|
if (!memory_region_is_iommu(section->mr)) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
QLIST_FOREACH(iommu, &dev->iommu_list, iommu_next) {
|
memory: add section range info for IOMMU notifier
In this patch, IOMMUNotifier.{start|end} are introduced to store section
information for a specific notifier. When notification occurs, we not
only check the notification type (MAP|UNMAP), but also check whether the
notified iova range overlaps with the range of specific IOMMU notifier,
and skip those notifiers if not in the listened range.
When removing an region, we need to make sure we removed the correct
VFIOGuestIOMMU by checking the IOMMUNotifier.start address as well.
This patch is solving the problem that vfio-pci devices receive
duplicated UNMAP notification on x86 platform when vIOMMU is there. The
issue is that x86 IOMMU has a (0, 2^64-1) IOMMU region, which is
splitted by the (0xfee00000, 0xfeefffff) IRQ region. AFAIK
this (splitted IOMMU region) is only happening on x86.
This patch also helps vhost to leverage the new interface as well, so
that vhost won't get duplicated cache flushes. In that sense, it's an
slight performance improvement.
Suggested-by: David Gibson <david@gibson.dropbear.id.au>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <1491562755-23867-2-git-send-email-peterx@redhat.com>
[ehabkost: included extra vhost_iommu_region_del() change from Peter Xu]
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2017-04-07 13:59:07 +03:00
|
|
|
if (iommu->mr == section->mr &&
|
|
|
|
iommu->n.start == section->offset_within_region) {
|
2017-03-29 07:10:04 +03:00
|
|
|
memory_region_unregister_iommu_notifier(iommu->mr,
|
|
|
|
&iommu->n);
|
|
|
|
QLIST_REMOVE(iommu, iommu_next);
|
|
|
|
g_free(iommu);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
|
|
|
|
struct vhost_virtqueue *vq,
|
|
|
|
unsigned idx, bool enable_log)
|
|
|
|
{
|
2020-07-01 17:55:33 +03:00
|
|
|
struct vhost_vring_addr addr;
|
|
|
|
int r;
|
|
|
|
memset(&addr, 0, sizeof(struct vhost_vring_addr));
|
|
|
|
|
|
|
|
if (dev->vhost_ops->vhost_vq_get_addr) {
|
|
|
|
r = dev->vhost_ops->vhost_vq_get_addr(dev, &addr, vq);
|
|
|
|
if (r < 0) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_vq_get_addr failed");
|
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc;
|
|
|
|
addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail;
|
|
|
|
addr.used_user_addr = (uint64_t)(unsigned long)vq->used;
|
|
|
|
}
|
|
|
|
addr.index = idx;
|
|
|
|
addr.log_guest_addr = vq->used_phys;
|
|
|
|
addr.flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0;
|
|
|
|
r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r < 0) {
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
|
2010-03-17 14:08:17 +03:00
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-01-11 07:32:12 +03:00
|
|
|
static int vhost_dev_set_features(struct vhost_dev *dev,
|
|
|
|
bool enable_log)
|
2010-03-17 14:08:17 +03:00
|
|
|
{
|
|
|
|
uint64_t features = dev->acked_features;
|
|
|
|
int r;
|
|
|
|
if (enable_log) {
|
2015-06-04 13:34:20 +03:00
|
|
|
features |= 0x1ULL << VHOST_F_LOG_ALL;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2020-03-02 07:24:54 +03:00
|
|
|
if (!vhost_dev_has_iommu(dev)) {
|
|
|
|
features &= ~(0x1ULL << VIRTIO_F_IOMMU_PLATFORM);
|
|
|
|
}
|
2020-07-01 17:55:35 +03:00
|
|
|
if (dev->vhost_ops->vhost_force_iommu) {
|
|
|
|
if (dev->vhost_ops->vhost_force_iommu(dev) == true) {
|
|
|
|
features |= 0x1ULL << VIRTIO_F_IOMMU_PLATFORM;
|
|
|
|
}
|
|
|
|
}
|
2015-10-09 18:17:28 +03:00
|
|
|
r = dev->vhost_ops->vhost_set_features(dev, features);
|
2016-07-27 00:15:06 +03:00
|
|
|
if (r < 0) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_set_features failed");
|
2020-09-07 13:49:02 +03:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (dev->vhost_ops->vhost_set_backend_cap) {
|
|
|
|
r = dev->vhost_ops->vhost_set_backend_cap(dev);
|
|
|
|
if (r < 0) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_set_backend_cap failed");
|
|
|
|
goto out;
|
|
|
|
}
|
2016-07-27 00:15:06 +03:00
|
|
|
}
|
2020-09-07 13:49:02 +03:00
|
|
|
|
|
|
|
out:
|
2010-03-17 14:08:17 +03:00
|
|
|
return r < 0 ? -errno : 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
|
|
|
|
{
|
2016-07-27 00:15:05 +03:00
|
|
|
int r, i, idx;
|
2020-09-11 11:39:44 +03:00
|
|
|
hwaddr addr;
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
r = vhost_dev_set_features(dev, enable_log);
|
|
|
|
if (r < 0) {
|
|
|
|
goto err_features;
|
|
|
|
}
|
|
|
|
for (i = 0; i < dev->nvqs; ++i) {
|
2015-10-19 15:59:27 +03:00
|
|
|
idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
|
2020-09-11 11:39:44 +03:00
|
|
|
addr = virtio_queue_get_desc_addr(dev->vdev, idx);
|
|
|
|
if (!addr) {
|
|
|
|
/*
|
|
|
|
* The queue might not be ready for start. If this
|
|
|
|
* is the case there is no reason to continue the process.
|
|
|
|
* The similar logic is used by the vhost_virtqueue_start()
|
|
|
|
* routine.
|
|
|
|
*/
|
|
|
|
continue;
|
|
|
|
}
|
2015-10-19 15:59:27 +03:00
|
|
|
r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
|
2010-03-17 14:08:17 +03:00
|
|
|
enable_log);
|
|
|
|
if (r < 0) {
|
|
|
|
goto err_vq;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
err_vq:
|
|
|
|
for (; i >= 0; --i) {
|
2015-10-19 15:59:27 +03:00
|
|
|
idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
|
2016-07-27 00:15:05 +03:00
|
|
|
vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
|
|
|
|
dev->log_enabled);
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2016-07-27 00:15:05 +03:00
|
|
|
vhost_dev_set_features(dev, dev->log_enabled);
|
2010-03-17 14:08:17 +03:00
|
|
|
err_features:
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2020-05-08 00:37:58 +03:00
|
|
|
static int vhost_migration_log(MemoryListener *listener, bool enable)
|
2010-03-17 14:08:17 +03:00
|
|
|
{
|
2011-12-18 16:06:05 +04:00
|
|
|
struct vhost_dev *dev = container_of(listener, struct vhost_dev,
|
|
|
|
memory_listener);
|
2010-03-17 14:08:17 +03:00
|
|
|
int r;
|
2020-05-08 00:37:58 +03:00
|
|
|
if (enable == dev->log_enabled) {
|
2010-03-17 14:08:17 +03:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (!dev->started) {
|
|
|
|
dev->log_enabled = enable;
|
|
|
|
return 0;
|
|
|
|
}
|
vhost: recheck dev state in the vhost_migration_log routine
vhost-user devices can get a disconnect in the middle of the VHOST-USER
handshake on the migration start. If disconnect event happened right
before sending next VHOST-USER command, then the vhost_dev_set_log()
call in the vhost_migration_log() function will return error. This error
will lead to the assert() and close the QEMU migration source process.
For the vhost-user devices the disconnect event should not break the
migration process, because:
- the device will be in the stopped state, so it will not be changed
during migration
- if reconnect will be made the migration log will be reinitialized as
part of reconnect/init process:
#0 vhost_log_global_start (listener=0x563989cf7be0)
at hw/virtio/vhost.c:920
#1 0x000056398603d8bc in listener_add_address_space (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2664
#2 0x000056398603dd30 in memory_listener_register (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2740
#3 0x0000563985fd6956 in vhost_dev_init (hdev=0x563989cf7bd8,
opaque=0x563989cf7e30, backend_type=VHOST_BACKEND_TYPE_USER,
busyloop_timeout=0)
at hw/virtio/vhost.c:1385
#4 0x0000563985f7d0b8 in vhost_user_blk_connect (dev=0x563989cf7990)
at hw/block/vhost-user-blk.c:315
#5 0x0000563985f7d3f6 in vhost_user_blk_event (opaque=0x563989cf7990,
event=CHR_EVENT_OPENED)
at hw/block/vhost-user-blk.c:379
Update the vhost-user-blk device with the internal started_vu field which
will be used for initialization (vhost_user_blk_start) and clean up
(vhost_user_blk_stop). This additional flag in the VhostUserBlk structure
will be used to track whether the device really needs to be stopped and
cleaned up on a vhost-user level.
The disconnect event will set the overall VHOST device (not vhost-user) to
the stopped state, so it can be used by the general vhost_migration_log
routine.
Such approach could be propogated to the other vhost-user devices, but
better idea is just to make the same connect/disconnect code for all the
vhost-user devices.
This migration issue was slightly discussed earlier:
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg01509.html
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg05241.html
Signed-off-by: Dima Stepanov <dimastep@yandex-team.ru>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
Message-Id: <9fbfba06791a87813fcee3e2315f0b904cc6789a.1599813294.git.dimastep@yandex-team.ru>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2020-09-11 11:39:43 +03:00
|
|
|
|
|
|
|
r = 0;
|
2010-03-17 14:08:17 +03:00
|
|
|
if (!enable) {
|
|
|
|
r = vhost_dev_set_log(dev, false);
|
|
|
|
if (r < 0) {
|
vhost: recheck dev state in the vhost_migration_log routine
vhost-user devices can get a disconnect in the middle of the VHOST-USER
handshake on the migration start. If disconnect event happened right
before sending next VHOST-USER command, then the vhost_dev_set_log()
call in the vhost_migration_log() function will return error. This error
will lead to the assert() and close the QEMU migration source process.
For the vhost-user devices the disconnect event should not break the
migration process, because:
- the device will be in the stopped state, so it will not be changed
during migration
- if reconnect will be made the migration log will be reinitialized as
part of reconnect/init process:
#0 vhost_log_global_start (listener=0x563989cf7be0)
at hw/virtio/vhost.c:920
#1 0x000056398603d8bc in listener_add_address_space (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2664
#2 0x000056398603dd30 in memory_listener_register (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2740
#3 0x0000563985fd6956 in vhost_dev_init (hdev=0x563989cf7bd8,
opaque=0x563989cf7e30, backend_type=VHOST_BACKEND_TYPE_USER,
busyloop_timeout=0)
at hw/virtio/vhost.c:1385
#4 0x0000563985f7d0b8 in vhost_user_blk_connect (dev=0x563989cf7990)
at hw/block/vhost-user-blk.c:315
#5 0x0000563985f7d3f6 in vhost_user_blk_event (opaque=0x563989cf7990,
event=CHR_EVENT_OPENED)
at hw/block/vhost-user-blk.c:379
Update the vhost-user-blk device with the internal started_vu field which
will be used for initialization (vhost_user_blk_start) and clean up
(vhost_user_blk_stop). This additional flag in the VhostUserBlk structure
will be used to track whether the device really needs to be stopped and
cleaned up on a vhost-user level.
The disconnect event will set the overall VHOST device (not vhost-user) to
the stopped state, so it can be used by the general vhost_migration_log
routine.
Such approach could be propogated to the other vhost-user devices, but
better idea is just to make the same connect/disconnect code for all the
vhost-user devices.
This migration issue was slightly discussed earlier:
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg01509.html
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg05241.html
Signed-off-by: Dima Stepanov <dimastep@yandex-team.ru>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
Message-Id: <9fbfba06791a87813fcee3e2315f0b904cc6789a.1599813294.git.dimastep@yandex-team.ru>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2020-09-11 11:39:43 +03:00
|
|
|
goto check_dev_state;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2015-06-04 12:28:46 +03:00
|
|
|
vhost_log_put(dev, false);
|
2010-03-17 14:08:17 +03:00
|
|
|
} else {
|
|
|
|
vhost_dev_log_resize(dev, vhost_get_log_size(dev));
|
|
|
|
r = vhost_dev_set_log(dev, true);
|
|
|
|
if (r < 0) {
|
vhost: recheck dev state in the vhost_migration_log routine
vhost-user devices can get a disconnect in the middle of the VHOST-USER
handshake on the migration start. If disconnect event happened right
before sending next VHOST-USER command, then the vhost_dev_set_log()
call in the vhost_migration_log() function will return error. This error
will lead to the assert() and close the QEMU migration source process.
For the vhost-user devices the disconnect event should not break the
migration process, because:
- the device will be in the stopped state, so it will not be changed
during migration
- if reconnect will be made the migration log will be reinitialized as
part of reconnect/init process:
#0 vhost_log_global_start (listener=0x563989cf7be0)
at hw/virtio/vhost.c:920
#1 0x000056398603d8bc in listener_add_address_space (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2664
#2 0x000056398603dd30 in memory_listener_register (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2740
#3 0x0000563985fd6956 in vhost_dev_init (hdev=0x563989cf7bd8,
opaque=0x563989cf7e30, backend_type=VHOST_BACKEND_TYPE_USER,
busyloop_timeout=0)
at hw/virtio/vhost.c:1385
#4 0x0000563985f7d0b8 in vhost_user_blk_connect (dev=0x563989cf7990)
at hw/block/vhost-user-blk.c:315
#5 0x0000563985f7d3f6 in vhost_user_blk_event (opaque=0x563989cf7990,
event=CHR_EVENT_OPENED)
at hw/block/vhost-user-blk.c:379
Update the vhost-user-blk device with the internal started_vu field which
will be used for initialization (vhost_user_blk_start) and clean up
(vhost_user_blk_stop). This additional flag in the VhostUserBlk structure
will be used to track whether the device really needs to be stopped and
cleaned up on a vhost-user level.
The disconnect event will set the overall VHOST device (not vhost-user) to
the stopped state, so it can be used by the general vhost_migration_log
routine.
Such approach could be propogated to the other vhost-user devices, but
better idea is just to make the same connect/disconnect code for all the
vhost-user devices.
This migration issue was slightly discussed earlier:
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg01509.html
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg05241.html
Signed-off-by: Dima Stepanov <dimastep@yandex-team.ru>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
Message-Id: <9fbfba06791a87813fcee3e2315f0b904cc6789a.1599813294.git.dimastep@yandex-team.ru>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2020-09-11 11:39:43 +03:00
|
|
|
goto check_dev_state;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
|
|
|
}
|
vhost: recheck dev state in the vhost_migration_log routine
vhost-user devices can get a disconnect in the middle of the VHOST-USER
handshake on the migration start. If disconnect event happened right
before sending next VHOST-USER command, then the vhost_dev_set_log()
call in the vhost_migration_log() function will return error. This error
will lead to the assert() and close the QEMU migration source process.
For the vhost-user devices the disconnect event should not break the
migration process, because:
- the device will be in the stopped state, so it will not be changed
during migration
- if reconnect will be made the migration log will be reinitialized as
part of reconnect/init process:
#0 vhost_log_global_start (listener=0x563989cf7be0)
at hw/virtio/vhost.c:920
#1 0x000056398603d8bc in listener_add_address_space (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2664
#2 0x000056398603dd30 in memory_listener_register (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2740
#3 0x0000563985fd6956 in vhost_dev_init (hdev=0x563989cf7bd8,
opaque=0x563989cf7e30, backend_type=VHOST_BACKEND_TYPE_USER,
busyloop_timeout=0)
at hw/virtio/vhost.c:1385
#4 0x0000563985f7d0b8 in vhost_user_blk_connect (dev=0x563989cf7990)
at hw/block/vhost-user-blk.c:315
#5 0x0000563985f7d3f6 in vhost_user_blk_event (opaque=0x563989cf7990,
event=CHR_EVENT_OPENED)
at hw/block/vhost-user-blk.c:379
Update the vhost-user-blk device with the internal started_vu field which
will be used for initialization (vhost_user_blk_start) and clean up
(vhost_user_blk_stop). This additional flag in the VhostUserBlk structure
will be used to track whether the device really needs to be stopped and
cleaned up on a vhost-user level.
The disconnect event will set the overall VHOST device (not vhost-user) to
the stopped state, so it can be used by the general vhost_migration_log
routine.
Such approach could be propogated to the other vhost-user devices, but
better idea is just to make the same connect/disconnect code for all the
vhost-user devices.
This migration issue was slightly discussed earlier:
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg01509.html
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg05241.html
Signed-off-by: Dima Stepanov <dimastep@yandex-team.ru>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
Message-Id: <9fbfba06791a87813fcee3e2315f0b904cc6789a.1599813294.git.dimastep@yandex-team.ru>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2020-09-11 11:39:43 +03:00
|
|
|
|
|
|
|
check_dev_state:
|
2010-03-17 14:08:17 +03:00
|
|
|
dev->log_enabled = enable;
|
vhost: recheck dev state in the vhost_migration_log routine
vhost-user devices can get a disconnect in the middle of the VHOST-USER
handshake on the migration start. If disconnect event happened right
before sending next VHOST-USER command, then the vhost_dev_set_log()
call in the vhost_migration_log() function will return error. This error
will lead to the assert() and close the QEMU migration source process.
For the vhost-user devices the disconnect event should not break the
migration process, because:
- the device will be in the stopped state, so it will not be changed
during migration
- if reconnect will be made the migration log will be reinitialized as
part of reconnect/init process:
#0 vhost_log_global_start (listener=0x563989cf7be0)
at hw/virtio/vhost.c:920
#1 0x000056398603d8bc in listener_add_address_space (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2664
#2 0x000056398603dd30 in memory_listener_register (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2740
#3 0x0000563985fd6956 in vhost_dev_init (hdev=0x563989cf7bd8,
opaque=0x563989cf7e30, backend_type=VHOST_BACKEND_TYPE_USER,
busyloop_timeout=0)
at hw/virtio/vhost.c:1385
#4 0x0000563985f7d0b8 in vhost_user_blk_connect (dev=0x563989cf7990)
at hw/block/vhost-user-blk.c:315
#5 0x0000563985f7d3f6 in vhost_user_blk_event (opaque=0x563989cf7990,
event=CHR_EVENT_OPENED)
at hw/block/vhost-user-blk.c:379
Update the vhost-user-blk device with the internal started_vu field which
will be used for initialization (vhost_user_blk_start) and clean up
(vhost_user_blk_stop). This additional flag in the VhostUserBlk structure
will be used to track whether the device really needs to be stopped and
cleaned up on a vhost-user level.
The disconnect event will set the overall VHOST device (not vhost-user) to
the stopped state, so it can be used by the general vhost_migration_log
routine.
Such approach could be propogated to the other vhost-user devices, but
better idea is just to make the same connect/disconnect code for all the
vhost-user devices.
This migration issue was slightly discussed earlier:
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg01509.html
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg05241.html
Signed-off-by: Dima Stepanov <dimastep@yandex-team.ru>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
Message-Id: <9fbfba06791a87813fcee3e2315f0b904cc6789a.1599813294.git.dimastep@yandex-team.ru>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2020-09-11 11:39:43 +03:00
|
|
|
/*
|
|
|
|
* vhost-user-* devices could change their state during log
|
|
|
|
* initialization due to disconnect. So check dev state after
|
|
|
|
* vhost communication.
|
|
|
|
*/
|
|
|
|
if (!dev->started) {
|
|
|
|
/*
|
|
|
|
* Since device is in the stopped state, it is okay for
|
|
|
|
* migration. Return success.
|
|
|
|
*/
|
|
|
|
r = 0;
|
|
|
|
}
|
|
|
|
if (r) {
|
2021-03-09 14:15:10 +03:00
|
|
|
/* An error occurred. */
|
vhost: recheck dev state in the vhost_migration_log routine
vhost-user devices can get a disconnect in the middle of the VHOST-USER
handshake on the migration start. If disconnect event happened right
before sending next VHOST-USER command, then the vhost_dev_set_log()
call in the vhost_migration_log() function will return error. This error
will lead to the assert() and close the QEMU migration source process.
For the vhost-user devices the disconnect event should not break the
migration process, because:
- the device will be in the stopped state, so it will not be changed
during migration
- if reconnect will be made the migration log will be reinitialized as
part of reconnect/init process:
#0 vhost_log_global_start (listener=0x563989cf7be0)
at hw/virtio/vhost.c:920
#1 0x000056398603d8bc in listener_add_address_space (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2664
#2 0x000056398603dd30 in memory_listener_register (listener=0x563989cf7be0,
as=0x563986ea4340 <address_space_memory>)
at softmmu/memory.c:2740
#3 0x0000563985fd6956 in vhost_dev_init (hdev=0x563989cf7bd8,
opaque=0x563989cf7e30, backend_type=VHOST_BACKEND_TYPE_USER,
busyloop_timeout=0)
at hw/virtio/vhost.c:1385
#4 0x0000563985f7d0b8 in vhost_user_blk_connect (dev=0x563989cf7990)
at hw/block/vhost-user-blk.c:315
#5 0x0000563985f7d3f6 in vhost_user_blk_event (opaque=0x563989cf7990,
event=CHR_EVENT_OPENED)
at hw/block/vhost-user-blk.c:379
Update the vhost-user-blk device with the internal started_vu field which
will be used for initialization (vhost_user_blk_start) and clean up
(vhost_user_blk_stop). This additional flag in the VhostUserBlk structure
will be used to track whether the device really needs to be stopped and
cleaned up on a vhost-user level.
The disconnect event will set the overall VHOST device (not vhost-user) to
the stopped state, so it can be used by the general vhost_migration_log
routine.
Such approach could be propogated to the other vhost-user devices, but
better idea is just to make the same connect/disconnect code for all the
vhost-user devices.
This migration issue was slightly discussed earlier:
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg01509.html
- https://lists.gnu.org/archive/html/qemu-devel/2020-05/msg05241.html
Signed-off-by: Dima Stepanov <dimastep@yandex-team.ru>
Reviewed-by: Raphael Norwitz <raphael.norwitz@nutanix.com>
Message-Id: <9fbfba06791a87813fcee3e2315f0b904cc6789a.1599813294.git.dimastep@yandex-team.ru>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2020-09-11 11:39:43 +03:00
|
|
|
dev->log_enabled = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return r;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
|
|
|
|
2011-12-18 16:06:05 +04:00
|
|
|
static void vhost_log_global_start(MemoryListener *listener)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
|
|
|
|
r = vhost_migration_log(listener, true);
|
|
|
|
if (r < 0) {
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_log_global_stop(MemoryListener *listener)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
|
|
|
|
r = vhost_migration_log(listener, false);
|
|
|
|
if (r < 0) {
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_log_start(MemoryListener *listener,
|
2015-04-25 15:38:30 +03:00
|
|
|
MemoryRegionSection *section,
|
|
|
|
int old, int new)
|
2011-12-18 16:06:05 +04:00
|
|
|
{
|
|
|
|
/* FIXME: implement */
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_log_stop(MemoryListener *listener,
|
2015-04-25 15:38:30 +03:00
|
|
|
MemoryRegionSection *section,
|
|
|
|
int old, int new)
|
2011-12-18 16:06:05 +04:00
|
|
|
{
|
|
|
|
/* FIXME: implement */
|
|
|
|
}
|
|
|
|
|
2016-02-05 13:46:04 +03:00
|
|
|
/* The vhost driver natively knows how to handle the vrings of non
|
|
|
|
* cross-endian legacy devices and modern devices. Only legacy devices
|
|
|
|
* exposed to a bi-endian guest may require the vhost driver to use a
|
|
|
|
* specific endianness.
|
|
|
|
*/
|
2016-02-05 13:45:40 +03:00
|
|
|
static inline bool vhost_needs_vring_endian(VirtIODevice *vdev)
|
|
|
|
{
|
2016-02-05 13:45:49 +03:00
|
|
|
if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
|
|
|
|
return false;
|
|
|
|
}
|
2016-02-05 13:45:40 +03:00
|
|
|
#ifdef HOST_WORDS_BIGENDIAN
|
2016-02-05 13:46:04 +03:00
|
|
|
return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_LITTLE;
|
2016-02-05 13:45:40 +03:00
|
|
|
#else
|
2016-02-05 13:46:04 +03:00
|
|
|
return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
|
2016-02-05 13:45:40 +03:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2015-06-17 16:23:39 +03:00
|
|
|
static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev,
|
|
|
|
bool is_big_endian,
|
|
|
|
int vhost_vq_index)
|
|
|
|
{
|
|
|
|
struct vhost_vring_state s = {
|
|
|
|
.index = vhost_vq_index,
|
|
|
|
.num = is_big_endian
|
|
|
|
};
|
|
|
|
|
2015-10-09 18:17:28 +03:00
|
|
|
if (!dev->vhost_ops->vhost_set_vring_endian(dev, &s)) {
|
2015-06-17 16:23:39 +03:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_vring_endian failed");
|
2015-06-17 16:23:39 +03:00
|
|
|
if (errno == ENOTTY) {
|
|
|
|
error_report("vhost does not support cross-endian");
|
|
|
|
return -ENOSYS;
|
|
|
|
}
|
|
|
|
|
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
|
2017-01-11 07:32:12 +03:00
|
|
|
static int vhost_memory_region_lookup(struct vhost_dev *hdev,
|
|
|
|
uint64_t gpa, uint64_t *uaddr,
|
|
|
|
uint64_t *len)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < hdev->mem->nregions; i++) {
|
|
|
|
struct vhost_memory_region *reg = hdev->mem->regions + i;
|
|
|
|
|
|
|
|
if (gpa >= reg->guest_phys_addr &&
|
|
|
|
reg->guest_phys_addr + reg->memory_size > gpa) {
|
|
|
|
*uaddr = reg->userspace_addr + gpa - reg->guest_phys_addr;
|
|
|
|
*len = reg->guest_phys_addr + reg->memory_size - gpa;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return -EFAULT;
|
|
|
|
}
|
|
|
|
|
2017-06-02 13:18:27 +03:00
|
|
|
int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write)
|
2017-01-11 07:32:12 +03:00
|
|
|
{
|
|
|
|
IOMMUTLBEntry iotlb;
|
|
|
|
uint64_t uaddr, len;
|
2017-06-02 13:18:27 +03:00
|
|
|
int ret = -EFAULT;
|
2017-01-11 07:32:12 +03:00
|
|
|
|
2019-10-25 13:34:02 +03:00
|
|
|
RCU_READ_LOCK_GUARD();
|
2017-01-11 07:32:12 +03:00
|
|
|
|
2018-04-27 12:07:24 +03:00
|
|
|
trace_vhost_iotlb_miss(dev, 1);
|
|
|
|
|
2017-01-11 07:32:12 +03:00
|
|
|
iotlb = address_space_get_iotlb_entry(dev->vdev->dma_as,
|
2018-05-31 16:50:53 +03:00
|
|
|
iova, write,
|
|
|
|
MEMTXATTRS_UNSPECIFIED);
|
2017-01-11 07:32:12 +03:00
|
|
|
if (iotlb.target_as != NULL) {
|
2017-06-02 13:18:27 +03:00
|
|
|
ret = vhost_memory_region_lookup(dev, iotlb.translated_addr,
|
|
|
|
&uaddr, &len);
|
|
|
|
if (ret) {
|
2018-04-27 12:07:24 +03:00
|
|
|
trace_vhost_iotlb_miss(dev, 3);
|
2017-01-11 07:32:12 +03:00
|
|
|
error_report("Fail to lookup the translated address "
|
|
|
|
"%"PRIx64, iotlb.translated_addr);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
len = MIN(iotlb.addr_mask + 1, len);
|
|
|
|
iova = iova & ~iotlb.addr_mask;
|
|
|
|
|
2017-06-02 13:18:28 +03:00
|
|
|
ret = vhost_backend_update_device_iotlb(dev, iova, uaddr,
|
|
|
|
len, iotlb.perm);
|
2017-06-02 13:18:27 +03:00
|
|
|
if (ret) {
|
2018-04-27 12:07:24 +03:00
|
|
|
trace_vhost_iotlb_miss(dev, 4);
|
2017-01-11 07:32:12 +03:00
|
|
|
error_report("Fail to update device iotlb");
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
2018-04-27 12:07:24 +03:00
|
|
|
|
|
|
|
trace_vhost_iotlb_miss(dev, 2);
|
|
|
|
|
2017-01-11 07:32:12 +03:00
|
|
|
out:
|
2017-06-02 13:18:27 +03:00
|
|
|
return ret;
|
2017-01-11 07:32:12 +03:00
|
|
|
}
|
|
|
|
|
2012-12-24 19:37:01 +04:00
|
|
|
static int vhost_virtqueue_start(struct vhost_dev *dev,
|
2010-03-17 14:08:17 +03:00
|
|
|
struct VirtIODevice *vdev,
|
|
|
|
struct vhost_virtqueue *vq,
|
|
|
|
unsigned idx)
|
|
|
|
{
|
2016-08-01 11:07:58 +03:00
|
|
|
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
|
|
|
|
VirtioBusState *vbus = VIRTIO_BUS(qbus);
|
|
|
|
VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
|
2012-10-23 14:30:10 +04:00
|
|
|
hwaddr s, l, a;
|
2010-03-17 14:08:17 +03:00
|
|
|
int r;
|
2015-10-09 18:17:28 +03:00
|
|
|
int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
|
2010-03-17 14:08:17 +03:00
|
|
|
struct vhost_vring_file file = {
|
2013-01-30 15:12:35 +04:00
|
|
|
.index = vhost_vq_index
|
2010-03-17 14:08:17 +03:00
|
|
|
};
|
|
|
|
struct vhost_vring_state state = {
|
2013-01-30 15:12:35 +04:00
|
|
|
.index = vhost_vq_index
|
2010-03-17 14:08:17 +03:00
|
|
|
};
|
|
|
|
struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
|
|
|
|
|
2018-02-28 12:35:28 +03:00
|
|
|
a = virtio_queue_get_desc_addr(vdev, idx);
|
|
|
|
if (a == 0) {
|
|
|
|
/* Queue might not be ready for start */
|
|
|
|
return 0;
|
|
|
|
}
|
2013-01-30 15:12:35 +04:00
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
vq->num = state.num = virtio_queue_get_num(vdev, idx);
|
2015-10-09 18:17:28 +03:00
|
|
|
r = dev->vhost_ops->vhost_set_vring_num(dev, &state);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r) {
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_vring_num failed");
|
2010-03-17 14:08:17 +03:00
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
|
|
|
|
state.num = virtio_queue_get_last_avail_idx(vdev, idx);
|
2015-10-09 18:17:28 +03:00
|
|
|
r = dev->vhost_ops->vhost_set_vring_base(dev, &state);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r) {
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_vring_base failed");
|
2010-03-17 14:08:17 +03:00
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
|
2016-02-05 13:45:49 +03:00
|
|
|
if (vhost_needs_vring_endian(vdev)) {
|
2015-06-17 16:23:39 +03:00
|
|
|
r = vhost_virtqueue_set_vring_endian_legacy(dev,
|
|
|
|
virtio_is_big_endian(vdev),
|
|
|
|
vhost_vq_index);
|
|
|
|
if (r) {
|
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-11-04 11:39:15 +03:00
|
|
|
vq->desc_size = s = l = virtio_queue_get_desc_size(vdev, idx);
|
2018-02-28 12:35:28 +03:00
|
|
|
vq->desc_phys = a;
|
2020-02-19 21:18:45 +03:00
|
|
|
vq->desc = vhost_memory_map(dev, a, &l, false);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (!vq->desc || l != s) {
|
|
|
|
r = -ENOMEM;
|
|
|
|
goto fail_alloc_desc;
|
|
|
|
}
|
2016-11-04 11:39:15 +03:00
|
|
|
vq->avail_size = s = l = virtio_queue_get_avail_size(vdev, idx);
|
|
|
|
vq->avail_phys = a = virtio_queue_get_avail_addr(vdev, idx);
|
2020-02-19 21:18:45 +03:00
|
|
|
vq->avail = vhost_memory_map(dev, a, &l, false);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (!vq->avail || l != s) {
|
|
|
|
r = -ENOMEM;
|
|
|
|
goto fail_alloc_avail;
|
|
|
|
}
|
|
|
|
vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx);
|
|
|
|
vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx);
|
2020-02-19 21:18:45 +03:00
|
|
|
vq->used = vhost_memory_map(dev, a, &l, true);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (!vq->used || l != s) {
|
|
|
|
r = -ENOMEM;
|
|
|
|
goto fail_alloc_used;
|
|
|
|
}
|
|
|
|
|
2013-01-30 15:12:35 +04:00
|
|
|
r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r < 0) {
|
|
|
|
r = -errno;
|
|
|
|
goto fail_alloc;
|
|
|
|
}
|
2013-01-30 15:12:35 +04:00
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
|
2015-10-09 18:17:28 +03:00
|
|
|
r = dev->vhost_ops->vhost_set_vring_kick(dev, &file);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r) {
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_vring_kick failed");
|
2010-10-06 17:20:28 +04:00
|
|
|
r = -errno;
|
2010-03-17 14:08:17 +03:00
|
|
|
goto fail_kick;
|
|
|
|
}
|
|
|
|
|
2012-12-24 19:37:01 +04:00
|
|
|
/* Clear and discard previous events if any. */
|
|
|
|
event_notifier_test_and_clear(&vq->masked_notifier);
|
2010-03-17 14:08:17 +03:00
|
|
|
|
2016-02-18 17:12:23 +03:00
|
|
|
/* Init vring in unmasked state, unless guest_notifier_mask
|
|
|
|
* will do it later.
|
|
|
|
*/
|
|
|
|
if (!vdev->use_guest_notifier_mask) {
|
|
|
|
/* TODO: check and handle errors. */
|
|
|
|
vhost_virtqueue_mask(dev, vdev, idx, false);
|
|
|
|
}
|
|
|
|
|
2016-08-01 11:07:58 +03:00
|
|
|
if (k->query_guest_notifiers &&
|
|
|
|
k->query_guest_notifiers(qbus->parent) &&
|
|
|
|
virtio_queue_vector(vdev, idx) == VIRTIO_NO_VECTOR) {
|
|
|
|
file.fd = -1;
|
|
|
|
r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
|
|
|
|
if (r) {
|
|
|
|
goto fail_vector;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
return 0;
|
|
|
|
|
2016-08-01 11:07:58 +03:00
|
|
|
fail_vector:
|
2010-03-17 14:08:17 +03:00
|
|
|
fail_kick:
|
|
|
|
fail_alloc:
|
2017-01-11 07:32:12 +03:00
|
|
|
vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx),
|
|
|
|
0, 0);
|
2010-03-17 14:08:17 +03:00
|
|
|
fail_alloc_used:
|
2017-01-11 07:32:12 +03:00
|
|
|
vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx),
|
|
|
|
0, 0);
|
2010-03-17 14:08:17 +03:00
|
|
|
fail_alloc_avail:
|
2017-01-11 07:32:12 +03:00
|
|
|
vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx),
|
|
|
|
0, 0);
|
2010-03-17 14:08:17 +03:00
|
|
|
fail_alloc_desc:
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2012-12-24 19:37:01 +04:00
|
|
|
static void vhost_virtqueue_stop(struct vhost_dev *dev,
|
2010-03-17 14:08:17 +03:00
|
|
|
struct VirtIODevice *vdev,
|
|
|
|
struct vhost_virtqueue *vq,
|
|
|
|
unsigned idx)
|
|
|
|
{
|
2015-10-09 18:17:28 +03:00
|
|
|
int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
|
2010-03-17 14:08:17 +03:00
|
|
|
struct vhost_vring_state state = {
|
2015-06-17 16:23:39 +03:00
|
|
|
.index = vhost_vq_index,
|
2010-03-17 14:08:17 +03:00
|
|
|
};
|
|
|
|
int r;
|
2018-02-28 12:35:28 +03:00
|
|
|
|
2018-07-13 17:04:05 +03:00
|
|
|
if (virtio_queue_get_desc_addr(vdev, idx) == 0) {
|
2018-02-28 12:35:28 +03:00
|
|
|
/* Don't stop the virtqueue which might have not been started */
|
|
|
|
return;
|
|
|
|
}
|
2015-09-23 07:19:59 +03:00
|
|
|
|
2015-10-09 18:17:28 +03:00
|
|
|
r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r < 0) {
|
2019-04-30 09:29:33 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost VQ %u ring restore failed: %d", idx, r);
|
2017-11-16 21:48:35 +03:00
|
|
|
/* Connection to the backend is broken, so let's sync internal
|
|
|
|
* last avail idx to the device used idx.
|
|
|
|
*/
|
|
|
|
virtio_queue_restore_last_avail_idx(vdev, idx);
|
2016-07-27 00:15:27 +03:00
|
|
|
} else {
|
|
|
|
virtio_queue_set_last_avail_idx(vdev, idx, state.num);
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2013-08-12 13:21:36 +04:00
|
|
|
virtio_queue_invalidate_signalled_used(vdev, idx);
|
2016-12-13 11:12:06 +03:00
|
|
|
virtio_queue_update_used_idx(vdev, idx);
|
2015-06-17 16:23:39 +03:00
|
|
|
|
|
|
|
/* In the cross-endian case, we need to reset the vring endianness to
|
|
|
|
* native as legacy devices expect so by default.
|
|
|
|
*/
|
2016-02-05 13:45:49 +03:00
|
|
|
if (vhost_needs_vring_endian(vdev)) {
|
2016-07-27 00:15:05 +03:00
|
|
|
vhost_virtqueue_set_vring_endian_legacy(dev,
|
|
|
|
!virtio_is_big_endian(vdev),
|
|
|
|
vhost_vq_index);
|
2015-06-17 16:23:39 +03:00
|
|
|
}
|
|
|
|
|
2017-01-11 07:32:12 +03:00
|
|
|
vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx),
|
|
|
|
1, virtio_queue_get_used_size(vdev, idx));
|
|
|
|
vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx),
|
|
|
|
0, virtio_queue_get_avail_size(vdev, idx));
|
|
|
|
vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx),
|
|
|
|
0, virtio_queue_get_desc_size(vdev, idx));
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
|
|
|
|
2012-02-08 18:39:06 +04:00
|
|
|
static void vhost_eventfd_add(MemoryListener *listener,
|
|
|
|
MemoryRegionSection *section,
|
2012-07-05 19:16:27 +04:00
|
|
|
bool match_data, uint64_t data, EventNotifier *e)
|
2012-02-08 18:39:06 +04:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_eventfd_del(MemoryListener *listener,
|
|
|
|
MemoryRegionSection *section,
|
2012-07-05 19:16:27 +04:00
|
|
|
bool match_data, uint64_t data, EventNotifier *e)
|
2012-02-08 18:39:06 +04:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2016-07-06 04:57:55 +03:00
|
|
|
static int vhost_virtqueue_set_busyloop_timeout(struct vhost_dev *dev,
|
|
|
|
int n, uint32_t timeout)
|
|
|
|
{
|
|
|
|
int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
|
|
|
|
struct vhost_vring_state state = {
|
|
|
|
.index = vhost_vq_index,
|
|
|
|
.num = timeout,
|
|
|
|
};
|
|
|
|
int r;
|
|
|
|
|
|
|
|
if (!dev->vhost_ops->vhost_set_vring_busyloop_timeout) {
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
r = dev->vhost_ops->vhost_set_vring_busyloop_timeout(dev, &state);
|
|
|
|
if (r) {
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_vring_busyloop_timeout failed");
|
2016-07-06 04:57:55 +03:00
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-12-24 19:37:01 +04:00
|
|
|
static int vhost_virtqueue_init(struct vhost_dev *dev,
|
|
|
|
struct vhost_virtqueue *vq, int n)
|
|
|
|
{
|
2015-10-09 18:17:28 +03:00
|
|
|
int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
|
2012-12-24 19:37:01 +04:00
|
|
|
struct vhost_vring_file file = {
|
vhost-user: add multiple queue support
This patch is initially based a patch from Nikolay Nikolaev.
This patch adds vhost-user multiple queue support, by creating a nc
and vhost_net pair for each queue.
Qemu exits if find that the backend can't support the number of requested
queues (by providing queues=# option). The max number is queried by a
new message, VHOST_USER_GET_QUEUE_NUM, and is sent only when protocol
feature VHOST_USER_PROTOCOL_F_MQ is present first.
The max queue check is done at vhost-user initiation stage. We initiate
one queue first, which, in the meantime, also gets the max_queues the
backend supports.
In older version, it was reported that some messages are sent more times
than necessary. Here we came an agreement with Michael that we could
categorize vhost user messages to 2 types: non-vring specific messages,
which should be sent only once, and vring specific messages, which should
be sent per queue.
Here I introduced a helper function vhost_user_one_time_request(), which
lists following messages as non-vring specific messages:
VHOST_USER_SET_OWNER
VHOST_USER_RESET_DEVICE
VHOST_USER_SET_MEM_TABLE
VHOST_USER_GET_QUEUE_NUM
For above messages, we simply ignore them when they are not sent the first
time.
Signed-off-by: Nikolay Nikolaev <n.nikolaev@virtualopensystems.com>
Signed-off-by: Changchun Ouyang <changchun.ouyang@intel.com>
Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Reviewed-by: Jason Wang <jasowang@redhat.com>
Tested-by: Marcel Apfelbaum <marcel@redhat.com>
2015-09-23 07:20:00 +03:00
|
|
|
.index = vhost_vq_index,
|
2012-12-24 19:37:01 +04:00
|
|
|
};
|
|
|
|
int r = event_notifier_init(&vq->masked_notifier, 0);
|
|
|
|
if (r < 0) {
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
file.fd = event_notifier_get_fd(&vq->masked_notifier);
|
2015-10-09 18:17:28 +03:00
|
|
|
r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
|
2012-12-24 19:37:01 +04:00
|
|
|
if (r) {
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_vring_call failed");
|
2012-12-24 19:37:01 +04:00
|
|
|
r = -errno;
|
|
|
|
goto fail_call;
|
|
|
|
}
|
2017-01-11 07:32:12 +03:00
|
|
|
|
|
|
|
vq->dev = dev;
|
|
|
|
|
2012-12-24 19:37:01 +04:00
|
|
|
return 0;
|
|
|
|
fail_call:
|
|
|
|
event_notifier_cleanup(&vq->masked_notifier);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq)
|
|
|
|
{
|
|
|
|
event_notifier_cleanup(&vq->masked_notifier);
|
|
|
|
}
|
|
|
|
|
2014-05-27 16:05:22 +04:00
|
|
|
int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
|
2021-06-09 18:46:52 +03:00
|
|
|
VhostBackendType backend_type, uint32_t busyloop_timeout,
|
|
|
|
Error **errp)
|
2010-03-17 14:08:17 +03:00
|
|
|
{
|
|
|
|
uint64_t features;
|
2016-07-27 00:15:04 +03:00
|
|
|
int i, r, n_initialized_vqs = 0;
|
2014-05-27 16:05:22 +04:00
|
|
|
|
2017-01-11 07:32:12 +03:00
|
|
|
hdev->vdev = NULL;
|
2015-10-09 18:17:27 +03:00
|
|
|
hdev->migration_blocker = NULL;
|
|
|
|
|
2016-07-27 00:14:58 +03:00
|
|
|
r = vhost_set_backend_type(hdev, backend_type);
|
|
|
|
assert(r >= 0);
|
2014-05-27 16:05:49 +04:00
|
|
|
|
2021-06-09 18:46:53 +03:00
|
|
|
r = hdev->vhost_ops->vhost_backend_init(hdev, opaque, errp);
|
2016-07-27 00:14:58 +03:00
|
|
|
if (r < 0) {
|
|
|
|
goto fail;
|
2014-05-27 16:05:35 +04:00
|
|
|
}
|
|
|
|
|
2015-10-09 18:17:28 +03:00
|
|
|
r = hdev->vhost_ops->vhost_set_owner(hdev);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r < 0) {
|
2021-06-09 18:46:54 +03:00
|
|
|
error_setg_errno(errp, -r, "vhost_set_owner failed");
|
2010-03-17 14:08:17 +03:00
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
2015-10-09 18:17:28 +03:00
|
|
|
r = hdev->vhost_ops->vhost_get_features(hdev, &features);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r < 0) {
|
2021-06-09 18:46:54 +03:00
|
|
|
error_setg_errno(errp, -r, "vhost_get_features failed");
|
2010-03-17 14:08:17 +03:00
|
|
|
goto fail;
|
|
|
|
}
|
2012-12-24 19:37:01 +04:00
|
|
|
|
2016-07-27 00:15:04 +03:00
|
|
|
for (i = 0; i < hdev->nvqs; ++i, ++n_initialized_vqs) {
|
vhost-user: add multiple queue support
This patch is initially based a patch from Nikolay Nikolaev.
This patch adds vhost-user multiple queue support, by creating a nc
and vhost_net pair for each queue.
Qemu exits if find that the backend can't support the number of requested
queues (by providing queues=# option). The max number is queried by a
new message, VHOST_USER_GET_QUEUE_NUM, and is sent only when protocol
feature VHOST_USER_PROTOCOL_F_MQ is present first.
The max queue check is done at vhost-user initiation stage. We initiate
one queue first, which, in the meantime, also gets the max_queues the
backend supports.
In older version, it was reported that some messages are sent more times
than necessary. Here we came an agreement with Michael that we could
categorize vhost user messages to 2 types: non-vring specific messages,
which should be sent only once, and vring specific messages, which should
be sent per queue.
Here I introduced a helper function vhost_user_one_time_request(), which
lists following messages as non-vring specific messages:
VHOST_USER_SET_OWNER
VHOST_USER_RESET_DEVICE
VHOST_USER_SET_MEM_TABLE
VHOST_USER_GET_QUEUE_NUM
For above messages, we simply ignore them when they are not sent the first
time.
Signed-off-by: Nikolay Nikolaev <n.nikolaev@virtualopensystems.com>
Signed-off-by: Changchun Ouyang <changchun.ouyang@intel.com>
Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Reviewed-by: Jason Wang <jasowang@redhat.com>
Tested-by: Marcel Apfelbaum <marcel@redhat.com>
2015-09-23 07:20:00 +03:00
|
|
|
r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i);
|
2012-12-24 19:37:01 +04:00
|
|
|
if (r < 0) {
|
2021-06-09 18:46:52 +03:00
|
|
|
error_setg_errno(errp, -r, "Failed to initialize virtqueue %d", i);
|
2016-07-27 00:15:04 +03:00
|
|
|
goto fail;
|
2012-12-24 19:37:01 +04:00
|
|
|
}
|
|
|
|
}
|
2016-07-06 04:57:55 +03:00
|
|
|
|
|
|
|
if (busyloop_timeout) {
|
|
|
|
for (i = 0; i < hdev->nvqs; ++i) {
|
|
|
|
r = vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i,
|
|
|
|
busyloop_timeout);
|
|
|
|
if (r < 0) {
|
2021-06-09 18:46:54 +03:00
|
|
|
error_setg_errno(errp, -r, "Failed to set busyloop timeout");
|
2016-07-06 04:57:55 +03:00
|
|
|
goto fail_busyloop;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
hdev->features = features;
|
|
|
|
|
2011-12-18 16:06:05 +04:00
|
|
|
hdev->memory_listener = (MemoryListener) {
|
2012-02-08 23:36:02 +04:00
|
|
|
.begin = vhost_begin,
|
|
|
|
.commit = vhost_commit,
|
2018-01-19 13:39:23 +03:00
|
|
|
.region_add = vhost_region_addnop,
|
|
|
|
.region_nop = vhost_region_addnop,
|
2011-12-18 16:06:05 +04:00
|
|
|
.log_start = vhost_log_start,
|
|
|
|
.log_stop = vhost_log_stop,
|
|
|
|
.log_sync = vhost_log_sync,
|
|
|
|
.log_global_start = vhost_log_global_start,
|
|
|
|
.log_global_stop = vhost_log_global_stop,
|
2012-02-08 18:39:06 +04:00
|
|
|
.eventfd_add = vhost_eventfd_add,
|
|
|
|
.eventfd_del = vhost_eventfd_del,
|
2012-02-08 17:05:50 +04:00
|
|
|
.priority = 10
|
2011-12-18 16:06:05 +04:00
|
|
|
};
|
2015-10-09 18:17:27 +03:00
|
|
|
|
2017-03-29 07:10:04 +03:00
|
|
|
hdev->iommu_listener = (MemoryListener) {
|
|
|
|
.region_add = vhost_iommu_region_add,
|
|
|
|
.region_del = vhost_iommu_region_del,
|
|
|
|
};
|
2017-01-11 07:32:12 +03:00
|
|
|
|
2015-10-09 18:17:27 +03:00
|
|
|
if (hdev->migration_blocker == NULL) {
|
|
|
|
if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) {
|
|
|
|
error_setg(&hdev->migration_blocker,
|
|
|
|
"Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
|
2018-03-28 15:18:04 +03:00
|
|
|
} else if (vhost_dev_log_is_shared(hdev) && !qemu_memfd_alloc_check()) {
|
2015-10-09 18:17:34 +03:00
|
|
|
error_setg(&hdev->migration_blocker,
|
|
|
|
"Migration disabled: failed to allocate shared memory");
|
2015-10-09 18:17:27 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (hdev->migration_blocker != NULL) {
|
2021-06-09 18:46:53 +03:00
|
|
|
r = migrate_add_blocker(hdev->migration_blocker, errp);
|
2021-07-20 15:54:01 +03:00
|
|
|
if (r < 0) {
|
2017-01-16 14:31:53 +03:00
|
|
|
error_free(hdev->migration_blocker);
|
|
|
|
goto fail_busyloop;
|
|
|
|
}
|
2014-06-18 18:20:42 +04:00
|
|
|
}
|
2015-10-09 18:17:27 +03:00
|
|
|
|
2011-08-21 07:09:37 +04:00
|
|
|
hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
|
2011-12-19 15:18:13 +04:00
|
|
|
hdev->n_mem_sections = 0;
|
|
|
|
hdev->mem_sections = NULL;
|
2010-03-17 14:08:17 +03:00
|
|
|
hdev->log = NULL;
|
|
|
|
hdev->log_size = 0;
|
|
|
|
hdev->log_enabled = false;
|
|
|
|
hdev->started = false;
|
2012-10-02 22:13:51 +04:00
|
|
|
memory_listener_register(&hdev->memory_listener, &address_space_memory);
|
2016-07-27 00:15:01 +03:00
|
|
|
QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
|
2018-02-27 10:10:04 +03:00
|
|
|
|
|
|
|
if (used_memslots > hdev->vhost_ops->vhost_backend_memslots_limit(hdev)) {
|
2021-06-09 18:46:52 +03:00
|
|
|
error_setg(errp, "vhost backend memory slots limit is less"
|
|
|
|
" than current number of present memory slots");
|
2021-06-09 18:46:54 +03:00
|
|
|
r = -EINVAL;
|
2021-02-22 14:49:31 +03:00
|
|
|
goto fail_busyloop;
|
2018-02-27 10:10:04 +03:00
|
|
|
}
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
return 0;
|
2016-07-27 00:15:04 +03:00
|
|
|
|
2016-07-06 04:57:55 +03:00
|
|
|
fail_busyloop:
|
2021-02-22 14:49:31 +03:00
|
|
|
if (busyloop_timeout) {
|
|
|
|
while (--i >= 0) {
|
|
|
|
vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 0);
|
|
|
|
}
|
2016-07-06 04:57:55 +03:00
|
|
|
}
|
2010-03-17 14:08:17 +03:00
|
|
|
fail:
|
2016-07-27 00:15:04 +03:00
|
|
|
hdev->nvqs = n_initialized_vqs;
|
|
|
|
vhost_dev_cleanup(hdev);
|
2010-03-17 14:08:17 +03:00
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
void vhost_dev_cleanup(struct vhost_dev *hdev)
|
|
|
|
{
|
2012-12-24 19:37:01 +04:00
|
|
|
int i;
|
2016-07-27 00:15:02 +03:00
|
|
|
|
2012-12-24 19:37:01 +04:00
|
|
|
for (i = 0; i < hdev->nvqs; ++i) {
|
|
|
|
vhost_virtqueue_cleanup(hdev->vqs + i);
|
|
|
|
}
|
2016-07-27 00:15:01 +03:00
|
|
|
if (hdev->mem) {
|
|
|
|
/* those are only safe after successful init */
|
|
|
|
memory_listener_unregister(&hdev->memory_listener);
|
|
|
|
QLIST_REMOVE(hdev, entry);
|
|
|
|
}
|
2014-06-18 18:20:42 +04:00
|
|
|
if (hdev->migration_blocker) {
|
|
|
|
migrate_del_blocker(hdev->migration_blocker);
|
|
|
|
error_free(hdev->migration_blocker);
|
|
|
|
}
|
2011-08-21 07:09:37 +04:00
|
|
|
g_free(hdev->mem);
|
2011-12-19 15:18:13 +04:00
|
|
|
g_free(hdev->mem_sections);
|
2016-07-27 00:15:02 +03:00
|
|
|
if (hdev->vhost_ops) {
|
|
|
|
hdev->vhost_ops->vhost_backend_cleanup(hdev);
|
|
|
|
}
|
2016-07-27 00:15:00 +03:00
|
|
|
assert(!hdev->log);
|
2016-07-27 00:15:02 +03:00
|
|
|
|
|
|
|
memset(hdev, 0, sizeof(struct vhost_dev));
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
|
|
|
|
2011-08-11 11:21:18 +04:00
|
|
|
/* Stop processing guest IO notifications in qemu.
|
|
|
|
* Start processing them in vhost in kernel.
|
|
|
|
*/
|
|
|
|
int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
|
|
|
|
{
|
2013-04-24 12:21:21 +04:00
|
|
|
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
|
2015-05-29 09:13:14 +03:00
|
|
|
int i, r, e;
|
2016-07-27 00:15:07 +03:00
|
|
|
|
2016-11-18 18:07:00 +03:00
|
|
|
/* We will pass the notifiers to the kernel, make sure that QEMU
|
|
|
|
* doesn't interfere.
|
|
|
|
*/
|
|
|
|
r = virtio_device_grab_ioeventfd(vdev);
|
|
|
|
if (r < 0) {
|
2016-07-27 00:15:07 +03:00
|
|
|
error_report("binding does not support host notifiers");
|
2011-08-11 11:21:18 +04:00
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < hdev->nvqs; ++i) {
|
2016-06-10 12:04:10 +03:00
|
|
|
r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
|
|
|
|
true);
|
2011-08-11 11:21:18 +04:00
|
|
|
if (r < 0) {
|
2016-07-27 00:15:07 +03:00
|
|
|
error_report("vhost VQ %d notifier binding failed: %d", i, -r);
|
2011-08-11 11:21:18 +04:00
|
|
|
goto fail_vq;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
fail_vq:
|
|
|
|
while (--i >= 0) {
|
2016-06-10 12:04:10 +03:00
|
|
|
e = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
|
|
|
|
false);
|
2015-05-29 09:13:14 +03:00
|
|
|
if (e < 0) {
|
2016-07-27 00:15:07 +03:00
|
|
|
error_report("vhost VQ %d notifier cleanup error: %d", i, -r);
|
2011-08-11 11:21:18 +04:00
|
|
|
}
|
2015-05-29 09:13:14 +03:00
|
|
|
assert (e >= 0);
|
2018-01-29 17:20:56 +03:00
|
|
|
virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i);
|
2011-08-11 11:21:18 +04:00
|
|
|
}
|
2016-11-18 18:07:00 +03:00
|
|
|
virtio_device_release_ioeventfd(vdev);
|
2011-08-11 11:21:18 +04:00
|
|
|
fail:
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Stop processing guest IO notifications in vhost.
|
|
|
|
* Start processing them in qemu.
|
|
|
|
* This might actually run the qemu handlers right away,
|
|
|
|
* so virtio in qemu must be completely setup when this is called.
|
|
|
|
*/
|
|
|
|
void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
|
|
|
|
{
|
2013-04-24 12:21:21 +04:00
|
|
|
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
|
2011-08-11 11:21:18 +04:00
|
|
|
int i, r;
|
|
|
|
|
|
|
|
for (i = 0; i < hdev->nvqs; ++i) {
|
2016-06-10 12:04:10 +03:00
|
|
|
r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
|
|
|
|
false);
|
2011-08-11 11:21:18 +04:00
|
|
|
if (r < 0) {
|
2016-07-27 00:15:07 +03:00
|
|
|
error_report("vhost VQ %d notifier cleanup failed: %d", i, -r);
|
2011-08-11 11:21:18 +04:00
|
|
|
}
|
|
|
|
assert (r >= 0);
|
2018-01-29 17:20:56 +03:00
|
|
|
virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i);
|
2011-08-11 11:21:18 +04:00
|
|
|
}
|
2016-11-18 18:07:00 +03:00
|
|
|
virtio_device_release_ioeventfd(vdev);
|
2011-08-11 11:21:18 +04:00
|
|
|
}
|
|
|
|
|
2012-12-24 19:37:01 +04:00
|
|
|
/* Test and clear event pending status.
|
|
|
|
* Should be called after unmask to avoid losing events.
|
|
|
|
*/
|
|
|
|
bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n)
|
|
|
|
{
|
2013-01-30 15:12:35 +04:00
|
|
|
struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index;
|
|
|
|
assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
|
2012-12-24 19:37:01 +04:00
|
|
|
return event_notifier_test_and_clear(&vq->masked_notifier);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Mask/unmask events from this vq. */
|
|
|
|
void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
|
|
|
|
bool mask)
|
|
|
|
{
|
|
|
|
struct VirtQueue *vvq = virtio_get_queue(vdev, n);
|
2013-01-30 15:12:35 +04:00
|
|
|
int r, index = n - hdev->vq_index;
|
2015-09-23 07:19:59 +03:00
|
|
|
struct vhost_vring_file file;
|
2012-12-24 19:37:01 +04:00
|
|
|
|
2016-07-27 00:15:16 +03:00
|
|
|
/* should only be called after backend is connected */
|
|
|
|
assert(hdev->vhost_ops);
|
|
|
|
|
2012-12-24 19:37:01 +04:00
|
|
|
if (mask) {
|
2016-02-18 17:12:23 +03:00
|
|
|
assert(vdev->use_guest_notifier_mask);
|
2013-01-30 15:12:35 +04:00
|
|
|
file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
|
2012-12-24 19:37:01 +04:00
|
|
|
} else {
|
|
|
|
file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
|
|
|
|
}
|
2015-09-23 07:19:59 +03:00
|
|
|
|
2015-10-09 18:17:28 +03:00
|
|
|
file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n);
|
|
|
|
r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file);
|
2016-07-27 00:15:05 +03:00
|
|
|
if (r < 0) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_set_vring_call failed");
|
|
|
|
}
|
2012-12-24 19:37:01 +04:00
|
|
|
}
|
|
|
|
|
2015-06-04 13:34:20 +03:00
|
|
|
uint64_t vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
|
|
|
|
uint64_t features)
|
2014-05-27 16:04:42 +04:00
|
|
|
{
|
|
|
|
const int *bit = feature_bits;
|
|
|
|
while (*bit != VHOST_INVALID_FEATURE_BIT) {
|
2015-06-04 13:34:20 +03:00
|
|
|
uint64_t bit_mask = (1ULL << *bit);
|
2014-05-27 16:04:42 +04:00
|
|
|
if (!(hdev->features & bit_mask)) {
|
|
|
|
features &= ~bit_mask;
|
|
|
|
}
|
|
|
|
bit++;
|
|
|
|
}
|
|
|
|
return features;
|
|
|
|
}
|
|
|
|
|
|
|
|
void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
|
2015-06-04 13:34:20 +03:00
|
|
|
uint64_t features)
|
2014-05-27 16:04:42 +04:00
|
|
|
{
|
|
|
|
const int *bit = feature_bits;
|
|
|
|
while (*bit != VHOST_INVALID_FEATURE_BIT) {
|
2015-06-04 13:34:20 +03:00
|
|
|
uint64_t bit_mask = (1ULL << *bit);
|
2014-05-27 16:04:42 +04:00
|
|
|
if (features & bit_mask) {
|
|
|
|
hdev->acked_features |= bit_mask;
|
|
|
|
}
|
|
|
|
bit++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-04 04:53:31 +03:00
|
|
|
int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config,
|
2021-06-09 18:46:56 +03:00
|
|
|
uint32_t config_len, Error **errp)
|
2018-01-04 04:53:31 +03:00
|
|
|
{
|
|
|
|
assert(hdev->vhost_ops);
|
|
|
|
|
|
|
|
if (hdev->vhost_ops->vhost_get_config) {
|
2021-07-20 15:54:04 +03:00
|
|
|
return hdev->vhost_ops->vhost_get_config(hdev, config, config_len,
|
|
|
|
errp);
|
2018-01-04 04:53:31 +03:00
|
|
|
}
|
|
|
|
|
2021-06-09 18:46:56 +03:00
|
|
|
error_setg(errp, "vhost_get_config not implemented");
|
|
|
|
return -ENOTSUP;
|
2018-01-04 04:53:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data,
|
|
|
|
uint32_t offset, uint32_t size, uint32_t flags)
|
|
|
|
{
|
|
|
|
assert(hdev->vhost_ops);
|
|
|
|
|
|
|
|
if (hdev->vhost_ops->vhost_set_config) {
|
|
|
|
return hdev->vhost_ops->vhost_set_config(hdev, data, offset,
|
|
|
|
size, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
void vhost_dev_set_config_notifier(struct vhost_dev *hdev,
|
|
|
|
const VhostDevConfigOps *ops)
|
|
|
|
{
|
|
|
|
hdev->config_ops = ops;
|
|
|
|
}
|
|
|
|
|
2019-02-28 11:53:49 +03:00
|
|
|
void vhost_dev_free_inflight(struct vhost_inflight *inflight)
|
|
|
|
{
|
2020-04-17 13:17:07 +03:00
|
|
|
if (inflight && inflight->addr) {
|
2019-02-28 11:53:49 +03:00
|
|
|
qemu_memfd_free(inflight->addr, inflight->size, inflight->fd);
|
|
|
|
inflight->addr = NULL;
|
|
|
|
inflight->fd = -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int vhost_dev_resize_inflight(struct vhost_inflight *inflight,
|
|
|
|
uint64_t new_size)
|
|
|
|
{
|
|
|
|
Error *err = NULL;
|
|
|
|
int fd = -1;
|
|
|
|
void *addr = qemu_memfd_alloc("vhost-inflight", new_size,
|
|
|
|
F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
|
|
|
|
&fd, &err);
|
|
|
|
|
|
|
|
if (err) {
|
|
|
|
error_report_err(err);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
vhost_dev_free_inflight(inflight);
|
|
|
|
inflight->offset = 0;
|
|
|
|
inflight->addr = addr;
|
|
|
|
inflight->fd = fd;
|
|
|
|
inflight->size = new_size;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void vhost_dev_save_inflight(struct vhost_inflight *inflight, QEMUFile *f)
|
|
|
|
{
|
|
|
|
if (inflight->addr) {
|
|
|
|
qemu_put_be64(f, inflight->size);
|
|
|
|
qemu_put_be16(f, inflight->queue_size);
|
|
|
|
qemu_put_buffer(f, inflight->addr, inflight->size);
|
|
|
|
} else {
|
|
|
|
qemu_put_be64(f, 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int vhost_dev_load_inflight(struct vhost_inflight *inflight, QEMUFile *f)
|
|
|
|
{
|
|
|
|
uint64_t size;
|
|
|
|
|
|
|
|
size = qemu_get_be64(f);
|
|
|
|
if (!size) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (inflight->size != size) {
|
|
|
|
if (vhost_dev_resize_inflight(inflight, size)) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
inflight->queue_size = qemu_get_be16(f);
|
|
|
|
|
|
|
|
qemu_get_buffer(f, inflight->addr, size);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-11-03 15:36:17 +03:00
|
|
|
int vhost_dev_prepare_inflight(struct vhost_dev *hdev, VirtIODevice *vdev)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
|
|
|
|
if (hdev->vhost_ops->vhost_get_inflight_fd == NULL ||
|
|
|
|
hdev->vhost_ops->vhost_set_inflight_fd == NULL) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
hdev->vdev = vdev;
|
|
|
|
|
|
|
|
r = vhost_dev_set_features(hdev, hdev->log_enabled);
|
|
|
|
if (r < 0) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_dev_prepare_inflight failed");
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-02-28 11:53:49 +03:00
|
|
|
int vhost_dev_set_inflight(struct vhost_dev *dev,
|
|
|
|
struct vhost_inflight *inflight)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
|
|
|
|
if (dev->vhost_ops->vhost_set_inflight_fd && inflight->addr) {
|
|
|
|
r = dev->vhost_ops->vhost_set_inflight_fd(dev, inflight);
|
|
|
|
if (r) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_set_inflight_fd failed");
|
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size,
|
|
|
|
struct vhost_inflight *inflight)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
|
|
|
|
if (dev->vhost_ops->vhost_get_inflight_fd) {
|
|
|
|
r = dev->vhost_ops->vhost_get_inflight_fd(dev, queue_size, inflight);
|
|
|
|
if (r) {
|
|
|
|
VHOST_OPS_DEBUG("vhost_get_inflight_fd failed");
|
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-08-11 11:21:18 +04:00
|
|
|
/* Host notifiers must be enabled at this point. */
|
2010-03-17 14:08:17 +03:00
|
|
|
int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
|
|
|
|
{
|
|
|
|
int i, r;
|
2012-12-25 19:41:07 +04:00
|
|
|
|
2016-07-27 00:15:16 +03:00
|
|
|
/* should only be called after backend is connected */
|
|
|
|
assert(hdev->vhost_ops);
|
|
|
|
|
2012-12-25 19:41:07 +04:00
|
|
|
hdev->started = true;
|
2017-01-11 07:32:12 +03:00
|
|
|
hdev->vdev = vdev;
|
2012-12-25 19:41:07 +04:00
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
r = vhost_dev_set_features(hdev, hdev->log_enabled);
|
|
|
|
if (r < 0) {
|
2010-10-06 17:20:17 +04:00
|
|
|
goto fail_features;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2017-01-11 07:32:12 +03:00
|
|
|
|
|
|
|
if (vhost_dev_has_iommu(hdev)) {
|
2017-03-29 07:10:04 +03:00
|
|
|
memory_listener_register(&hdev->iommu_listener, vdev->dma_as);
|
2017-01-11 07:32:12 +03:00
|
|
|
}
|
|
|
|
|
2015-10-09 18:17:28 +03:00
|
|
|
r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r < 0) {
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_mem_table failed");
|
2010-03-17 14:08:17 +03:00
|
|
|
r = -errno;
|
2010-10-06 17:20:17 +04:00
|
|
|
goto fail_mem;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2010-07-16 18:11:46 +04:00
|
|
|
for (i = 0; i < hdev->nvqs; ++i) {
|
2012-12-24 19:37:01 +04:00
|
|
|
r = vhost_virtqueue_start(hdev,
|
2013-01-30 15:12:35 +04:00
|
|
|
vdev,
|
|
|
|
hdev->vqs + i,
|
|
|
|
hdev->vq_index + i);
|
2010-07-16 18:11:46 +04:00
|
|
|
if (r < 0) {
|
|
|
|
goto fail_vq;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
if (hdev->log_enabled) {
|
2015-04-17 18:13:24 +03:00
|
|
|
uint64_t log_base;
|
|
|
|
|
2010-03-17 14:08:17 +03:00
|
|
|
hdev->log_size = vhost_get_log_size(hdev);
|
2015-10-09 18:17:25 +03:00
|
|
|
hdev->log = vhost_log_get(hdev->log_size,
|
|
|
|
vhost_dev_log_is_shared(hdev));
|
2015-06-04 12:28:46 +03:00
|
|
|
log_base = (uintptr_t)hdev->log->log;
|
2015-10-09 18:17:23 +03:00
|
|
|
r = hdev->vhost_ops->vhost_set_log_base(hdev,
|
2015-10-09 18:17:26 +03:00
|
|
|
hdev->log_size ? log_base : 0,
|
|
|
|
hdev->log);
|
2010-03-17 14:08:17 +03:00
|
|
|
if (r < 0) {
|
2016-07-27 00:15:06 +03:00
|
|
|
VHOST_OPS_DEBUG("vhost_set_log_base failed");
|
2010-03-17 14:08:17 +03:00
|
|
|
r = -errno;
|
2010-10-06 17:20:17 +04:00
|
|
|
goto fail_log;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
|
|
|
}
|
2020-07-01 17:55:31 +03:00
|
|
|
if (hdev->vhost_ops->vhost_dev_start) {
|
|
|
|
r = hdev->vhost_ops->vhost_dev_start(hdev, true);
|
|
|
|
if (r) {
|
|
|
|
goto fail_log;
|
|
|
|
}
|
|
|
|
}
|
2020-07-01 17:55:29 +03:00
|
|
|
if (vhost_dev_has_iommu(hdev) &&
|
|
|
|
hdev->vhost_ops->vhost_set_iotlb_callback) {
|
|
|
|
hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true);
|
2017-01-11 07:32:12 +03:00
|
|
|
|
|
|
|
/* Update used ring information for IOTLB to work correctly,
|
|
|
|
* vhost-kernel code requires for this.*/
|
|
|
|
for (i = 0; i < hdev->nvqs; ++i) {
|
|
|
|
struct vhost_virtqueue *vq = hdev->vqs + i;
|
|
|
|
vhost_device_iotlb_miss(hdev, vq->used_phys, true);
|
|
|
|
}
|
|
|
|
}
|
2010-03-17 14:08:17 +03:00
|
|
|
return 0;
|
2010-10-06 17:20:17 +04:00
|
|
|
fail_log:
|
2015-06-05 06:05:58 +03:00
|
|
|
vhost_log_put(hdev, false);
|
2010-03-17 14:08:17 +03:00
|
|
|
fail_vq:
|
|
|
|
while (--i >= 0) {
|
2012-12-24 19:37:01 +04:00
|
|
|
vhost_virtqueue_stop(hdev,
|
2013-01-30 15:12:35 +04:00
|
|
|
vdev,
|
|
|
|
hdev->vqs + i,
|
|
|
|
hdev->vq_index + i);
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2017-01-11 07:32:12 +03:00
|
|
|
|
2010-10-06 17:20:17 +04:00
|
|
|
fail_mem:
|
|
|
|
fail_features:
|
2012-12-25 19:41:07 +04:00
|
|
|
|
|
|
|
hdev->started = false;
|
2010-03-17 14:08:17 +03:00
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2011-08-11 11:21:18 +04:00
|
|
|
/* Host notifiers must be enabled at this point. */
|
2010-03-17 14:08:17 +03:00
|
|
|
void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
|
|
|
|
{
|
2013-01-30 15:12:35 +04:00
|
|
|
int i;
|
2010-10-06 17:20:17 +04:00
|
|
|
|
2016-07-27 00:15:16 +03:00
|
|
|
/* should only be called after backend is connected */
|
|
|
|
assert(hdev->vhost_ops);
|
|
|
|
|
2020-07-01 17:55:31 +03:00
|
|
|
if (hdev->vhost_ops->vhost_dev_start) {
|
|
|
|
hdev->vhost_ops->vhost_dev_start(hdev, false);
|
|
|
|
}
|
2010-03-17 14:08:17 +03:00
|
|
|
for (i = 0; i < hdev->nvqs; ++i) {
|
2012-12-24 19:37:01 +04:00
|
|
|
vhost_virtqueue_stop(hdev,
|
2013-01-30 15:12:35 +04:00
|
|
|
vdev,
|
|
|
|
hdev->vqs + i,
|
|
|
|
hdev->vq_index + i);
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2010-10-06 17:20:17 +04:00
|
|
|
|
2017-01-11 07:32:12 +03:00
|
|
|
if (vhost_dev_has_iommu(hdev)) {
|
2020-07-01 17:55:29 +03:00
|
|
|
if (hdev->vhost_ops->vhost_set_iotlb_callback) {
|
|
|
|
hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
|
|
|
|
}
|
2017-03-29 07:10:04 +03:00
|
|
|
memory_listener_unregister(&hdev->iommu_listener);
|
2017-01-11 07:32:12 +03:00
|
|
|
}
|
2015-06-04 12:28:46 +03:00
|
|
|
vhost_log_put(hdev, true);
|
2010-03-17 14:08:17 +03:00
|
|
|
hdev->started = false;
|
2017-01-11 07:32:12 +03:00
|
|
|
hdev->vdev = NULL;
|
2010-03-17 14:08:17 +03:00
|
|
|
}
|
2016-07-27 00:15:25 +03:00
|
|
|
|
|
|
|
int vhost_net_set_backend(struct vhost_dev *hdev,
|
|
|
|
struct vhost_vring_file *file)
|
|
|
|
{
|
|
|
|
if (hdev->vhost_ops->vhost_net_set_backend) {
|
|
|
|
return hdev->vhost_ops->vhost_net_set_backend(hdev, file);
|
|
|
|
}
|
|
|
|
|
|
|
|
return -1;
|
|
|
|
}
|