mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pull request

Browse Source 
-----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEEhpWov9P5fNqsNXdanKSrs4Grc8gFAmHfDFIACgkQnKSrs4Gr
 c8hoMwf/QPaU1svRdP9pPiMkJiwmtmgacEKEfrF3I8w8aOOf3dLPyUKafuStJtfZ
 Fhl2631jHL7JKKQKGomJhdzQovHAPsPEC8YFxesB1LvO0LIX4UtYplkxkj27In2D
 9w+cIMVMTkFyIv/5GgTaFBbnmk2at4tqXkcGmcblp0qZCMsElJvGWOkToM+Fjot4
 A4jYUCviqQqdt4j558UjIdecdaWy+5Cnej3NsKwH5V62o2uZY1+7vu0cf0ARcja1
 kptZBbvMIfjyl1TeuJWuEya8aWo0KwIbbs3tVKz16Na7RXlG01mYCwGLAVkBADCD
 mJaM1jZVADtUZyoCkh4M4KBBwFnFCw==
 =ITwP
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/stefanha-gitlab/tags/block-pull-request' into staging

Pull request

# gpg: Signature made Wed 12 Jan 2022 17:13:54 GMT
# gpg:                using RSA key 8695A8BFD3F97CDAAC35775A9CA4ABB381AB73C8
# gpg: Good signature from "Stefan Hajnoczi <stefanha@redhat.com>" [full]
# gpg:                 aka "Stefan Hajnoczi <stefanha@gmail.com>" [full]
# Primary key fingerprint: 8695 A8BF D3F9 7CDA AC35  775A 9CA4 ABB3 81AB 73C8

* remotes/stefanha-gitlab/tags/block-pull-request:
  virtio: unify dataplane and non-dataplane ->handle_output()
  virtio: use ->handle_output() instead of ->handle_aio_output()
  virtio-scsi: prepare virtio_scsi_handle_cmd for dataplane
  virtio-blk: drop unused virtio_blk_handle_vq() return value
  virtio: get rid of VirtIOHandleAIOOutput
  aio-posix: split poll check from ready handler

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2022-01-14 10:43:32 +00:00
commit 1001c9d9c0
31 changed files with 221 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
block/curl.c
View File

@ -125,7 +125,7 @@ static gboolean curl_drop_socket(void *key, void *value, void *opaque)
BDRVCURLState *s = socket->s; BDRVCURLState *s = socket->s;
aio_set_fd_handler(s->aio_context, socket->fd, false, aio_set_fd_handler(s->aio_context, socket->fd, false,
NULL, NULL, NULL, NULL); NULL, NULL, NULL, NULL, NULL);
return true; return true;
} }
@ -173,19 +173,20 @@ static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
switch (action) { switch (action) {
case CURL_POLL_IN: case CURL_POLL_IN:
aio_set_fd_handler(s->aio_context, fd, false, aio_set_fd_handler(s->aio_context, fd, false,
curl_multi_do, NULL, NULL, socket); curl_multi_do, NULL, NULL, NULL, socket);
break; break;
case CURL_POLL_OUT: case CURL_POLL_OUT:
aio_set_fd_handler(s->aio_context, fd, false, aio_set_fd_handler(s->aio_context, fd, false,
NULL, curl_multi_do, NULL, socket); NULL, curl_multi_do, NULL, NULL, socket);
break; break;
case CURL_POLL_INOUT: case CURL_POLL_INOUT:
aio_set_fd_handler(s->aio_context, fd, false, aio_set_fd_handler(s->aio_context, fd, false,
curl_multi_do, curl_multi_do, NULL, socket); curl_multi_do, curl_multi_do,
NULL, NULL, socket);
break; break;
case CURL_POLL_REMOVE: case CURL_POLL_REMOVE:
aio_set_fd_handler(s->aio_context, fd, false, aio_set_fd_handler(s->aio_context, fd, false,
NULL, NULL, NULL, NULL); NULL, NULL, NULL, NULL, NULL);
break; break;
} }

4
block/export/fuse.c
View File

@ -223,7 +223,7 @@ static int setup_fuse_export(FuseExport *exp, const char *mountpoint,
aio_set_fd_handler(exp->common.ctx, aio_set_fd_handler(exp->common.ctx,
fuse_session_fd(exp->fuse_session), true, fuse_session_fd(exp->fuse_session), true,
read_from_fuse_export, NULL, NULL, exp); read_from_fuse_export, NULL, NULL, NULL, exp);
exp->fd_handler_set_up = true; exp->fd_handler_set_up = true;
return 0; return 0;
@ -267,7 +267,7 @@ static void fuse_export_shutdown(BlockExport *blk_exp)
if (exp->fd_handler_set_up) { if (exp->fd_handler_set_up) {
aio_set_fd_handler(exp->common.ctx, aio_set_fd_handler(exp->common.ctx,
fuse_session_fd(exp->fuse_session), true, fuse_session_fd(exp->fuse_session), true,
NULL, NULL, NULL, NULL); NULL, NULL, NULL, NULL, NULL);
exp->fd_handler_set_up = false; exp->fd_handler_set_up = false;
} }
} }

19
block/io_uring.c
View File

@ -292,12 +292,14 @@ static bool qemu_luring_poll_cb(void *opaque)
{ {
LuringState *s = opaque; LuringState *s = opaque;
if (io_uring_cq_ready(&s->ring)) { return io_uring_cq_ready(&s->ring);
luring_process_completions_and_submit(s); }
return true;
}
return false; static void qemu_luring_poll_ready(void *opaque)
{
LuringState *s = opaque;
luring_process_completions_and_submit(s);
} }
static void ioq_init(LuringQueue *io_q) static void ioq_init(LuringQueue *io_q)
@ -402,8 +404,8 @@ int coroutine_fn luring_co_submit(BlockDriverState *bs, LuringState *s, int fd,
void luring_detach_aio_context(LuringState *s, AioContext *old_context) void luring_detach_aio_context(LuringState *s, AioContext *old_context)
{ {
aio_set_fd_handler(old_context, s->ring.ring_fd, false, NULL, NULL, NULL, aio_set_fd_handler(old_context, s->ring.ring_fd, false,
s); NULL, NULL, NULL, NULL, s);
qemu_bh_delete(s->completion_bh); qemu_bh_delete(s->completion_bh);
s->aio_context = NULL; s->aio_context = NULL;
} }
@ -413,7 +415,8 @@ void luring_attach_aio_context(LuringState *s, AioContext *new_context)
s->aio_context = new_context; s->aio_context = new_context;
s->completion_bh = aio_bh_new(new_context, qemu_luring_completion_bh, s); s->completion_bh = aio_bh_new(new_context, qemu_luring_completion_bh, s);
aio_set_fd_handler(s->aio_context, s->ring.ring_fd, false, aio_set_fd_handler(s->aio_context, s->ring.ring_fd, false,
qemu_luring_completion_cb, NULL, qemu_luring_poll_cb, s); qemu_luring_completion_cb, NULL,
qemu_luring_poll_cb, qemu_luring_poll_ready, s);
} }
LuringState *luring_init(Error **errp) LuringState *luring_init(Error **errp)

4
block/iscsi.c
View File

@ -363,7 +363,7 @@ iscsi_set_events(IscsiLun *iscsilun)
false, false,
(ev & POLLIN) ? iscsi_process_read : NULL, (ev & POLLIN) ? iscsi_process_read : NULL,
(ev & POLLOUT) ? iscsi_process_write : NULL, (ev & POLLOUT) ? iscsi_process_write : NULL,
NULL, NULL, NULL,
iscsilun); iscsilun);
iscsilun->events = ev; iscsilun->events = ev;
} }
@ -1534,7 +1534,7 @@ static void iscsi_detach_aio_context(BlockDriverState *bs)
IscsiLun *iscsilun = bs->opaque; IscsiLun *iscsilun = bs->opaque;
aio_set_fd_handler(iscsilun->aio_context, iscsi_get_fd(iscsilun->iscsi), aio_set_fd_handler(iscsilun->aio_context, iscsi_get_fd(iscsilun->iscsi),
false, NULL, NULL, NULL, NULL); false, NULL, NULL, NULL, NULL, NULL);
iscsilun->events = 0; iscsilun->events = 0;
if (iscsilun->nop_timer) { if (iscsilun->nop_timer) {

16
block/linux-aio.c
View File

@ -263,12 +263,15 @@ static bool qemu_laio_poll_cb(void *opaque)
LinuxAioState *s = container_of(e, LinuxAioState, e); LinuxAioState *s = container_of(e, LinuxAioState, e);
struct io_event *events; struct io_event *events;
if (!io_getevents_peek(s->ctx, &events)) { return io_getevents_peek(s->ctx, &events);
return false; }
}
static void qemu_laio_poll_ready(EventNotifier *opaque)
{
EventNotifier *e = opaque;
LinuxAioState *s = container_of(e, LinuxAioState, e);
qemu_laio_process_completions_and_submit(s); qemu_laio_process_completions_and_submit(s);
return true;
} }
static void ioq_init(LaioQueue *io_q) static void ioq_init(LaioQueue *io_q)
@ -427,7 +430,7 @@ int coroutine_fn laio_co_submit(BlockDriverState *bs, LinuxAioState *s, int fd,
void laio_detach_aio_context(LinuxAioState *s, AioContext *old_context) void laio_detach_aio_context(LinuxAioState *s, AioContext *old_context)
{ {
aio_set_event_notifier(old_context, &s->e, false, NULL, NULL); aio_set_event_notifier(old_context, &s->e, false, NULL, NULL, NULL);
qemu_bh_delete(s->completion_bh); qemu_bh_delete(s->completion_bh);
s->aio_context = NULL; s->aio_context = NULL;
} }
@ -438,7 +441,8 @@ void laio_attach_aio_context(LinuxAioState *s, AioContext *new_context)
s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s); s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s);
aio_set_event_notifier(new_context, &s->e, false, aio_set_event_notifier(new_context, &s->e, false,
qemu_laio_completion_cb, qemu_laio_completion_cb,
qemu_laio_poll_cb); qemu_laio_poll_cb,
qemu_laio_poll_ready);
} }
LinuxAioState *laio_init(Error **errp) LinuxAioState *laio_init(Error **errp)

6
block/nfs.c
View File

@ -197,7 +197,7 @@ static void nfs_set_events(NFSClient *client)
false, false,
(ev & POLLIN) ? nfs_process_read : NULL, (ev & POLLIN) ? nfs_process_read : NULL,
(ev & POLLOUT) ? nfs_process_write : NULL, (ev & POLLOUT) ? nfs_process_write : NULL,
NULL, client); NULL, NULL, client);
} }
client->events = ev; client->events = ev;
@ -372,7 +372,7 @@ static void nfs_detach_aio_context(BlockDriverState *bs)
NFSClient *client = bs->opaque; NFSClient *client = bs->opaque;
aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context), aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
false, NULL, NULL, NULL, NULL); false, NULL, NULL, NULL, NULL, NULL);
client->events = 0; client->events = 0;
} }
@ -390,7 +390,7 @@ static void nfs_client_close(NFSClient *client)
if (client->context) { if (client->context) {
qemu_mutex_lock(&client->mutex); qemu_mutex_lock(&client->mutex);
aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context), aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
false, NULL, NULL, NULL, NULL); false, NULL, NULL, NULL, NULL, NULL);
qemu_mutex_unlock(&client->mutex); qemu_mutex_unlock(&client->mutex);
if (client->fh) { if (client->fh) {
nfs_close(client->context, client->fh); nfs_close(client->context, client->fh);

51
block/nvme.c
View File

@ -605,10 +605,8 @@ out:
return ret; return ret;
} }
static bool nvme_poll_queue(NVMeQueuePair *q) static void nvme_poll_queue(NVMeQueuePair *q)
{ {
bool progress = false;
const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES; const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES;
NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset]; NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset];
@ -619,30 +617,23 @@ static bool nvme_poll_queue(NVMeQueuePair *q)
* cannot race with itself. * cannot race with itself.
*/ */
if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) { if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) {
return false; return;
} }
qemu_mutex_lock(&q->lock); qemu_mutex_lock(&q->lock);
while (nvme_process_completion(q)) { while (nvme_process_completion(q)) {
/* Keep polling */ /* Keep polling */
progress = true;
} }
qemu_mutex_unlock(&q->lock); qemu_mutex_unlock(&q->lock);
return progress;
} }
static bool nvme_poll_queues(BDRVNVMeState *s) static void nvme_poll_queues(BDRVNVMeState *s)
{ {
bool progress = false;
int i; int i;
for (i = 0; i < s->queue_count; i++) { for (i = 0; i < s->queue_count; i++) {
if (nvme_poll_queue(s->queues[i])) { nvme_poll_queue(s->queues[i]);
progress = true;
} }
}
return progress;
} }
static void nvme_handle_event(EventNotifier *n) static void nvme_handle_event(EventNotifier *n)
@ -703,8 +694,30 @@ static bool nvme_poll_cb(void *opaque)
EventNotifier *e = opaque; EventNotifier *e = opaque;
BDRVNVMeState *s = container_of(e, BDRVNVMeState, BDRVNVMeState *s = container_of(e, BDRVNVMeState,
irq_notifier[MSIX_SHARED_IRQ_IDX]); irq_notifier[MSIX_SHARED_IRQ_IDX]);
int i;
return nvme_poll_queues(s); for (i = 0; i < s->queue_count; i++) {
NVMeQueuePair *q = s->queues[i];
const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES;
NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset];
/*
* q->lock isn't needed because nvme_process_completion() only runs in
* the event loop thread and cannot race with itself.
*/
if ((le16_to_cpu(cqe->status) & 0x1) != q->cq_phase) {
return true;
}
}
return false;
}
static void nvme_poll_ready(EventNotifier *e)
{
BDRVNVMeState *s = container_of(e, BDRVNVMeState,
irq_notifier[MSIX_SHARED_IRQ_IDX]);
nvme_poll_queues(s);
} }
static int nvme_init(BlockDriverState *bs, const char *device, int namespace, static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
@ -839,7 +852,8 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
} }
aio_set_event_notifier(bdrv_get_aio_context(bs), aio_set_event_notifier(bdrv_get_aio_context(bs),
&s->irq_notifier[MSIX_SHARED_IRQ_IDX], &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
false, nvme_handle_event, nvme_poll_cb); false, nvme_handle_event, nvme_poll_cb,
nvme_poll_ready);
if (!nvme_identify(bs, namespace, errp)) { if (!nvme_identify(bs, namespace, errp)) {
ret = -EIO; ret = -EIO;
@ -924,7 +938,7 @@ static void nvme_close(BlockDriverState *bs)
g_free(s->queues); g_free(s->queues);
aio_set_event_notifier(bdrv_get_aio_context(bs), aio_set_event_notifier(bdrv_get_aio_context(bs),
&s->irq_notifier[MSIX_SHARED_IRQ_IDX], &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
false, NULL, NULL); false, NULL, NULL, NULL);
event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]); event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]);
qemu_vfio_pci_unmap_bar(s->vfio, 0, s->bar0_wo_map, qemu_vfio_pci_unmap_bar(s->vfio, 0, s->bar0_wo_map,
0, sizeof(NvmeBar) + NVME_DOORBELL_SIZE); 0, sizeof(NvmeBar) + NVME_DOORBELL_SIZE);
@ -1520,7 +1534,7 @@ static void nvme_detach_aio_context(BlockDriverState *bs)
aio_set_event_notifier(bdrv_get_aio_context(bs), aio_set_event_notifier(bdrv_get_aio_context(bs),
&s->irq_notifier[MSIX_SHARED_IRQ_IDX], &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
false, NULL, NULL); false, NULL, NULL, NULL);
} }
static void nvme_attach_aio_context(BlockDriverState *bs, static void nvme_attach_aio_context(BlockDriverState *bs,
@ -1530,7 +1544,8 @@ static void nvme_attach_aio_context(BlockDriverState *bs,
s->aio_context = new_context; s->aio_context = new_context;
aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX], aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
false, nvme_handle_event, nvme_poll_cb); false, nvme_handle_event, nvme_poll_cb,
nvme_poll_ready);
for (unsigned i = 0; i < s->queue_count; i++) { for (unsigned i = 0; i < s->queue_count; i++) {
NVMeQueuePair *q = s->queues[i]; NVMeQueuePair *q = s->queues[i];

4
block/ssh.c
View File

@ -990,7 +990,7 @@ static void restart_coroutine(void *opaque)
AioContext *ctx = bdrv_get_aio_context(bs); AioContext *ctx = bdrv_get_aio_context(bs);
trace_ssh_restart_coroutine(restart->co); trace_ssh_restart_coroutine(restart->co);
aio_set_fd_handler(ctx, s->sock, false, NULL, NULL, NULL, NULL); aio_set_fd_handler(ctx, s->sock, false, NULL, NULL, NULL, NULL, NULL);
aio_co_wake(restart->co); aio_co_wake(restart->co);
} }
@ -1020,7 +1020,7 @@ static coroutine_fn void co_yield(BDRVSSHState *s, BlockDriverState *bs)
trace_ssh_co_yield(s->sock, rd_handler, wr_handler); trace_ssh_co_yield(s->sock, rd_handler, wr_handler);
aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock, aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock,
false, rd_handler, wr_handler, NULL, &restart); false, rd_handler, wr_handler, NULL, NULL, &restart);
qemu_coroutine_yield(); qemu_coroutine_yield();
trace_ssh_co_yield_back(s->sock); trace_ssh_co_yield_back(s->sock);
} }

4
block/win32-aio.c
View File

@ -172,7 +172,7 @@ int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile)
void win32_aio_detach_aio_context(QEMUWin32AIOState *aio, void win32_aio_detach_aio_context(QEMUWin32AIOState *aio,
AioContext *old_context) AioContext *old_context)
{ {
aio_set_event_notifier(old_context, &aio->e, false, NULL, NULL); aio_set_event_notifier(old_context, &aio->e, false, NULL, NULL, NULL);
aio->aio_ctx = NULL; aio->aio_ctx = NULL;
} }
@ -181,7 +181,7 @@ void win32_aio_attach_aio_context(QEMUWin32AIOState *aio,
{ {
aio->aio_ctx = new_context; aio->aio_ctx = new_context;
aio_set_event_notifier(new_context, &aio->e, false, aio_set_event_notifier(new_context, &aio->e, false,
win32_aio_completion_cb, NULL); win32_aio_completion_cb, NULL, NULL);
} }
QEMUWin32AIOState *win32_aio_init(void) QEMUWin32AIOState *win32_aio_init(void)

16
hw/block/dataplane/virtio-blk.c
View File

@ -154,17 +154,6 @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
g_free(s); g_free(s);
} }
static bool virtio_blk_data_plane_handle_output(VirtIODevice *vdev,
VirtQueue *vq)
{
VirtIOBlock *s = (VirtIOBlock *)vdev;
assert(s->dataplane);
assert(s->dataplane_started);
return virtio_blk_handle_vq(s, vq);
}
/* Context: QEMU global mutex held */ /* Context: QEMU global mutex held */
int virtio_blk_data_plane_start(VirtIODevice *vdev) int virtio_blk_data_plane_start(VirtIODevice *vdev)
{ {
@ -258,8 +247,7 @@ int virtio_blk_data_plane_start(VirtIODevice *vdev)
for (i = 0; i < nvqs; i++) { for (i = 0; i < nvqs; i++) {
VirtQueue *vq = virtio_get_queue(s->vdev, i); VirtQueue *vq = virtio_get_queue(s->vdev, i);
virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, virtio_queue_aio_attach_host_notifier(vq, s->ctx);
virtio_blk_data_plane_handle_output);
} }
aio_context_release(s->ctx); aio_context_release(s->ctx);
return 0; return 0;
@ -302,7 +290,7 @@ static void virtio_blk_data_plane_stop_bh(void *opaque)
for (i = 0; i < s->conf->num_queues; i++) { for (i = 0; i < s->conf->num_queues; i++) {
VirtQueue *vq = virtio_get_queue(s->vdev, i); VirtQueue *vq = virtio_get_queue(s->vdev, i);
virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, NULL); virtio_queue_aio_detach_host_notifier(vq, s->ctx);
} }
} }

14
hw/block/virtio-blk.c
View File

@ -767,12 +767,11 @@ static int virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
return 0; return 0;
} }
bool virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq) void virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq)
{ {
VirtIOBlockReq *req; VirtIOBlockReq *req;
MultiReqBuffer mrb = {}; MultiReqBuffer mrb = {};
bool suppress_notifications = virtio_queue_get_notification(vq); bool suppress_notifications = virtio_queue_get_notification(vq);
bool progress = false;
aio_context_acquire(blk_get_aio_context(s->blk)); aio_context_acquire(blk_get_aio_context(s->blk));
blk_io_plug(s->blk); blk_io_plug(s->blk);
@ -783,7 +782,6 @@ bool virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq)
} }
while ((req = virtio_blk_get_request(s, vq))) { while ((req = virtio_blk_get_request(s, vq))) {
progress = true;
if (virtio_blk_handle_request(req, &mrb)) { if (virtio_blk_handle_request(req, &mrb)) {
virtqueue_detach_element(req->vq, &req->elem, 0); virtqueue_detach_element(req->vq, &req->elem, 0);
virtio_blk_free_request(req); virtio_blk_free_request(req);
@ -802,19 +800,13 @@ bool virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq)
blk_io_unplug(s->blk); blk_io_unplug(s->blk);
aio_context_release(blk_get_aio_context(s->blk)); aio_context_release(blk_get_aio_context(s->blk));
return progress;
}
static void virtio_blk_handle_output_do(VirtIOBlock *s, VirtQueue *vq)
{
virtio_blk_handle_vq(s, vq);
} }
static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq) static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
{ {
VirtIOBlock *s = (VirtIOBlock *)vdev; VirtIOBlock *s = (VirtIOBlock *)vdev;
if (s->dataplane) { if (s->dataplane && !s->dataplane_started) {
/* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
* dataplane here instead of waiting for .set_status(). * dataplane here instead of waiting for .set_status().
*/ */
@ -823,7 +815,7 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
return; return;
} }
} }
virtio_blk_handle_output_do(s, vq); virtio_blk_handle_vq(s, vq);
} }
void virtio_blk_process_queued_requests(VirtIOBlock *s, bool is_bh) void virtio_blk_process_queued_requests(VirtIOBlock *s, bool is_bh)

60
hw/scsi/virtio-scsi-dataplane.c
View File

@ -49,51 +49,6 @@ void virtio_scsi_dataplane_setup(VirtIOSCSI *s, Error **errp)
} }
} }
static bool virtio_scsi_data_plane_handle_cmd(VirtIODevice *vdev,
VirtQueue *vq)
{
bool progress = false;
VirtIOSCSI *s = VIRTIO_SCSI(vdev);
virtio_scsi_acquire(s);
if (!s->dataplane_fenced) {
assert(s->ctx && s->dataplane_started);
progress = virtio_scsi_handle_cmd_vq(s, vq);
}
virtio_scsi_release(s);
return progress;
}
static bool virtio_scsi_data_plane_handle_ctrl(VirtIODevice *vdev,
VirtQueue *vq)
{
bool progress = false;
VirtIOSCSI *s = VIRTIO_SCSI(vdev);
virtio_scsi_acquire(s);
if (!s->dataplane_fenced) {
assert(s->ctx && s->dataplane_started);
progress = virtio_scsi_handle_ctrl_vq(s, vq);
}
virtio_scsi_release(s);
return progress;
}
static bool virtio_scsi_data_plane_handle_event(VirtIODevice *vdev,
VirtQueue *vq)
{
bool progress = false;
VirtIOSCSI *s = VIRTIO_SCSI(vdev);
virtio_scsi_acquire(s);
if (!s->dataplane_fenced) {
assert(s->ctx && s->dataplane_started);
progress = virtio_scsi_handle_event_vq(s, vq);
}
virtio_scsi_release(s);
return progress;
}
static int virtio_scsi_set_host_notifier(VirtIOSCSI *s, VirtQueue *vq, int n) static int virtio_scsi_set_host_notifier(VirtIOSCSI *s, VirtQueue *vq, int n)
{ {
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s))); BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
@ -118,10 +73,10 @@ static void virtio_scsi_dataplane_stop_bh(void *opaque)
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s); VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
int i; int i;
virtio_queue_aio_set_host_notifier_handler(vs->ctrl_vq, s->ctx, NULL); virtio_queue_aio_detach_host_notifier(vs->ctrl_vq, s->ctx);
virtio_queue_aio_set_host_notifier_handler(vs->event_vq, s->ctx, NULL); virtio_queue_aio_detach_host_notifier(vs->event_vq, s->ctx);
for (i = 0; i < vs->conf.num_queues; i++) { for (i = 0; i < vs->conf.num_queues; i++) {
virtio_queue_aio_set_host_notifier_handler(vs->cmd_vqs[i], s->ctx, NULL); virtio_queue_aio_detach_host_notifier(vs->cmd_vqs[i], s->ctx);
} }
} }
@ -182,14 +137,11 @@ int virtio_scsi_dataplane_start(VirtIODevice *vdev)
memory_region_transaction_commit(); memory_region_transaction_commit();
aio_context_acquire(s->ctx); aio_context_acquire(s->ctx);
virtio_queue_aio_set_host_notifier_handler(vs->ctrl_vq, s->ctx, virtio_queue_aio_attach_host_notifier(vs->ctrl_vq, s->ctx);
virtio_scsi_data_plane_handle_ctrl); virtio_queue_aio_attach_host_notifier(vs->event_vq, s->ctx);
virtio_queue_aio_set_host_notifier_handler(vs->event_vq, s->ctx,
virtio_scsi_data_plane_handle_event);
for (i = 0; i < vs->conf.num_queues; i++) { for (i = 0; i < vs->conf.num_queues; i++) {
virtio_queue_aio_set_host_notifier_handler(vs->cmd_vqs[i], s->ctx, virtio_queue_aio_attach_host_notifier(vs->cmd_vqs[i], s->ctx);
virtio_scsi_data_plane_handle_cmd);
} }
s->dataplane_starting = false; s->dataplane_starting = false;

2
hw/scsi/virtio-scsi.c
View File

@ -720,7 +720,7 @@ static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
/* use non-QOM casts in the data path */ /* use non-QOM casts in the data path */
VirtIOSCSI *s = (VirtIOSCSI *)vdev; VirtIOSCSI *s = (VirtIOSCSI *)vdev;
if (s->ctx) { if (s->ctx && !s->dataplane_started) {
virtio_device_start_ioeventfd(vdev); virtio_device_start_ioeventfd(vdev);
if (!s->dataplane_fenced) { if (!s->dataplane_fenced) {
return; return;

61
hw/virtio/virtio.c
View File

@ -125,7 +125,6 @@ struct VirtQueue
uint16_t vector; uint16_t vector;
VirtIOHandleOutput handle_output; VirtIOHandleOutput handle_output;
VirtIOHandleAIOOutput handle_aio_output;
VirtIODevice *vdev; VirtIODevice *vdev;
EventNotifier guest_notifier; EventNotifier guest_notifier;
EventNotifier host_notifier; EventNotifier host_notifier;
@ -2303,24 +2302,6 @@ void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
} }
} }
static bool virtio_queue_notify_aio_vq(VirtQueue *vq)
{
bool ret = false;
if (vq->vring.desc && vq->handle_aio_output) {
VirtIODevice *vdev = vq->vdev;
trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
ret = vq->handle_aio_output(vdev, vq);
if (unlikely(vdev->start_on_kick)) {
virtio_set_started(vdev, true);
}
}
return ret;
}
static void virtio_queue_notify_vq(VirtQueue *vq) static void virtio_queue_notify_vq(VirtQueue *vq)
{ {
if (vq->vring.desc && vq->handle_output) { if (vq->vring.desc && vq->handle_output) {
@ -2399,7 +2380,6 @@ VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
vdev->vq[i].vring.num_default = queue_size; vdev->vq[i].vring.num_default = queue_size;
vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
vdev->vq[i].handle_output = handle_output; vdev->vq[i].handle_output = handle_output;
vdev->vq[i].handle_aio_output = NULL;
vdev->vq[i].used_elems = g_malloc0(sizeof(VirtQueueElement) * vdev->vq[i].used_elems = g_malloc0(sizeof(VirtQueueElement) *
queue_size); queue_size);
@ -2411,7 +2391,6 @@ void virtio_delete_queue(VirtQueue *vq)
vq->vring.num = 0; vq->vring.num = 0;
vq->vring.num_default = 0; vq->vring.num_default = 0;
vq->handle_output = NULL; vq->handle_output = NULL;
vq->handle_aio_output = NULL;
g_free(vq->used_elems); g_free(vq->used_elems);
vq->used_elems = NULL; vq->used_elems = NULL;
virtio_virtqueue_reset_region_cache(vq); virtio_virtqueue_reset_region_cache(vq);
@ -3516,14 +3495,6 @@ EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
return &vq->guest_notifier; return &vq->guest_notifier;
} }
static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
{
VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
if (event_notifier_test_and_clear(n)) {
virtio_queue_notify_aio_vq(vq);
}
}
static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n) static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
{ {
VirtQueue *vq = container_of(n, VirtQueue, host_notifier); VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
@ -3536,11 +3507,14 @@ static bool virtio_queue_host_notifier_aio_poll(void *opaque)
EventNotifier *n = opaque; EventNotifier *n = opaque;
VirtQueue *vq = container_of(n, VirtQueue, host_notifier); VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
if (!vq->vring.desc || virtio_queue_empty(vq)) { return vq->vring.desc && !virtio_queue_empty(vq);
return false; }
}
return virtio_queue_notify_aio_vq(vq); static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier *n)
{
VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
virtio_queue_notify_vq(vq);
} }
static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n) static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
@ -3551,24 +3525,23 @@ static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
virtio_queue_set_notification(vq, 1); virtio_queue_set_notification(vq, 1);
} }
void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx, void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx)
VirtIOHandleAIOOutput handle_output)
{ {
if (handle_output) {
vq->handle_aio_output = handle_output;
aio_set_event_notifier(ctx, &vq->host_notifier, true, aio_set_event_notifier(ctx, &vq->host_notifier, true,
virtio_queue_host_notifier_aio_read, virtio_queue_host_notifier_read,
virtio_queue_host_notifier_aio_poll); virtio_queue_host_notifier_aio_poll,
virtio_queue_host_notifier_aio_poll_ready);
aio_set_event_notifier_poll(ctx, &vq->host_notifier, aio_set_event_notifier_poll(ctx, &vq->host_notifier,
virtio_queue_host_notifier_aio_poll_begin, virtio_queue_host_notifier_aio_poll_begin,
virtio_queue_host_notifier_aio_poll_end); virtio_queue_host_notifier_aio_poll_end);
} else { }
aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL);
void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx)
{
aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL, NULL);
/* Test and clear notifier before after disabling event, /* Test and clear notifier before after disabling event,
* in case poll callback didn't have time to run. */ * in case poll callback didn't have time to run. */
virtio_queue_host_notifier_aio_read(&vq->host_notifier); virtio_queue_host_notifier_read(&vq->host_notifier);
vq->handle_aio_output = NULL;
}
} }
void virtio_queue_host_notifier_read(EventNotifier *n) void virtio_queue_host_notifier_read(EventNotifier *n)

6
hw/xen/xen-bus.c
View File

@ -1115,11 +1115,11 @@ void xen_device_set_event_channel_context(XenDevice *xendev,
if (channel->ctx) if (channel->ctx)
aio_set_fd_handler(channel->ctx, xenevtchn_fd(channel->xeh), true, aio_set_fd_handler(channel->ctx, xenevtchn_fd(channel->xeh), true,
NULL, NULL, NULL, NULL); NULL, NULL, NULL, NULL, NULL);
channel->ctx = ctx; channel->ctx = ctx;
aio_set_fd_handler(channel->ctx, xenevtchn_fd(channel->xeh), true, aio_set_fd_handler(channel->ctx, xenevtchn_fd(channel->xeh), true,
xen_device_event, NULL, xen_device_poll, channel); xen_device_event, NULL, xen_device_poll, NULL, channel);
} }
XenEventChannel *xen_device_bind_event_channel(XenDevice *xendev, XenEventChannel *xen_device_bind_event_channel(XenDevice *xendev,
@ -1193,7 +1193,7 @@ void xen_device_unbind_event_channel(XenDevice *xendev,
QLIST_REMOVE(channel, list); QLIST_REMOVE(channel, list);
aio_set_fd_handler(channel->ctx, xenevtchn_fd(channel->xeh), true, aio_set_fd_handler(channel->ctx, xenevtchn_fd(channel->xeh), true,
NULL, NULL, NULL, NULL); NULL, NULL, NULL, NULL, NULL);
if (xenevtchn_unbind(channel->xeh, channel->local_port) < 0) { if (xenevtchn_unbind(channel->xeh, channel->local_port) < 0) {
error_setg_errno(errp, errno, "xenevtchn_unbind failed"); error_setg_errno(errp, errno, "xenevtchn_unbind failed");

4
include/block/aio.h
View File

@ -469,6 +469,7 @@ void aio_set_fd_handler(AioContext *ctx,
IOHandler *io_read, IOHandler *io_read,
IOHandler *io_write, IOHandler *io_write,
AioPollFn *io_poll, AioPollFn *io_poll,
IOHandler *io_poll_ready,
void *opaque); void *opaque);
/* Set polling begin/end callbacks for a file descriptor that has already been /* Set polling begin/end callbacks for a file descriptor that has already been
@ -490,7 +491,8 @@ void aio_set_event_notifier(AioContext *ctx,
EventNotifier *notifier, EventNotifier *notifier,
bool is_external, bool is_external,
EventNotifierHandler *io_read, EventNotifierHandler *io_read,
AioPollFn *io_poll); AioPollFn *io_poll,
EventNotifierHandler *io_poll_ready);
/* Set polling begin/end callbacks for an event notifier that has already been /* Set polling begin/end callbacks for an event notifier that has already been
* registered with aio_set_event_notifier. Do nothing if the event notifier is * registered with aio_set_event_notifier. Do nothing if the event notifier is

2
include/hw/virtio/virtio-blk.h
View File

@ -90,7 +90,7 @@ typedef struct MultiReqBuffer {
bool is_write; bool is_write;
} MultiReqBuffer; } MultiReqBuffer;
bool virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq); void virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq);
void virtio_blk_process_queued_requests(VirtIOBlock *s, bool is_bh); void virtio_blk_process_queued_requests(VirtIOBlock *s, bool is_bh);
#endif #endif

5
include/hw/virtio/virtio.h
View File

@ -175,7 +175,6 @@ void virtio_error(VirtIODevice *vdev, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name); void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name);
typedef void (*VirtIOHandleOutput)(VirtIODevice *, VirtQueue *); typedef void (*VirtIOHandleOutput)(VirtIODevice *, VirtQueue *);
typedef bool (*VirtIOHandleAIOOutput)(VirtIODevice *, VirtQueue *);
VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
VirtIOHandleOutput handle_output); VirtIOHandleOutput handle_output);
@ -317,8 +316,8 @@ bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev);
EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq); EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq);
void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled); void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled);
void virtio_queue_host_notifier_read(EventNotifier *n); void virtio_queue_host_notifier_read(EventNotifier *n);
void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx, void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx);
VirtIOHandleAIOOutput handle_output); void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx);
VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector); VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector);
VirtQueue *virtio_vector_next_queue(VirtQueue *vq); VirtQueue *virtio_vector_next_queue(VirtQueue *vq);

6
io/channel-command.c
View File

@ -346,8 +346,10 @@ static void qio_channel_command_set_aio_fd_handler(QIOChannel *ioc,
void *opaque) void *opaque)
{ {
QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc); QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
aio_set_fd_handler(ctx, cioc->readfd, false, io_read, NULL, NULL, opaque); aio_set_fd_handler(ctx, cioc->readfd, false,
aio_set_fd_handler(ctx, cioc->writefd, false, NULL, io_write, NULL, opaque); io_read, NULL, NULL, NULL, opaque);
aio_set_fd_handler(ctx, cioc->writefd, false,
NULL, io_write, NULL, NULL, opaque);
} }

3
io/channel-file.c
View File

@ -191,7 +191,8 @@ static void qio_channel_file_set_aio_fd_handler(QIOChannel *ioc,
void *opaque) void *opaque)
{ {
QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc); QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
aio_set_fd_handler(ctx, fioc->fd, false, io_read, io_write, NULL, opaque); aio_set_fd_handler(ctx, fioc->fd, false, io_read, io_write,
NULL, NULL, opaque);
} }
static GSource *qio_channel_file_create_watch(QIOChannel *ioc, static GSource *qio_channel_file_create_watch(QIOChannel *ioc,

3
io/channel-socket.c
View File

@ -761,7 +761,8 @@ static void qio_channel_socket_set_aio_fd_handler(QIOChannel *ioc,
void *opaque) void *opaque)
{ {
QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc); QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
aio_set_fd_handler(ctx, sioc->fd, false, io_read, io_write, NULL, opaque); aio_set_fd_handler(ctx, sioc->fd, false,
io_read, io_write, NULL, NULL, opaque);
} }
static GSource *qio_channel_socket_create_watch(QIOChannel *ioc, static GSource *qio_channel_socket_create_watch(QIOChannel *ioc,

8
migration/rdma.c
View File

@ -3161,14 +3161,14 @@ static void qio_channel_rdma_set_aio_fd_handler(QIOChannel *ioc,
QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc); QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc);
if (io_read) { if (io_read) {
aio_set_fd_handler(ctx, rioc->rdmain->recv_comp_channel->fd, aio_set_fd_handler(ctx, rioc->rdmain->recv_comp_channel->fd,
false, io_read, io_write, NULL, opaque); false, io_read, io_write, NULL, NULL, opaque);
aio_set_fd_handler(ctx, rioc->rdmain->send_comp_channel->fd, aio_set_fd_handler(ctx, rioc->rdmain->send_comp_channel->fd,
false, io_read, io_write, NULL, opaque); false, io_read, io_write, NULL, NULL, opaque);
} else { } else {
aio_set_fd_handler(ctx, rioc->rdmaout->recv_comp_channel->fd, aio_set_fd_handler(ctx, rioc->rdmaout->recv_comp_channel->fd,
false, io_read, io_write, NULL, opaque); false, io_read, io_write, NULL, NULL, opaque);
aio_set_fd_handler(ctx, rioc->rdmaout->send_comp_channel->fd, aio_set_fd_handler(ctx, rioc->rdmaout->send_comp_channel->fd,
false, io_read, io_write, NULL, opaque); false, io_read, io_write, NULL, NULL, opaque);
} }
} }

4
tests/unit/test-aio.c
View File

@ -130,7 +130,7 @@ static void *test_acquire_thread(void *opaque)
static void set_event_notifier(AioContext *ctx, EventNotifier *notifier, static void set_event_notifier(AioContext *ctx, EventNotifier *notifier,
EventNotifierHandler *handler) EventNotifierHandler *handler)
{ {
aio_set_event_notifier(ctx, notifier, false, handler, NULL); aio_set_event_notifier(ctx, notifier, false, handler, NULL, NULL);
} }
static void dummy_notifier_read(EventNotifier *n) static void dummy_notifier_read(EventNotifier *n)
@ -390,7 +390,7 @@ static void test_aio_external_client(void)
for (i = 1; i < 3; i++) { for (i = 1; i < 3; i++) {
EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true }; EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
event_notifier_init(&data.e, false); event_notifier_init(&data.e, false);
aio_set_event_notifier(ctx, &data.e, true, event_ready_cb, NULL); aio_set_event_notifier(ctx, &data.e, true, event_ready_cb, NULL, NULL);
event_notifier_set(&data.e); event_notifier_set(&data.e);
for (j = 0; j < i; j++) { for (j = 0; j < i; j++) {
aio_disable_external(ctx); aio_disable_external(ctx);

4
tests/unit/test-fdmon-epoll.c
View File

@ -22,14 +22,14 @@ static void add_event_notifiers(EventNotifier *notifiers, size_t n)
for (size_t i = 0; i < n; i++) { for (size_t i = 0; i < n; i++) {
event_notifier_init(&notifiers[i], false); event_notifier_init(&notifiers[i], false);
aio_set_event_notifier(ctx, &notifiers[i], false, aio_set_event_notifier(ctx, &notifiers[i], false,
dummy_fd_handler, NULL); dummy_fd_handler, NULL, NULL);
} }
} }
static void remove_event_notifiers(EventNotifier *notifiers, size_t n) static void remove_event_notifiers(EventNotifier *notifiers, size_t n)
{ {
for (size_t i = 0; i < n; i++) { for (size_t i = 0; i < n; i++) {
aio_set_event_notifier(ctx, &notifiers[i], false, NULL, NULL); aio_set_event_notifier(ctx, &notifiers[i], false, NULL, NULL, NULL);
event_notifier_cleanup(&notifiers[i]); event_notifier_cleanup(&notifiers[i]);
} }
} }

87
util/aio-posix.c
View File

@ -23,6 +23,15 @@
#include "trace.h" #include "trace.h"
#include "aio-posix.h" #include "aio-posix.h"
/*
* G_IO_IN and G_IO_OUT are not appropriate revents values for polling, since
* the handler may not need to access the file descriptor. For example, the
* handler doesn't need to read from an EventNotifier if it polled a memory
* location and a read syscall would be slow. Define our own unique revents
* value to indicate that polling determined this AioHandler is ready.
*/
#define REVENTS_POLL_READY 0
/* Stop userspace polling on a handler if it isn't active for some time */ /* Stop userspace polling on a handler if it isn't active for some time */
#define POLL_IDLE_INTERVAL_NS (7 * NANOSECONDS_PER_SECOND) #define POLL_IDLE_INTERVAL_NS (7 * NANOSECONDS_PER_SECOND)
@ -93,6 +102,7 @@ void aio_set_fd_handler(AioContext *ctx,
IOHandler *io_read, IOHandler *io_read,
IOHandler *io_write, IOHandler *io_write,
AioPollFn *io_poll, AioPollFn *io_poll,
IOHandler *io_poll_ready,
void *opaque) void *opaque)
{ {
AioHandler *node; AioHandler *node;
@ -101,6 +111,10 @@ void aio_set_fd_handler(AioContext *ctx,
bool deleted = false; bool deleted = false;
int poll_disable_change; int poll_disable_change;
if (io_poll && !io_poll_ready) {
io_poll = NULL; /* polling only makes sense if there is a handler */
}
qemu_lockcnt_lock(&ctx->list_lock); qemu_lockcnt_lock(&ctx->list_lock);
node = find_aio_handler(ctx, fd); node = find_aio_handler(ctx, fd);
@ -127,6 +141,7 @@ void aio_set_fd_handler(AioContext *ctx,
new_node->io_read = io_read; new_node->io_read = io_read;
new_node->io_write = io_write; new_node->io_write = io_write;
new_node->io_poll = io_poll; new_node->io_poll = io_poll;
new_node->io_poll_ready = io_poll_ready;
new_node->opaque = opaque; new_node->opaque = opaque;
new_node->is_external = is_external; new_node->is_external = is_external;
@ -182,10 +197,12 @@ void aio_set_event_notifier(AioContext *ctx,
EventNotifier *notifier, EventNotifier *notifier,
bool is_external, bool is_external,
EventNotifierHandler *io_read, EventNotifierHandler *io_read,
AioPollFn *io_poll) AioPollFn *io_poll,
EventNotifierHandler *io_poll_ready)
{ {
aio_set_fd_handler(ctx, event_notifier_get_fd(notifier), is_external, aio_set_fd_handler(ctx, event_notifier_get_fd(notifier), is_external,
(IOHandler *)io_read, NULL, io_poll, notifier); (IOHandler *)io_read, NULL, io_poll,
(IOHandler *)io_poll_ready, notifier);
} }
void aio_set_event_notifier_poll(AioContext *ctx, void aio_set_event_notifier_poll(AioContext *ctx,
@ -198,7 +215,8 @@ void aio_set_event_notifier_poll(AioContext *ctx,
(IOHandler *)io_poll_end); (IOHandler *)io_poll_end);
} }
static bool poll_set_started(AioContext *ctx, bool started) static bool poll_set_started(AioContext *ctx, AioHandlerList *ready_list,
bool started)
{ {
AioHandler *node; AioHandler *node;
bool progress = false; bool progress = false;
@ -228,8 +246,9 @@ static bool poll_set_started(AioContext *ctx, bool started)
} }
/* Poll one last time in case ->io_poll_end() raced with the event */ /* Poll one last time in case ->io_poll_end() raced with the event */
if (!started) { if (!started && node->io_poll(node->opaque)) {
progress = node->io_poll(node->opaque) || progress; aio_add_ready_handler(ready_list, node, REVENTS_POLL_READY);
progress = true;
} }
} }
qemu_lockcnt_dec(&ctx->list_lock); qemu_lockcnt_dec(&ctx->list_lock);
@ -240,8 +259,11 @@ static bool poll_set_started(AioContext *ctx, bool started)
bool aio_prepare(AioContext *ctx) bool aio_prepare(AioContext *ctx)
{ {
AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);
/* Poll mode cannot be used with glib's event loop, disable it. */ /* Poll mode cannot be used with glib's event loop, disable it. */
poll_set_started(ctx, false); poll_set_started(ctx, &ready_list, false);
/* TODO what to do with this list? */
return false; return false;
} }
@ -321,6 +343,18 @@ static bool aio_dispatch_handler(AioContext *ctx, AioHandler *node)
} }
QLIST_INSERT_HEAD(&ctx->poll_aio_handlers, node, node_poll); QLIST_INSERT_HEAD(&ctx->poll_aio_handlers, node, node_poll);
} }
if (!QLIST_IS_INSERTED(node, node_deleted) &&
revents == 0 &&
aio_node_check(ctx, node->is_external) &&
node->io_poll_ready) {
node->io_poll_ready(node->opaque);
/*
* Return early since revents was zero. aio_notify() does not count as
* progress.
*/
return node->opaque != &ctx->notifier;
}
if (!QLIST_IS_INSERTED(node, node_deleted) && if (!QLIST_IS_INSERTED(node, node_deleted) &&
(revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) && (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) &&
@ -387,6 +421,7 @@ void aio_dispatch(AioContext *ctx)
} }
static bool run_poll_handlers_once(AioContext *ctx, static bool run_poll_handlers_once(AioContext *ctx,
AioHandlerList *ready_list,
int64_t now, int64_t now,
int64_t *timeout) int64_t *timeout)
{ {
@ -397,6 +432,8 @@ static bool run_poll_handlers_once(AioContext *ctx,
QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) { QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
if (aio_node_check(ctx, node->is_external) && if (aio_node_check(ctx, node->is_external) &&
node->io_poll(node->opaque)) { node->io_poll(node->opaque)) {
aio_add_ready_handler(ready_list, node, REVENTS_POLL_READY);
node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS; node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS;
/* /*
@ -420,7 +457,9 @@ static bool fdmon_supports_polling(AioContext *ctx)
return ctx->fdmon_ops->need_wait != aio_poll_disabled; return ctx->fdmon_ops->need_wait != aio_poll_disabled;
} }
static bool remove_idle_poll_handlers(AioContext *ctx, int64_t now) static bool remove_idle_poll_handlers(AioContext *ctx,
AioHandlerList *ready_list,
int64_t now)
{ {
AioHandler *node; AioHandler *node;
AioHandler *tmp; AioHandler *tmp;
@ -451,7 +490,11 @@ static bool remove_idle_poll_handlers(AioContext *ctx, int64_t now)
* Nevermind about re-adding the handler in the rare case where * Nevermind about re-adding the handler in the rare case where
* this causes progress. * this causes progress.
*/ */
progress = node->io_poll(node->opaque) || progress; if (node->io_poll(node->opaque)) {
aio_add_ready_handler(ready_list, node,
REVENTS_POLL_READY);
progress = true;
}
} }
} }
} }
@ -461,6 +504,7 @@ static bool remove_idle_poll_handlers(AioContext *ctx, int64_t now)
/* run_poll_handlers: /* run_poll_handlers:
* @ctx: the AioContext * @ctx: the AioContext
* @ready_list: the list to place ready handlers on
* @max_ns: maximum time to poll for, in nanoseconds * @max_ns: maximum time to poll for, in nanoseconds
* *
* Polls for a given time. * Polls for a given time.
@ -469,7 +513,8 @@ static bool remove_idle_poll_handlers(AioContext *ctx, int64_t now)
* *
* Returns: true if progress was made, false otherwise * Returns: true if progress was made, false otherwise
*/ */
static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout) static bool run_poll_handlers(AioContext *ctx, AioHandlerList *ready_list,
int64_t max_ns, int64_t *timeout)
{ {
bool progress; bool progress;
int64_t start_time, elapsed_time; int64_t start_time, elapsed_time;
@ -490,13 +535,15 @@ static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout)
start_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); start_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
do { do {
progress = run_poll_handlers_once(ctx, start_time, timeout); progress = run_poll_handlers_once(ctx, ready_list,
start_time, timeout);
elapsed_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time; elapsed_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time;
max_ns = qemu_soonest_timeout(*timeout, max_ns); max_ns = qemu_soonest_timeout(*timeout, max_ns);
assert(!(max_ns && progress)); assert(!(max_ns && progress));
} while (elapsed_time < max_ns && !ctx->fdmon_ops->need_wait(ctx)); } while (elapsed_time < max_ns && !ctx->fdmon_ops->need_wait(ctx));
if (remove_idle_poll_handlers(ctx, start_time + elapsed_time)) { if (remove_idle_poll_handlers(ctx, ready_list,
start_time + elapsed_time)) {
*timeout = 0; *timeout = 0;
progress = true; progress = true;
} }
@ -514,6 +561,7 @@ static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout)
/* try_poll_mode: /* try_poll_mode:
* @ctx: the AioContext * @ctx: the AioContext
* @ready_list: list to add handlers that need to be run
* @timeout: timeout for blocking wait, computed by the caller and updated if * @timeout: timeout for blocking wait, computed by the caller and updated if
* polling succeeds. * polling succeeds.
* *
@ -521,7 +569,8 @@ static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout)
* *
* Returns: true if progress was made, false otherwise * Returns: true if progress was made, false otherwise
*/ */
static bool try_poll_mode(AioContext *ctx, int64_t *timeout) static bool try_poll_mode(AioContext *ctx, AioHandlerList *ready_list,
int64_t *timeout)
{ {
int64_t max_ns; int64_t max_ns;
@ -531,14 +580,14 @@ static bool try_poll_mode(AioContext *ctx, int64_t *timeout)
max_ns = qemu_soonest_timeout(*timeout, ctx->poll_ns); max_ns = qemu_soonest_timeout(*timeout, ctx->poll_ns);
if (max_ns && !ctx->fdmon_ops->need_wait(ctx)) { if (max_ns && !ctx->fdmon_ops->need_wait(ctx)) {
poll_set_started(ctx, true); poll_set_started(ctx, ready_list, true);
if (run_poll_handlers(ctx, max_ns, timeout)) { if (run_poll_handlers(ctx, ready_list, max_ns, timeout)) {
return true; return true;
} }
} }
if (poll_set_started(ctx, false)) { if (poll_set_started(ctx, ready_list, false)) {
*timeout = 0; *timeout = 0;
return true; return true;
} }
@ -549,7 +598,6 @@ static bool try_poll_mode(AioContext *ctx, int64_t *timeout)
bool aio_poll(AioContext *ctx, bool blocking) bool aio_poll(AioContext *ctx, bool blocking)
{ {
AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list); AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);
int ret = 0;
bool progress; bool progress;
bool use_notify_me; bool use_notify_me;
int64_t timeout; int64_t timeout;
@ -574,7 +622,7 @@ bool aio_poll(AioContext *ctx, bool blocking)
} }
timeout = blocking ? aio_compute_timeout(ctx) : 0; timeout = blocking ? aio_compute_timeout(ctx) : 0;
progress = try_poll_mode(ctx, &timeout); progress = try_poll_mode(ctx, &ready_list, &timeout);
assert(!(timeout && progress)); assert(!(timeout && progress));
/* /*
@ -604,7 +652,7 @@ bool aio_poll(AioContext *ctx, bool blocking)
* system call---a single round of run_poll_handlers_once suffices. * system call---a single round of run_poll_handlers_once suffices.
*/ */
if (timeout || ctx->fdmon_ops->need_wait(ctx)) { if (timeout || ctx->fdmon_ops->need_wait(ctx)) {
ret = ctx->fdmon_ops->wait(ctx, &ready_list, timeout); ctx->fdmon_ops->wait(ctx, &ready_list, timeout);
} }
if (use_notify_me) { if (use_notify_me) {
@ -657,10 +705,7 @@ bool aio_poll(AioContext *ctx, bool blocking)
} }
progress |= aio_bh_poll(ctx); progress |= aio_bh_poll(ctx);
if (ret > 0) {
progress |= aio_dispatch_ready_handlers(ctx, &ready_list); progress |= aio_dispatch_ready_handlers(ctx, &ready_list);
}
aio_free_deleted_handlers(ctx); aio_free_deleted_handlers(ctx);

1
util/aio-posix.h
View File

@ -24,6 +24,7 @@ struct AioHandler {
IOHandler *io_read; IOHandler *io_read;
IOHandler *io_write; IOHandler *io_write;
AioPollFn *io_poll; AioPollFn *io_poll;
IOHandler *io_poll_ready;
IOHandler *io_poll_begin; IOHandler *io_poll_begin;
IOHandler *io_poll_end; IOHandler *io_poll_end;
void *opaque; void *opaque;

4
util/aio-win32.c
View File

@ -68,6 +68,7 @@ void aio_set_fd_handler(AioContext *ctx,
IOHandler *io_read, IOHandler *io_read,
IOHandler *io_write, IOHandler *io_write,
AioPollFn *io_poll, AioPollFn *io_poll,
IOHandler *io_poll_ready,
void *opaque) void *opaque)
{ {
/* fd is a SOCKET in our case */ /* fd is a SOCKET in our case */
@ -136,7 +137,8 @@ void aio_set_event_notifier(AioContext *ctx,
EventNotifier *e, EventNotifier *e,
bool is_external, bool is_external,
EventNotifierHandler *io_notify, EventNotifierHandler *io_notify,
AioPollFn *io_poll) AioPollFn *io_poll,
EventNotifierHandler *io_poll_ready)
{ {
AioHandler *node; AioHandler *node;

10
util/async.c
View File

@ -362,7 +362,7 @@ aio_ctx_finalize(GSource *source)
g_free(bh); g_free(bh);
} }
aio_set_event_notifier(ctx, &ctx->notifier, false, NULL, NULL); aio_set_event_notifier(ctx, &ctx->notifier, false, NULL, NULL, NULL);
event_notifier_cleanup(&ctx->notifier); event_notifier_cleanup(&ctx->notifier);
qemu_rec_mutex_destroy(&ctx->lock); qemu_rec_mutex_destroy(&ctx->lock);
qemu_lockcnt_destroy(&ctx->list_lock); qemu_lockcnt_destroy(&ctx->list_lock);
@ -485,6 +485,11 @@ static bool aio_context_notifier_poll(void *opaque)
return qatomic_read(&ctx->notified); return qatomic_read(&ctx->notified);
} }
static void aio_context_notifier_poll_ready(EventNotifier *e)
{
/* Do nothing, we just wanted to kick the event loop */
}
static void co_schedule_bh_cb(void *opaque) static void co_schedule_bh_cb(void *opaque)
{ {
AioContext *ctx = opaque; AioContext *ctx = opaque;
@ -536,7 +541,8 @@ AioContext *aio_context_new(Error **errp)
aio_set_event_notifier(ctx, &ctx->notifier, aio_set_event_notifier(ctx, &ctx->notifier,
false, false,
aio_context_notifier_cb, aio_context_notifier_cb,
aio_context_notifier_poll); aio_context_notifier_poll,
aio_context_notifier_poll_ready);
#ifdef CONFIG_LINUX_AIO #ifdef CONFIG_LINUX_AIO
ctx->linux_aio = NULL; ctx->linux_aio = NULL;
#endif #endif

4
util/main-loop.c
View File

@ -582,7 +582,7 @@ void qemu_set_fd_handler(int fd,
{ {
iohandler_init(); iohandler_init();
aio_set_fd_handler(iohandler_ctx, fd, false, aio_set_fd_handler(iohandler_ctx, fd, false,
fd_read, fd_write, NULL, opaque); fd_read, fd_write, NULL, NULL, opaque);
} }
void event_notifier_set_handler(EventNotifier *e, void event_notifier_set_handler(EventNotifier *e,
@ -590,5 +590,5 @@ void event_notifier_set_handler(EventNotifier *e,
{ {
iohandler_init(); iohandler_init();
aio_set_event_notifier(iohandler_ctx, e, false, aio_set_event_notifier(iohandler_ctx, e, false,
handler, NULL); handler, NULL, NULL);
} }

5
util/qemu-coroutine-io.c
View File

@ -75,7 +75,8 @@ typedef struct {
static void fd_coroutine_enter(void *opaque) static void fd_coroutine_enter(void *opaque)
{ {
FDYieldUntilData *data = opaque; FDYieldUntilData *data = opaque;
aio_set_fd_handler(data->ctx, data->fd, false, NULL, NULL, NULL, NULL); aio_set_fd_handler(data->ctx, data->fd, false,
NULL, NULL, NULL, NULL, NULL);
qemu_coroutine_enter(data->co); qemu_coroutine_enter(data->co);
} }
@ -88,6 +89,6 @@ void coroutine_fn yield_until_fd_readable(int fd)
data.co = qemu_coroutine_self(); data.co = qemu_coroutine_self();
data.fd = fd; data.fd = fd;
aio_set_fd_handler( aio_set_fd_handler(
data.ctx, fd, false, fd_coroutine_enter, NULL, NULL, &data); data.ctx, fd, false, fd_coroutine_enter, NULL, NULL, NULL, &data);
qemu_coroutine_yield(); qemu_coroutine_yield();
} }

11
util/vhost-user-server.c
View File

@ -250,7 +250,7 @@ set_watch(VuDev *vu_dev, int fd, int vu_evt,
vu_fd_watch->cb = cb; vu_fd_watch->cb = cb;
qemu_set_nonblock(fd); qemu_set_nonblock(fd);
aio_set_fd_handler(server->ioc->ctx, fd, true, kick_handler, aio_set_fd_handler(server->ioc->ctx, fd, true, kick_handler,
NULL, NULL, vu_fd_watch); NULL, NULL, NULL, vu_fd_watch);
vu_fd_watch->vu_dev = vu_dev; vu_fd_watch->vu_dev = vu_dev;
vu_fd_watch->pvt = pvt; vu_fd_watch->pvt = pvt;
} }
@ -270,7 +270,8 @@ static void remove_watch(VuDev *vu_dev, int fd)
if (!vu_fd_watch) { if (!vu_fd_watch) {
return; return;
} }
aio_set_fd_handler(server->ioc->ctx, fd, true, NULL, NULL, NULL, NULL); aio_set_fd_handler(server->ioc->ctx, fd, true,
NULL, NULL, NULL, NULL, NULL);
QTAILQ_REMOVE(&server->vu_fd_watches, vu_fd_watch, next); QTAILQ_REMOVE(&server->vu_fd_watches, vu_fd_watch, next);
g_free(vu_fd_watch); g_free(vu_fd_watch);
@ -334,7 +335,7 @@ void vhost_user_server_stop(VuServer *server)
QTAILQ_FOREACH(vu_fd_watch, &server->vu_fd_watches, next) { QTAILQ_FOREACH(vu_fd_watch, &server->vu_fd_watches, next) {
aio_set_fd_handler(server->ctx, vu_fd_watch->fd, true, aio_set_fd_handler(server->ctx, vu_fd_watch->fd, true,
NULL, NULL, NULL, vu_fd_watch); NULL, NULL, NULL, NULL, vu_fd_watch);
} }
qio_channel_shutdown(server->ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); qio_channel_shutdown(server->ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL);
@ -377,7 +378,7 @@ void vhost_user_server_attach_aio_context(VuServer *server, AioContext *ctx)
QTAILQ_FOREACH(vu_fd_watch, &server->vu_fd_watches, next) { QTAILQ_FOREACH(vu_fd_watch, &server->vu_fd_watches, next) {
aio_set_fd_handler(ctx, vu_fd_watch->fd, true, kick_handler, NULL, aio_set_fd_handler(ctx, vu_fd_watch->fd, true, kick_handler, NULL,
NULL, vu_fd_watch); NULL, NULL, vu_fd_watch);
} }
aio_co_schedule(ctx, server->co_trip); aio_co_schedule(ctx, server->co_trip);
@ -391,7 +392,7 @@ void vhost_user_server_detach_aio_context(VuServer *server)
QTAILQ_FOREACH(vu_fd_watch, &server->vu_fd_watches, next) { QTAILQ_FOREACH(vu_fd_watch, &server->vu_fd_watches, next) {
aio_set_fd_handler(server->ctx, vu_fd_watch->fd, true, aio_set_fd_handler(server->ctx, vu_fd_watch->fd, true,
NULL, NULL, NULL, vu_fd_watch); NULL, NULL, NULL, NULL, vu_fd_watch);
} }
qio_channel_detach_aio_context(server->ioc); qio_channel_detach_aio_context(server->ioc);