virtio: support unlocked queue kick

Based on patch by Christoph for virtio_blk speedup:

	Split virtqueue_kick to be able to do the actual notification
	outside the lock protecting the virtqueue.  This patch was
	originally done by Stefan Hajnoczi, but I can't find the
	original one anymore and had to recreated it from memory.
	Pointers to the original or corrections for the commit message
	are welcome.

Stefan's patch was here:

	https://github.com/stefanha/linux/commit/a6d06644e3a58e57a774e77d7dc34c4a5a2e7496
	http://www.spinics.net/lists/linux-virtualization/msg14616.html

Third time's the charm!

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 6ea92a6d1134a51ed9c9a4b0c38b8851e9a8658e..c56bbe799241fe81b68e006a7f56ce6f73546606 100644 (file)
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -262,19 +262,22 @@ add_head:
 EXPORT_SYMBOL_GPL(virtqueue_add_buf);
 
 /**
- * virtqueue_kick - update after add_buf
+ * virtqueue_kick_prepare - first half of split virtqueue_kick call.
  * @vq: the struct virtqueue
  *
- * After one or more virtqueue_add_buf calls, invoke this to kick
- * the other side.
+ * Instead of virtqueue_kick(), you can do:
+ *     if (virtqueue_kick_prepare(vq))
+ *             virtqueue_notify(vq);
  *
- * Caller must ensure we don't call this with other virtqueue
- * operations at the same time (except where noted).
+ * This is sometimes useful because the virtqueue_kick_prepare() needs
+ * to be serialized, but the actual virtqueue_notify() call does not.
  */
-void virtqueue_kick(struct virtqueue *_vq)
+bool virtqueue_kick_prepare(struct virtqueue *_vq)
 {
        struct vring_virtqueue *vq = to_vvq(_vq);
        u16 new, old;
+       bool needs_kick;
+
        START_USE(vq);
        /* Descriptors and available array need to be set before we expose the
         * new available array entries. */
@@ -287,13 +290,46 @@ void virtqueue_kick(struct virtqueue *_vq)
        /* Need to update avail index before checking if we should notify */
        virtio_mb(vq);
 
-       if (vq->event ?
-           vring_need_event(vring_avail_event(&vq->vring), new, old) :
-           !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
-               /* Prod other side to tell it about changes. */
-               vq->notify(&vq->vq);
-
+       if (vq->event) {
+               needs_kick = vring_need_event(vring_avail_event(&vq->vring),
+                                             new, old);
+       } else {
+               needs_kick = !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY);
+       }
        END_USE(vq);
+       return needs_kick;
+}
+EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
+
+/**
+ * virtqueue_notify - second half of split virtqueue_kick call.
+ * @vq: the struct virtqueue
+ *
+ * This does not need to be serialized.
+ */
+void virtqueue_notify(struct virtqueue *_vq)
+{
+       struct vring_virtqueue *vq = to_vvq(_vq);
+
+       /* Prod other side to tell it about changes. */
+       vq->notify(_vq);
+}
+EXPORT_SYMBOL_GPL(virtqueue_notify);
+
+/**
+ * virtqueue_kick - update after add_buf
+ * @vq: the struct virtqueue
+ *
+ * After one or more virtqueue_add_buf calls, invoke this to kick
+ * the other side.
+ *
+ * Caller must ensure we don't call this with other virtqueue
+ * operations at the same time (except where noted).
+ */
+void virtqueue_kick(struct virtqueue *vq)
+{
+       if (virtqueue_kick_prepare(vq))
+               virtqueue_notify(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_kick);
 

diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index ec1706e7df50c7b6a4451801b2b782c4947c8bd8..31fe3a62874b16506bd5740cc38ea741cc67e798 100644 (file)
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -34,6 +34,10 @@ int virtqueue_add_buf(struct virtqueue *vq,
 
 void virtqueue_kick(struct virtqueue *vq);
 
+bool virtqueue_kick_prepare(struct virtqueue *vq);
+
+void virtqueue_notify(struct virtqueue *vq);
+
 void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len);
 
 void virtqueue_disable_cb(struct virtqueue *vq);