[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / vhost / net.c

/* Copyright (C) 2009 Red Hat, Inc.
 * Author: Michael S. Tsirkin <mst@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.
 *
 * virtio-net server in host kernel.
 */

#include <linux/compat.h>
#include <linux/eventfd.h>
#include <linux/vhost.h>
#include <linux/virtio_net.h>
#include <linux/mmu_context.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/workqueue.h>
#include <linux/rcupdate.h>
#include <linux/file.h>
#include <linux/slab.h>

#include <linux/net.h>
#include <linux/if_packet.h>
#include <linux/if_arp.h>
#include <linux/if_tun.h>
#include <linux/if_macvlan.h>

#include <net/sock.h>

#include "vhost.h"

/* Max number of bytes transferred before requeueing the job.
 * Using this limit prevents one virtqueue from starving others. */
#define VHOST_NET_WEIGHT 0x80000

enum {
	VHOST_NET_VQ_RX = 0,
	VHOST_NET_VQ_TX = 1,
	VHOST_NET_VQ_MAX = 2,
};

enum vhost_net_poll_state {
	VHOST_NET_POLL_DISABLED = 0,
	VHOST_NET_POLL_STARTED = 1,
	VHOST_NET_POLL_STOPPED = 2,
};

struct vhost_net {
	struct vhost_dev dev;
	struct vhost_virtqueue vqs[VHOST_NET_VQ_MAX];
	struct vhost_poll poll[VHOST_NET_VQ_MAX];
	/* Tells us whether we are polling a socket for TX.
	 * We only do this when socket buffer fills up.
	 * Protected by tx vq lock. */
	enum vhost_net_poll_state tx_poll_state;
};

/* Pop first len bytes from iovec. Return number of segments used. */
static int move_iovec_hdr(struct iovec *from, struct iovec *to,
			  size_t len, int iov_count)
{
	int seg = 0;
	size_t size;
	while (len && seg < iov_count) {
		size = min(from->iov_len, len);
		to->iov_base = from->iov_base;
		to->iov_len = size;
		from->iov_len -= size;
		from->iov_base += size;
		len -= size;
		++from;
		++to;
		++seg;
	}
	return seg;
}

/* Caller must have TX VQ lock */
static void tx_poll_stop(struct vhost_net *net)
{
	if (likely(net->tx_poll_state != VHOST_NET_POLL_STARTED))
		return;
	vhost_poll_stop(net->poll + VHOST_NET_VQ_TX);
	net->tx_poll_state = VHOST_NET_POLL_STOPPED;
}

/* Caller must have TX VQ lock */
static void tx_poll_start(struct vhost_net *net, struct socket *sock)
{
	if (unlikely(net->tx_poll_state != VHOST_NET_POLL_STOPPED))
		return;
	vhost_poll_start(net->poll + VHOST_NET_VQ_TX, sock->file);
	net->tx_poll_state = VHOST_NET_POLL_STARTED;
}

/* Expects to be always run from workqueue - which acts as
 * read-size critical section for our kind of RCU. */
static void handle_tx(struct vhost_net *net)
{
	struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_TX];
	unsigned out, in, s;
	int head;
	struct msghdr msg = {
		.msg_name = NULL,
		.msg_namelen = 0,
		.msg_control = NULL,
		.msg_controllen = 0,
		.msg_iov = vq->iov,
		.msg_flags = MSG_DONTWAIT,
	};
	size_t len, total_len = 0;
	int err, wmem;
	size_t hdr_size;
	struct socket *sock = rcu_dereference(vq->private_data);
	if (!sock)
		return;

	wmem = atomic_read(&sock->sk->sk_wmem_alloc);
	if (wmem >= sock->sk->sk_sndbuf) {
		mutex_lock(&vq->mutex);
		tx_poll_start(net, sock);
		mutex_unlock(&vq->mutex);
		return;
	}

	use_mm(net->dev.mm);
	mutex_lock(&vq->mutex);
	vhost_disable_notify(vq);

	if (wmem < sock->sk->sk_sndbuf / 2)
		tx_poll_stop(net);
	hdr_size = vq->hdr_size;

	for (;;) {
		head = vhost_get_vq_desc(&net->dev, vq, vq->iov,
					 ARRAY_SIZE(vq->iov),
					 &out, &in,
					 NULL, NULL);
		/* On error, stop handling until the next kick. */
		if (unlikely(head < 0))
			break;
		/* Nothing new?  Wait for eventfd to tell us they refilled. */
		if (head == vq->num) {
			wmem = atomic_read(&sock->sk->sk_wmem_alloc);
			if (wmem >= sock->sk->sk_sndbuf * 3 / 4) {
				tx_poll_start(net, sock);
				set_bit(SOCK_ASYNC_NOSPACE, &sock->flags);
				break;
			}
			if (unlikely(vhost_enable_notify(vq))) {
				vhost_disable_notify(vq);
				continue;
			}
			break;
		}
		if (in) {
			vq_err(vq, "Unexpected descriptor format for TX: "
			       "out %d, int %d\n", out, in);
			break;
		}
		/* Skip header. TODO: support TSO. */
		s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, out);
		msg.msg_iovlen = out;
		len = iov_length(vq->iov, out);
		/* Sanity check */
		if (!len) {
			vq_err(vq, "Unexpected header len for TX: "
			       "%zd expected %zd\n",
			       iov_length(vq->hdr, s), hdr_size);
			break;
		}
		/* TODO: Check specific error and bomb out unless ENOBUFS? */
		err = sock->ops->sendmsg(NULL, sock, &msg, len);
		if (unlikely(err < 0)) {
			vhost_discard_vq_desc(vq);
			tx_poll_start(net, sock);
			break;
		}
		if (err != len)
			pr_err("Truncated TX packet: "
			       " len %d != %zd\n", err, len);
		vhost_add_used_and_signal(&net->dev, vq, head, 0);
		total_len += len;
		if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
			vhost_poll_queue(&vq->poll);
			break;
		}
	}

	mutex_unlock(&vq->mutex);
	unuse_mm(net->dev.mm);
}

/* Expects to be always run from workqueue - which acts as
 * read-size critical section for our kind of RCU. */
static void handle_rx(struct vhost_net *net)
{
	struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX];
	unsigned out, in, log, s;
	int head;
	struct vhost_log *vq_log;
	struct msghdr msg = {
		.msg_name = NULL,
		.msg_namelen = 0,
		.msg_control = NULL, /* FIXME: get and handle RX aux data. */
		.msg_controllen = 0,
		.msg_iov = vq->iov,
		.msg_flags = MSG_DONTWAIT,
	};

	struct virtio_net_hdr hdr = {
		.flags = 0,
		.gso_type = VIRTIO_NET_HDR_GSO_NONE
	};

	size_t len, total_len = 0;
	int err;
	size_t hdr_size;
	struct socket *sock = rcu_dereference(vq->private_data);
	if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue))
		return;

	use_mm(net->dev.mm);
	mutex_lock(&vq->mutex);
	vhost_disable_notify(vq);
	hdr_size = vq->hdr_size;

	vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ?
		vq->log : NULL;

	for (;;) {
		head = vhost_get_vq_desc(&net->dev, vq, vq->iov,
					 ARRAY_SIZE(vq->iov),
					 &out, &in,
					 vq_log, &log);
		/* On error, stop handling until the next kick. */
		if (unlikely(head < 0))
			break;
		/* OK, now we need to know about added descriptors. */
		if (head == vq->num) {
			if (unlikely(vhost_enable_notify(vq))) {
				/* They have slipped one in as we were
				 * doing that: check again. */
				vhost_disable_notify(vq);
				continue;
			}
			/* Nothing new?  Wait for eventfd to tell us
			 * they refilled. */
			break;
		}
		/* We don't need to be notified again. */
		if (out) {
			vq_err(vq, "Unexpected descriptor format for RX: "
			       "out %d, int %d\n",
			       out, in);
			break;
		}
		/* Skip header. TODO: support TSO/mergeable rx buffers. */
		s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, in);
		msg.msg_iovlen = in;
		len = iov_length(vq->iov, in);
		/* Sanity check */
		if (!len) {
			vq_err(vq, "Unexpected header len for RX: "
			       "%zd expected %zd\n",
			       iov_length(vq->hdr, s), hdr_size);
			break;
		}
		err = sock->ops->recvmsg(NULL, sock, &msg,
					 len, MSG_DONTWAIT | MSG_TRUNC);
		/* TODO: Check specific error and bomb out unless EAGAIN? */
		if (err < 0) {
			vhost_discard_vq_desc(vq);
			break;
		}
		/* TODO: Should check and handle checksum. */
		if (err > len) {
			pr_err("Discarded truncated rx packet: "
			       " len %d > %zd\n", err, len);
			vhost_discard_vq_desc(vq);
			continue;
		}
		len = err;
		err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size);
		if (err) {
			vq_err(vq, "Unable to write vnet_hdr at addr %p: %d\n",
			       vq->iov->iov_base, err);
			break;
		}
		len += hdr_size;
		vhost_add_used_and_signal(&net->dev, vq, head, len);
		if (unlikely(vq_log))
			vhost_log_write(vq, vq_log, log, len);
		total_len += len;
		if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
			vhost_poll_queue(&vq->poll);
			break;
		}
	}

	mutex_unlock(&vq->mutex);
	unuse_mm(net->dev.mm);
}

static void handle_tx_kick(struct work_struct *work)
{
	struct vhost_virtqueue *vq;
	struct vhost_net *net;
	vq = container_of(work, struct vhost_virtqueue, poll.work);
	net = container_of(vq->dev, struct vhost_net, dev);
	handle_tx(net);
}

static void handle_rx_kick(struct work_struct *work)
{
	struct vhost_virtqueue *vq;
	struct vhost_net *net;
	vq = container_of(work, struct vhost_virtqueue, poll.work);
	net = container_of(vq->dev, struct vhost_net, dev);
	handle_rx(net);
}

static void handle_tx_net(struct work_struct *work)
{
	struct vhost_net *net;
	net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_TX].work);
	handle_tx(net);
}

static void handle_rx_net(struct work_struct *work)
{
	struct vhost_net *net;
	net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_RX].work);
	handle_rx(net);
}

static int vhost_net_open(struct inode *inode, struct file *f)
{
	struct vhost_net *n = kmalloc(sizeof *n, GFP_KERNEL);
	int r;
	if (!n)
		return -ENOMEM;
	n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick;
	n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick;
	r = vhost_dev_init(&n->dev, n->vqs, VHOST_NET_VQ_MAX);
	if (r < 0) {
		kfree(n);
		return r;
	}

	vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT);
	vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN);
	n->tx_poll_state = VHOST_NET_POLL_DISABLED;

	f->private_data = n;

	return 0;
}

static void vhost_net_disable_vq(struct vhost_net *n,
				 struct vhost_virtqueue *vq)
{
	if (!vq->private_data)
		return;
	if (vq == n->vqs + VHOST_NET_VQ_TX) {
		tx_poll_stop(n);
		n->tx_poll_state = VHOST_NET_POLL_DISABLED;
	} else
		vhost_poll_stop(n->poll + VHOST_NET_VQ_RX);
}

static void vhost_net_enable_vq(struct vhost_net *n,
				struct vhost_virtqueue *vq)
{
	struct socket *sock = vq->private_data;
	if (!sock)
		return;
	if (vq == n->vqs + VHOST_NET_VQ_TX) {
		n->tx_poll_state = VHOST_NET_POLL_STOPPED;
		tx_poll_start(n, sock);
	} else
		vhost_poll_start(n->poll + VHOST_NET_VQ_RX, sock->file);
}

static struct socket *vhost_net_stop_vq(struct vhost_net *n,
					struct vhost_virtqueue *vq)
{
	struct socket *sock;

	mutex_lock(&vq->mutex);
	sock = vq->private_data;
	vhost_net_disable_vq(n, vq);
	rcu_assign_pointer(vq->private_data, NULL);
	mutex_unlock(&vq->mutex);
	return sock;
}

static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
			   struct socket **rx_sock)
{
	*tx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_TX);
	*rx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_RX);
}

static void vhost_net_flush_vq(struct vhost_net *n, int index)
{
	vhost_poll_flush(n->poll + index);
	vhost_poll_flush(&n->dev.vqs[index].poll);
}

static void vhost_net_flush(struct vhost_net *n)
{
	vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
	vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
}

static int vhost_net_release(struct inode *inode, struct file *f)
{
	struct vhost_net *n = f->private_data;
	struct socket *tx_sock;
	struct socket *rx_sock;

	vhost_net_stop(n, &tx_sock, &rx_sock);
	vhost_net_flush(n);
	vhost_dev_cleanup(&n->dev);
	if (tx_sock)
		fput(tx_sock->file);
	if (rx_sock)
		fput(rx_sock->file);
	/* We do an extra flush before freeing memory,
	 * since jobs can re-queue themselves. */
	vhost_net_flush(n);
	kfree(n);
	return 0;
}

static struct socket *get_raw_socket(int fd)
{
	struct {
		struct sockaddr_ll sa;
		char  buf[MAX_ADDR_LEN];
	} uaddr;
	int uaddr_len = sizeof uaddr, r;
	struct socket *sock = sockfd_lookup(fd, &r);
	if (!sock)
		return ERR_PTR(-ENOTSOCK);

	/* Parameter checking */
	if (sock->sk->sk_type != SOCK_RAW) {
		r = -ESOCKTNOSUPPORT;
		goto err;
	}

	r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa,
			       &uaddr_len, 0);
	if (r)
		goto err;

	if (uaddr.sa.sll_family != AF_PACKET) {
		r = -EPFNOSUPPORT;
		goto err;
	}
	return sock;
err:
	fput(sock->file);
	return ERR_PTR(r);
}

static struct socket *get_tap_socket(int fd)
{
	struct file *file = fget(fd);
	struct socket *sock;
	if (!file)
		return ERR_PTR(-EBADF);
	sock = tun_get_socket(file);
	if (!IS_ERR(sock))
		return sock;
	sock = macvtap_get_socket(file);
	if (IS_ERR(sock))
		fput(file);
	return sock;
}

static struct socket *get_socket(int fd)
{
	struct socket *sock;
	/* special case to disable backend */
	if (fd == -1)
		return NULL;
	sock = get_raw_socket(fd);
	if (!IS_ERR(sock))
		return sock;
	sock = get_tap_socket(fd);
	if (!IS_ERR(sock))
		return sock;
	return ERR_PTR(-ENOTSOCK);
}

static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
{
	struct socket *sock, *oldsock;
	struct vhost_virtqueue *vq;
	int r;

	mutex_lock(&n->dev.mutex);
	r = vhost_dev_check_owner(&n->dev);
	if (r)
		goto err;

	if (index >= VHOST_NET_VQ_MAX) {
		r = -ENOBUFS;
		goto err;
	}
	vq = n->vqs + index;
	mutex_lock(&vq->mutex);

	/* Verify that ring has been setup correctly. */
	if (!vhost_vq_access_ok(vq)) {
		r = -EFAULT;
		goto err_vq;
	}
	sock = get_socket(fd);
	if (IS_ERR(sock)) {
		r = PTR_ERR(sock);
		goto err_vq;
	}

	/* start polling new socket */
	oldsock = vq->private_data;
	if (sock != oldsock){
                vhost_net_disable_vq(n, vq);
                rcu_assign_pointer(vq->private_data, sock);
                vhost_net_enable_vq(n, vq);
	}

	if (oldsock) {
		vhost_net_flush_vq(n, index);
		fput(oldsock->file);
	}

err_vq:
	mutex_unlock(&vq->mutex);
err:
	mutex_unlock(&n->dev.mutex);
	return r;
}

static long vhost_net_reset_owner(struct vhost_net *n)
{
	struct socket *tx_sock = NULL;
	struct socket *rx_sock = NULL;
	long err;
	mutex_lock(&n->dev.mutex);
	err = vhost_dev_check_owner(&n->dev);
	if (err)
		goto done;
	vhost_net_stop(n, &tx_sock, &rx_sock);
	vhost_net_flush(n);
	err = vhost_dev_reset_owner(&n->dev);
done:
	mutex_unlock(&n->dev.mutex);
	if (tx_sock)
		fput(tx_sock->file);
	if (rx_sock)
		fput(rx_sock->file);
	return err;
}

static int vhost_net_set_features(struct vhost_net *n, u64 features)
{
	size_t hdr_size = features & (1 << VHOST_NET_F_VIRTIO_NET_HDR) ?
		sizeof(struct virtio_net_hdr) : 0;
	int i;
	mutex_lock(&n->dev.mutex);
	if ((features & (1 << VHOST_F_LOG_ALL)) &&
	    !vhost_log_access_ok(&n->dev)) {
		mutex_unlock(&n->dev.mutex);
		return -EFAULT;
	}
	n->dev.acked_features = features;
	smp_wmb();
	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
		mutex_lock(&n->vqs[i].mutex);
		n->vqs[i].hdr_size = hdr_size;
		mutex_unlock(&n->vqs[i].mutex);
	}
	vhost_net_flush(n);
	mutex_unlock(&n->dev.mutex);
	return 0;
}

static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
			    unsigned long arg)
{
	struct vhost_net *n = f->private_data;
	void __user *argp = (void __user *)arg;
	u64 __user *featurep = argp;
	struct vhost_vring_file backend;
	u64 features;
	int r;
	switch (ioctl) {
	case VHOST_NET_SET_BACKEND:
		if (copy_from_user(&backend, argp, sizeof backend))
			return -EFAULT;
		return vhost_net_set_backend(n, backend.index, backend.fd);
	case VHOST_GET_FEATURES:
		features = VHOST_FEATURES;
		if (copy_to_user(featurep, &features, sizeof features))
			return -EFAULT;
		return 0;
	case VHOST_SET_FEATURES:
		if (copy_from_user(&features, featurep, sizeof features))
			return -EFAULT;
		if (features & ~VHOST_FEATURES)
			return -EOPNOTSUPP;
		return vhost_net_set_features(n, features);
	case VHOST_RESET_OWNER:
		return vhost_net_reset_owner(n);
	default:
		mutex_lock(&n->dev.mutex);
		r = vhost_dev_ioctl(&n->dev, ioctl, arg);
		vhost_net_flush(n);
		mutex_unlock(&n->dev.mutex);
		return r;
	}
}

#ifdef CONFIG_COMPAT
static long vhost_net_compat_ioctl(struct file *f, unsigned int ioctl,
				   unsigned long arg)
{
	return vhost_net_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
}
#endif

static const struct file_operations vhost_net_fops = {
	.owner          = THIS_MODULE,
	.release        = vhost_net_release,
	.unlocked_ioctl = vhost_net_ioctl,
#ifdef CONFIG_COMPAT
	.compat_ioctl   = vhost_net_compat_ioctl,
#endif
	.open           = vhost_net_open,
};

static struct miscdevice vhost_net_misc = {
	VHOST_NET_MINOR,
	"vhost-net",
	&vhost_net_fops,
};

static int vhost_net_init(void)
{
	int r = vhost_init();
	if (r)
		goto err_init;
	r = misc_register(&vhost_net_misc);
	if (r)
		goto err_reg;
	return 0;
err_reg:
	vhost_cleanup();
err_init:
	return r;

}
module_init(vhost_net_init);

static void vhost_net_exit(void)
{
	misc_deregister(&vhost_net_misc);
	vhost_cleanup();
}
module_exit(vhost_net_exit);

MODULE_VERSION("0.0.1");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Michael S. Tsirkin");
MODULE_DESCRIPTION("Host kernel accelerator for virtio net");
Commit	Line	Data
3a4d5c94 MT	1	/* Copyright (C) 2009 Red Hat, Inc.
	2	* Author: Michael S. Tsirkin <mst@redhat.com>
	3	*
	4	* This work is licensed under the terms of the GNU GPL, version 2.
	5	*
	6	* virtio-net server in host kernel.
	7	*/
	8
	9	#include <linux/compat.h>
	10	#include <linux/eventfd.h>
	11	#include <linux/vhost.h>
	12	#include <linux/virtio_net.h>
	13	#include <linux/mmu_context.h>
	14	#include <linux/miscdevice.h>
	15	#include <linux/module.h>
	16	#include <linux/mutex.h>
	17	#include <linux/workqueue.h>
	18	#include <linux/rcupdate.h>
	19	#include <linux/file.h>
5a0e3ad6	20	#include <linux/slab.h>
3a4d5c94 MT	21
	22	#include <linux/net.h>
	23	#include <linux/if_packet.h>
	24	#include <linux/if_arp.h>
	25	#include <linux/if_tun.h>
501c774c	26	#include <linux/if_macvlan.h>
3a4d5c94 MT	27
	28	#include <net/sock.h>
	29
	30	#include "vhost.h"
	31
	32	/* Max number of bytes transferred before requeueing the job.
	33	* Using this limit prevents one virtqueue from starving others. */
	34	#define VHOST_NET_WEIGHT 0x80000
	35
	36	enum {
	37	VHOST_NET_VQ_RX = 0,
	38	VHOST_NET_VQ_TX = 1,
	39	VHOST_NET_VQ_MAX = 2,
	40	};
	41
	42	enum vhost_net_poll_state {
	43	VHOST_NET_POLL_DISABLED = 0,
	44	VHOST_NET_POLL_STARTED = 1,
	45	VHOST_NET_POLL_STOPPED = 2,
	46	};
	47
	48	struct vhost_net {
	49	struct vhost_dev dev;
	50	struct vhost_virtqueue vqs[VHOST_NET_VQ_MAX];
	51	struct vhost_poll poll[VHOST_NET_VQ_MAX];
	52	/* Tells us whether we are polling a socket for TX.
	53	* We only do this when socket buffer fills up.
	54	* Protected by tx vq lock. */
	55	enum vhost_net_poll_state tx_poll_state;
	56	};
	57
	58	/* Pop first len bytes from iovec. Return number of segments used. */
	59	static int move_iovec_hdr(struct iovec from, struct iovec to,
	60	size_t len, int iov_count)
	61	{
	62	int seg = 0;
	63	size_t size;
	64	while (len && seg < iov_count) {
	65	size = min(from->iov_len, len);
	66	to->iov_base = from->iov_base;
	67	to->iov_len = size;
	68	from->iov_len -= size;
	69	from->iov_base += size;
	70	len -= size;
	71	++from;
	72	++to;
	73	++seg;
	74	}
	75	return seg;
	76	}
	77
	78	/* Caller must have TX VQ lock */
	79	static void tx_poll_stop(struct vhost_net *net)
	80	{
	81	if (likely(net->tx_poll_state != VHOST_NET_POLL_STARTED))
	82	return;
	83	vhost_poll_stop(net->poll + VHOST_NET_VQ_TX);
	84	net->tx_poll_state = VHOST_NET_POLL_STOPPED;
	85	}
	86
	87	/* Caller must have TX VQ lock */
	88	static void tx_poll_start(struct vhost_net net, struct socket sock)
	89	{
	90	if (unlikely(net->tx_poll_state != VHOST_NET_POLL_STOPPED))
91	return;
92	vhost_poll_start(net->poll + VHOST_NET_VQ_TX, sock->file);
93	net->tx_poll_state = VHOST_NET_POLL_STARTED;
94	}
95
96	/* Expects to be always run from workqueue - which acts as
97	* read-size critical section for our kind of RCU. */
98	static void handle_tx(struct vhost_net *net)
99	{
100	struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_TX];
d5675bd2 MT	101	unsigned out, in, s;
d5675bd2 MT	102	int head;
3a4d5c94 MT	103	struct msghdr msg = {
	104	.msg_name = NULL,
	105	.msg_namelen = 0,
	106	.msg_control = NULL,
	107	.msg_controllen = 0,
	108	.msg_iov = vq->iov,
	109	.msg_flags = MSG_DONTWAIT,
	110	};
	111	size_t len, total_len = 0;
	112	int err, wmem;
	113	size_t hdr_size;
	114	struct socket *sock = rcu_dereference(vq->private_data);
	115	if (!sock)
	116	return;
	117
	118	wmem = atomic_read(&sock->sk->sk_wmem_alloc);
39286fa4 SS	119	if (wmem >= sock->sk->sk_sndbuf) {
	120	mutex_lock(&vq->mutex);
	121	tx_poll_start(net, sock);
	122	mutex_unlock(&vq->mutex);
3a4d5c94	123	return;
39286fa4	124	}
3a4d5c94 MT	125
	126	use_mm(net->dev.mm);
	127	mutex_lock(&vq->mutex);
	128	vhost_disable_notify(vq);
	129
0e255572	130	if (wmem < sock->sk->sk_sndbuf / 2)
3a4d5c94 MT	131	tx_poll_stop(net);
	132	hdr_size = vq->hdr_size;
	133
	134	for (;;) {
	135	head = vhost_get_vq_desc(&net->dev, vq, vq->iov,
	136	ARRAY_SIZE(vq->iov),
	137	&out, &in,
	138	NULL, NULL);
d5675bd2	139	/* On error, stop handling until the next kick. */
7b3384fc	140	if (unlikely(head < 0))
d5675bd2	141	break;
3a4d5c94 MT	142	/* Nothing new? Wait for eventfd to tell us they refilled. */
	143	if (head == vq->num) {
	144	wmem = atomic_read(&sock->sk->sk_wmem_alloc);
	145	if (wmem >= sock->sk->sk_sndbuf * 3 / 4) {
	146	tx_poll_start(net, sock);
	147	set_bit(SOCK_ASYNC_NOSPACE, &sock->flags);
	148	break;
	149	}
	150	if (unlikely(vhost_enable_notify(vq))) {
	151	vhost_disable_notify(vq);
	152	continue;
	153	}
	154	break;
	155	}
	156	if (in) {
	157	vq_err(vq, "Unexpected descriptor format for TX: "
	158	"out %d, int %d\n", out, in);
	159	break;
	160	}
	161	/* Skip header. TODO: support TSO. */
	162	s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, out);
	163	msg.msg_iovlen = out;
	164	len = iov_length(vq->iov, out);
	165	/* Sanity check */
	166	if (!len) {
	167	vq_err(vq, "Unexpected header len for TX: "
	168	"%zd expected %zd\n",
	169	iov_length(vq->hdr, s), hdr_size);
	170	break;
	171	}
	172	/* TODO: Check specific error and bomb out unless ENOBUFS? */
	173	err = sock->ops->sendmsg(NULL, sock, &msg, len);
	174	if (unlikely(err < 0)) {
	175	vhost_discard_vq_desc(vq);
	176	tx_poll_start(net, sock);
	177	break;
	178	}
	179	if (err != len)
	180	pr_err("Truncated TX packet: "
	181	" len %d != %zd\n", err, len);
	182	vhost_add_used_and_signal(&net->dev, vq, head, 0);
	183	total_len += len;
	184	if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
	185	vhost_poll_queue(&vq->poll);
	186	break;
	187	}
	188	}
	189
	190	mutex_unlock(&vq->mutex);
	191	unuse_mm(net->dev.mm);
	192	}
	193
	194	/* Expects to be always run from workqueue - which acts as
	195	* read-size critical section for our kind of RCU. */
	196	static void handle_rx(struct vhost_net *net)
	197	{
	198	struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX];
d5675bd2 MT	199	unsigned out, in, log, s;
d5675bd2 MT	200	int head;
3a4d5c94 MT	201	struct vhost_log *vq_log;
	202	struct msghdr msg = {
	203	.msg_name = NULL,
	204	.msg_namelen = 0,
	205	.msg_control = NULL, /* FIXME: get and handle RX aux data. */
	206	.msg_controllen = 0,
	207	.msg_iov = vq->iov,
	208	.msg_flags = MSG_DONTWAIT,
	209	};
	210
	211	struct virtio_net_hdr hdr = {
	212	.flags = 0,
	213	.gso_type = VIRTIO_NET_HDR_GSO_NONE
	214	};
	215
	216	size_t len, total_len = 0;
	217	int err;
	218	size_t hdr_size;
	219	struct socket *sock = rcu_dereference(vq->private_data);
	220	if (!sock \|\| skb_queue_empty(&sock->sk->sk_receive_queue))
	221	return;
	222
	223	use_mm(net->dev.mm);
	224	mutex_lock(&vq->mutex);
	225	vhost_disable_notify(vq);
	226	hdr_size = vq->hdr_size;
	227
	228	vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ?
	229	vq->log : NULL;
	230
	231	for (;;) {
	232	head = vhost_get_vq_desc(&net->dev, vq, vq->iov,
	233	ARRAY_SIZE(vq->iov),
	234	&out, &in,
	235	vq_log, &log);
d5675bd2	236	/* On error, stop handling until the next kick. */
7b3384fc	237	if (unlikely(head < 0))
d5675bd2	238	break;
3a4d5c94 MT	239	/* OK, now we need to know about added descriptors. */
	240	if (head == vq->num) {
	241	if (unlikely(vhost_enable_notify(vq))) {
	242	/* They have slipped one in as we were
	243	* doing that: check again. */
	244	vhost_disable_notify(vq);
	245	continue;
	246	}
	247	/* Nothing new? Wait for eventfd to tell us
	248	* they refilled. */
	249	break;
	250	}
	251	/* We don't need to be notified again. */
	252	if (out) {
	253	vq_err(vq, "Unexpected descriptor format for RX: "
	254	"out %d, int %d\n",
	255	out, in);
	256	break;
	257	}
	258	/* Skip header. TODO: support TSO/mergeable rx buffers. */
	259	s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, in);
	260	msg.msg_iovlen = in;
	261	len = iov_length(vq->iov, in);
	262	/* Sanity check */
	263	if (!len) {
	264	vq_err(vq, "Unexpected header len for RX: "
	265	"%zd expected %zd\n",
	266	iov_length(vq->hdr, s), hdr_size);
	267	break;
	268	}
	269	err = sock->ops->recvmsg(NULL, sock, &msg,
	270	len, MSG_DONTWAIT \| MSG_TRUNC);
	271	/* TODO: Check specific error and bomb out unless EAGAIN? */
	272	if (err < 0) {
	273	vhost_discard_vq_desc(vq);
	274	break;
	275	}
	276	/* TODO: Should check and handle checksum. */
	277	if (err > len) {
	278	pr_err("Discarded truncated rx packet: "
	279	" len %d > %zd\n", err, len);
	280	vhost_discard_vq_desc(vq);
	281	continue;
	282	}
	283	len = err;
	284	err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size);
	285	if (err) {
	286	vq_err(vq, "Unable to write vnet_hdr at addr %p: %d\n",
	287	vq->iov->iov_base, err);
	288	break;
	289	}
	290	len += hdr_size;
	291	vhost_add_used_and_signal(&net->dev, vq, head, len);
	292	if (unlikely(vq_log))
	293	vhost_log_write(vq, vq_log, log, len);
	294	total_len += len;
	295	if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
	296	vhost_poll_queue(&vq->poll);
	297	break;
	298	}
	299	}
	300
	301	mutex_unlock(&vq->mutex);
	302	unuse_mm(net->dev.mm);
303	}
304
305	static void handle_tx_kick(struct work_struct *work)
306	{
307	struct vhost_virtqueue *vq;
308	struct vhost_net *net;
309	vq = container_of(work, struct vhost_virtqueue, poll.work);
310	net = container_of(vq->dev, struct vhost_net, dev);
311	handle_tx(net);
312	}
313
314	static void handle_rx_kick(struct work_struct *work)
315	{
316	struct vhost_virtqueue *vq;
317	struct vhost_net *net;
318	vq = container_of(work, struct vhost_virtqueue, poll.work);
319	net = container_of(vq->dev, struct vhost_net, dev);
320	handle_rx(net);
321	}
322
323	static void handle_tx_net(struct work_struct *work)
324	{
325	struct vhost_net *net;
326	net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_TX].work);
327	handle_tx(net);
328	}
329
330	static void handle_rx_net(struct work_struct *work)
331	{
332	struct vhost_net *net;
333	net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_RX].work);
334	handle_rx(net);
335	}
336
337	static int vhost_net_open(struct inode inode, struct file f)
338	{
339	struct vhost_net n = kmalloc(sizeof n, GFP_KERNEL);
340	int r;
341	if (!n)
342	return -ENOMEM;
343	n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick;
344	n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick;
345	r = vhost_dev_init(&n->dev, n->vqs, VHOST_NET_VQ_MAX);
346	if (r < 0) {
347	kfree(n);
348	return r;
349	}
350
351	vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT);
352	vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN);
353	n->tx_poll_state = VHOST_NET_POLL_DISABLED;
354
355	f->private_data = n;
356
357	return 0;
358	}
359
360	static void vhost_net_disable_vq(struct vhost_net *n,
361	struct vhost_virtqueue *vq)
362	{
363	if (!vq->private_data)
364	return;
365	if (vq == n->vqs + VHOST_NET_VQ_TX) {
366	tx_poll_stop(n);
367	n->tx_poll_state = VHOST_NET_POLL_DISABLED;
368	} else
369	vhost_poll_stop(n->poll + VHOST_NET_VQ_RX);
370	}
371
372	static void vhost_net_enable_vq(struct vhost_net *n,
373	struct vhost_virtqueue *vq)
374	{
375	struct socket *sock = vq->private_data;
376	if (!sock)
377	return;
378	if (vq == n->vqs + VHOST_NET_VQ_TX) {
379	n->tx_poll_state = VHOST_NET_POLL_STOPPED;
380	tx_poll_start(n, sock);
381	} else
382	vhost_poll_start(n->poll + VHOST_NET_VQ_RX, sock->file);
383	}
384
385	static struct socket vhost_net_stop_vq(struct vhost_net n,
386	struct vhost_virtqueue *vq)
387	{
388	struct socket *sock;
389
390	mutex_lock(&vq->mutex);
391	sock = vq->private_data;
392	vhost_net_disable_vq(n, vq);
393	rcu_assign_pointer(vq->private_data, NULL);
394	mutex_unlock(&vq->mutex);
395	return sock;
396	}
397
398	static void vhost_net_stop(struct vhost_net n, struct socket *tx_sock,
399	struct socket **rx_sock)
400	{
401	*tx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_TX);
402	*rx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_RX);
403	}
404
405	static void vhost_net_flush_vq(struct vhost_net *n, int index)
406	{
407	vhost_poll_flush(n->poll + index);
408	vhost_poll_flush(&n->dev.vqs[index].poll);
409	}
410
411	static void vhost_net_flush(struct vhost_net *n)
412	{
413	vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
414	vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
415	}
416
417	static int vhost_net_release(struct inode inode, struct file f)
418	{
419	struct vhost_net *n = f->private_data;
420	struct socket *tx_sock;
421	struct socket *rx_sock;
422
423	vhost_net_stop(n, &tx_sock, &rx_sock);
424	vhost_net_flush(n);
425	vhost_dev_cleanup(&n->dev);
426	if (tx_sock)
427	fput(tx_sock->file);
428	if (rx_sock)
429	fput(rx_sock->file);
430	/* We do an extra flush before freeing memory,
431	* since jobs can re-queue themselves. */
432	vhost_net_flush(n);
433	kfree(n);
434	return 0;
435	}
436
437	static struct socket *get_raw_socket(int fd)
438	{
439	struct {
440	struct sockaddr_ll sa;
441	char buf[MAX_ADDR_LEN];
442	} uaddr;
443	int uaddr_len = sizeof uaddr, r;
444	struct socket *sock = sockfd_lookup(fd, &r);
445	if (!sock)
446	return ERR_PTR(-ENOTSOCK);
447
448	/* Parameter checking */
449	if (sock->sk->sk_type != SOCK_RAW) {
450	r = -ESOCKTNOSUPPORT;
451	goto err;
452	}
453
454	r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa,
455	&uaddr_len, 0);
456	if (r)
457	goto err;
458
459	if (uaddr.sa.sll_family != AF_PACKET) {
460	r = -EPFNOSUPPORT;
461	goto err;
462	}
463	return sock;
464	err:
465	fput(sock->file);
466	return ERR_PTR(r);
467	}
468
501c774c	469	static struct socket *get_tap_socket(int fd)
3a4d5c94 MT	470	{
	471	struct file *file = fget(fd);
	472	struct socket *sock;
	473	if (!file)
	474	return ERR_PTR(-EBADF);
	475	sock = tun_get_socket(file);
501c774c AB	476	if (!IS_ERR(sock))
	477	return sock;
	478	sock = macvtap_get_socket(file);
3a4d5c94 MT	479	if (IS_ERR(sock))
	480	fput(file);
	481	return sock;
	482	}
	483
	484	static struct socket *get_socket(int fd)
	485	{
	486	struct socket *sock;
	487	/* special case to disable backend */
	488	if (fd == -1)
	489	return NULL;
	490	sock = get_raw_socket(fd);
	491	if (!IS_ERR(sock))
	492	return sock;
501c774c	493	sock = get_tap_socket(fd);
3a4d5c94 MT	494	if (!IS_ERR(sock))
	495	return sock;
	496	return ERR_PTR(-ENOTSOCK);
	497	}
	498
	499	static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
	500	{
	501	struct socket sock, oldsock;
	502	struct vhost_virtqueue *vq;
	503	int r;
	504
	505	mutex_lock(&n->dev.mutex);
	506	r = vhost_dev_check_owner(&n->dev);
	507	if (r)
	508	goto err;
	509
	510	if (index >= VHOST_NET_VQ_MAX) {
	511	r = -ENOBUFS;
	512	goto err;
	513	}
	514	vq = n->vqs + index;
	515	mutex_lock(&vq->mutex);
	516
	517	/* Verify that ring has been setup correctly. */
	518	if (!vhost_vq_access_ok(vq)) {
	519	r = -EFAULT;
1dace8c8	520	goto err_vq;
3a4d5c94 MT	521	}
	522	sock = get_socket(fd);
	523	if (IS_ERR(sock)) {
	524	r = PTR_ERR(sock);
1dace8c8	525	goto err_vq;
3a4d5c94 MT	526	}
	527
	528	/* start polling new socket */
	529	oldsock = vq->private_data;
dd1f4078 JD	530	if (sock != oldsock){
	531	vhost_net_disable_vq(n, vq);
	532	rcu_assign_pointer(vq->private_data, sock);
	533	vhost_net_enable_vq(n, vq);
	534	}
3a4d5c94	535
3a4d5c94 MT	536	if (oldsock) {
	537	vhost_net_flush_vq(n, index);
	538	fput(oldsock->file);
	539	}
1dace8c8 JD	540
	541	err_vq:
	542	mutex_unlock(&vq->mutex);
3a4d5c94 MT	543	err:
	544	mutex_unlock(&n->dev.mutex);
	545	return r;
	546	}
	547
	548	static long vhost_net_reset_owner(struct vhost_net *n)
	549	{
	550	struct socket *tx_sock = NULL;
	551	struct socket *rx_sock = NULL;
	552	long err;
	553	mutex_lock(&n->dev.mutex);
	554	err = vhost_dev_check_owner(&n->dev);
	555	if (err)
	556	goto done;
	557	vhost_net_stop(n, &tx_sock, &rx_sock);
	558	vhost_net_flush(n);
	559	err = vhost_dev_reset_owner(&n->dev);
	560	done:
	561	mutex_unlock(&n->dev.mutex);
	562	if (tx_sock)
	563	fput(tx_sock->file);
	564	if (rx_sock)
	565	fput(rx_sock->file);
	566	return err;
	567	}
	568
	569	static int vhost_net_set_features(struct vhost_net *n, u64 features)
	570	{
	571	size_t hdr_size = features & (1 << VHOST_NET_F_VIRTIO_NET_HDR) ?
	572	sizeof(struct virtio_net_hdr) : 0;
	573	int i;
	574	mutex_lock(&n->dev.mutex);
	575	if ((features & (1 << VHOST_F_LOG_ALL)) &&
	576	!vhost_log_access_ok(&n->dev)) {
	577	mutex_unlock(&n->dev.mutex);
	578	return -EFAULT;
	579	}
	580	n->dev.acked_features = features;
	581	smp_wmb();
	582	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
	583	mutex_lock(&n->vqs[i].mutex);
	584	n->vqs[i].hdr_size = hdr_size;
	585	mutex_unlock(&n->vqs[i].mutex);
	586	}
	587	vhost_net_flush(n);
	588	mutex_unlock(&n->dev.mutex);
	589	return 0;
	590	}
	591
	592	static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
	593	unsigned long arg)
	594	{
	595	struct vhost_net *n = f->private_data;
	596	void __user argp = (void __user )arg;
	597	u64 __user *featurep = argp;
	598	struct vhost_vring_file backend;
	599	u64 features;
	600	int r;
	601	switch (ioctl) {
	602	case VHOST_NET_SET_BACKEND:
d3553a52 TY	603	if (copy_from_user(&backend, argp, sizeof backend))
d3553a52 TY	604	return -EFAULT;
3a4d5c94 MT	605	return vhost_net_set_backend(n, backend.index, backend.fd);
	606	case VHOST_GET_FEATURES:
	607	features = VHOST_FEATURES;
d3553a52 TY	608	if (copy_to_user(featurep, &features, sizeof features))
	609	return -EFAULT;
	610	return 0;
3a4d5c94	611	case VHOST_SET_FEATURES:
d3553a52 TY	612	if (copy_from_user(&features, featurep, sizeof features))
d3553a52 TY	613	return -EFAULT;
3a4d5c94 MT	614	if (features & ~VHOST_FEATURES)
	615	return -EOPNOTSUPP;
	616	return vhost_net_set_features(n, features);
	617	case VHOST_RESET_OWNER:
	618	return vhost_net_reset_owner(n);
	619	default:
	620	mutex_lock(&n->dev.mutex);
	621	r = vhost_dev_ioctl(&n->dev, ioctl, arg);
	622	vhost_net_flush(n);
	623	mutex_unlock(&n->dev.mutex);
	624	return r;
	625	}
	626	}
	627
	628	#ifdef CONFIG_COMPAT
	629	static long vhost_net_compat_ioctl(struct file *f, unsigned int ioctl,
	630	unsigned long arg)
	631	{
	632	return vhost_net_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
	633	}
	634	#endif
	635
373a83a6	636	static const struct file_operations vhost_net_fops = {
3a4d5c94 MT	637	.owner = THIS_MODULE,
	638	.release = vhost_net_release,
	639	.unlocked_ioctl = vhost_net_ioctl,
	640	#ifdef CONFIG_COMPAT
	641	.compat_ioctl = vhost_net_compat_ioctl,
	642	#endif
	643	.open = vhost_net_open,
	644	};
	645
	646	static struct miscdevice vhost_net_misc = {
	647	VHOST_NET_MINOR,
	648	"vhost-net",
	649	&vhost_net_fops,
	650	};
	651
a8d3782f	652	static int vhost_net_init(void)
3a4d5c94 MT	653	{
	654	int r = vhost_init();
	655	if (r)
	656	goto err_init;
	657	r = misc_register(&vhost_net_misc);
	658	if (r)
	659	goto err_reg;
	660	return 0;
	661	err_reg:
	662	vhost_cleanup();
	663	err_init:
	664	return r;
	665
	666	}
	667	module_init(vhost_net_init);
	668
a8d3782f	669	static void vhost_net_exit(void)
3a4d5c94 MT	670	{
	671	misc_deregister(&vhost_net_misc);
	672	vhost_cleanup();
	673	}
	674	module_exit(vhost_net_exit);
	675
	676	MODULE_VERSION("0.0.1");
	677	MODULE_LICENSE("GPL v2");
	678	MODULE_AUTHOR("Michael S. Tsirkin");
	679	MODULE_DESCRIPTION("Host kernel accelerator for virtio net");