drivers/vhost/net.c

   1 /* Copyright (C) 2009 Red Hat, Inc.
   2  * Author: Michael S. Tsirkin <mst@redhat.com>
   3  *
   4  * This work is licensed under the terms of the GNU GPL, version 2.
   5  *
   6  * virtio-net server in host kernel.
   7  */
   8
   9 #include <linux/compat.h>
  10 #include <linux/eventfd.h>
  11 #include <linux/vhost.h>
  12 #include <linux/virtio_net.h>
  13 #include <linux/miscdevice.h>
  14 #include <linux/module.h>
  15 #include <linux/moduleparam.h>
  16 #include <linux/mutex.h>
  17 #include <linux/workqueue.h>
  18 #include <linux/file.h>
  19 #include <linux/slab.h>
  20 #include <linux/sched/clock.h>
  21 #include <linux/sched/signal.h>
  22 #include <linux/vmalloc.h>
  23
  24 #include <linux/net.h>
  25 #include <linux/if_packet.h>
  26 #include <linux/if_arp.h>
  27 #include <linux/if_tun.h>
  28 #include <linux/if_macvlan.h>
  29 #include <linux/if_tap.h>
  30 #include <linux/if_vlan.h>
  31 #include <linux/skb_array.h>
  32 #include <linux/skbuff.h>
  33
  34 #include <net/sock.h>
  35
  36 #include "vhost.h"
  37
  38 static int experimental_zcopytx = 0;
  39 module_param(experimental_zcopytx, int, 0444);
  40 MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
  41                                        " 1 -Enable; 0 - Disable");
  42
  43 /* Max number of bytes transferred before requeueing the job.
  44  * Using this limit prevents one virtqueue from starving others. */
  45 #define VHOST_NET_WEIGHT 0x80000
  46
  47 /* Max number of packets transferred before requeueing the job.
  48  * Using this limit prevents one virtqueue from starving others with small
  49  * pkts.
  50  */
  51 #define VHOST_NET_PKT_WEIGHT 256
  52
  53 /* MAX number of TX used buffers for outstanding zerocopy */
  54 #define VHOST_MAX_PEND 128
  55 #define VHOST_GOODCOPY_LEN 256
  56
  57 /*
  58  * For transmit, used buffer len is unused; we override it to track buffer
  59  * status internally; used for zerocopy tx only.
  60  */
  61 /* Lower device DMA failed */
  62 #define VHOST_DMA_FAILED_LEN    ((__force __virtio32)3)
  63 /* Lower device DMA done */
  64 #define VHOST_DMA_DONE_LEN      ((__force __virtio32)2)
  65 /* Lower device DMA in progress */
  66 #define VHOST_DMA_IN_PROGRESS   ((__force __virtio32)1)
  67 /* Buffer unused */
  68 #define VHOST_DMA_CLEAR_LEN     ((__force __virtio32)0)
  69
  70 #define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN)
  71
  72 enum {
  73         VHOST_NET_FEATURES = VHOST_FEATURES |
  74                          (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
  75                          (1ULL << VIRTIO_NET_F_MRG_RXBUF) |
  76                          (1ULL << VIRTIO_F_IOMMU_PLATFORM)
  77 };
  78
  79 enum {
  80         VHOST_NET_VQ_RX = 0,
  81         VHOST_NET_VQ_TX = 1,
  82         VHOST_NET_VQ_MAX = 2,
  83 };
  84
  85 struct vhost_net_ubuf_ref {
  86         /* refcount follows semantics similar to kref:
  87          *  0: object is released
  88          *  1: no outstanding ubufs
  89          * >1: outstanding ubufs
  90          */
  91         atomic_t refcount;
  92         wait_queue_head_t wait;
  93         struct vhost_virtqueue *vq;
  94 };
  95
  96 #define VHOST_RX_BATCH 64
  97 struct vhost_net_buf {
  98         struct sk_buff **queue;
  99         int tail;
 100         int head;
 101 };
 102
 103 struct vhost_net_virtqueue {
 104         struct vhost_virtqueue vq;
 105         size_t vhost_hlen;
 106         size_t sock_hlen;
 107         /* vhost zerocopy support fields below: */
 108         /* last used idx for outstanding DMA zerocopy buffers */
 109         int upend_idx;
 110         /* first used idx for DMA done zerocopy buffers */
 111         int done_idx;
 112         /* an array of userspace buffers info */
 113         struct ubuf_info *ubuf_info;
 114         /* Reference counting for outstanding ubufs.
 115          * Protected by vq mutex. Writers must also take device mutex. */
 116         struct vhost_net_ubuf_ref *ubufs;
 117         struct skb_array *rx_array;
 118         struct vhost_net_buf rxq;
 119 };
 120
 121 struct vhost_net {
 122         struct vhost_dev dev;
 123         struct vhost_net_virtqueue vqs[VHOST_NET_VQ_MAX];
 124         struct vhost_poll poll[VHOST_NET_VQ_MAX];
 125         /* Number of TX recently submitted.
 126          * Protected by tx vq lock. */
 127         unsigned tx_packets;
 128         /* Number of times zerocopy TX recently failed.
 129          * Protected by tx vq lock. */
 130         unsigned tx_zcopy_err;
 131         /* Flush in progress. Protected by tx vq lock. */
 132         bool tx_flush;
 133 };
 134
 135 static unsigned vhost_net_zcopy_mask __read_mostly;
 136
 137 static void *vhost_net_buf_get_ptr(struct vhost_net_buf *rxq)
 138 {
 139         if (rxq->tail != rxq->head)
 140                 return rxq->queue[rxq->head];
 141         else
 142                 return NULL;
 143 }
 144
 145 static int vhost_net_buf_get_size(struct vhost_net_buf *rxq)
 146 {
 147         return rxq->tail - rxq->head;
 148 }
 149
 150 static int vhost_net_buf_is_empty(struct vhost_net_buf *rxq)
 151 {
 152         return rxq->tail == rxq->head;
 153 }
 154
 155 static void *vhost_net_buf_consume(struct vhost_net_buf *rxq)
 156 {
 157         void *ret = vhost_net_buf_get_ptr(rxq);
 158         ++rxq->head;
 159         return ret;
 160 }
 161
 162 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)
 163 {
 164         struct vhost_net_buf *rxq = &nvq->rxq;
 165
 166         rxq->head = 0;
 167         rxq->tail = skb_array_consume_batched(nvq->rx_array, rxq->queue,
 168                                               VHOST_RX_BATCH);
 169         return rxq->tail;
 170 }
 171
 172 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)
 173 {
 174         struct vhost_net_buf *rxq = &nvq->rxq;
 175
 176         if (nvq->rx_array && !vhost_net_buf_is_empty(rxq)) {
 177                 skb_array_unconsume(nvq->rx_array, rxq->queue + rxq->head,
 178                                     vhost_net_buf_get_size(rxq));
 179                 rxq->head = rxq->tail = 0;
 180         }
 181 }
 182
 183 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)
 184 {
 185         struct vhost_net_buf *rxq = &nvq->rxq;
 186
 187         if (!vhost_net_buf_is_empty(rxq))
 188                 goto out;
 189
 190         if (!vhost_net_buf_produce(nvq))
 191                 return 0;
 192
 193 out:
 194         return __skb_array_len_with_tag(vhost_net_buf_get_ptr(rxq));
 195 }
 196
 197 static void vhost_net_buf_init(struct vhost_net_buf *rxq)
 198 {
 199         rxq->head = rxq->tail = 0;
 200 }
 201
 202 static void vhost_net_enable_zcopy(int vq)
 203 {
 204         vhost_net_zcopy_mask |= 0x1 << vq;
 205 }
 206
 207 static struct vhost_net_ubuf_ref *
 208 vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy)
 209 {
 210         struct vhost_net_ubuf_ref *ubufs;
 211         /* No zero copy backend? Nothing to count. */
 212         if (!zcopy)
 213                 return NULL;
 214         ubufs = kmalloc(sizeof(*ubufs), GFP_KERNEL);
 215         if (!ubufs)
 216                 return ERR_PTR(-ENOMEM);
 217         atomic_set(&ubufs->refcount, 1);
 218         init_waitqueue_head(&ubufs->wait);
 219         ubufs->vq = vq;
 220         return ubufs;
 221 }
 222
 223 static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs)
 224 {
 225         int r = atomic_sub_return(1, &ubufs->refcount);
 226         if (unlikely(!r))
 227                 wake_up(&ubufs->wait);
 228         return r;
 229 }
 230
 231 static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs)
 232 {
 233         vhost_net_ubuf_put(ubufs);
 234         wait_event(ubufs->wait, !atomic_read(&ubufs->refcount));
 235 }
 236
 237 static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs)
 238 {
 239         vhost_net_ubuf_put_and_wait(ubufs);
 240         kfree(ubufs);
 241 }
 242
 243 static void vhost_net_clear_ubuf_info(struct vhost_net *n)
 244 {
 245         int i;
 246
 247         for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
 248                 kfree(n->vqs[i].ubuf_info);
 249                 n->vqs[i].ubuf_info = NULL;
 250         }
 251 }
 252
 253 static int vhost_net_set_ubuf_info(struct vhost_net *n)
 254 {
 255         bool zcopy;
 256         int i;
 257
 258         for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
 259                 zcopy = vhost_net_zcopy_mask & (0x1 << i);
 260                 if (!zcopy)
 261                         continue;
 262                 n->vqs[i].ubuf_info = kmalloc(sizeof(*n->vqs[i].ubuf_info) *
 263                                               UIO_MAXIOV, GFP_KERNEL);
 264                 if  (!n->vqs[i].ubuf_info)
 265                         goto err;
 266         }
 267         return 0;
 268
 269 err:
 270         vhost_net_clear_ubuf_info(n);
 271         return -ENOMEM;
 272 }
 273
 274 static void vhost_net_vq_reset(struct vhost_net *n)
 275 {
 276         int i;
 277
 278         vhost_net_clear_ubuf_info(n);
 279
 280         for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
 281                 n->vqs[i].done_idx = 0;
 282                 n->vqs[i].upend_idx = 0;
 283                 n->vqs[i].ubufs = NULL;
 284                 n->vqs[i].vhost_hlen = 0;
 285                 n->vqs[i].sock_hlen = 0;
 286                 vhost_net_buf_init(&n->vqs[i].rxq);
 287         }
 288
 289 }
 290
 291 static void vhost_net_tx_packet(struct vhost_net *net)
 292 {
 293         ++net->tx_packets;
 294         if (net->tx_packets < 1024)
 295                 return;
 296         net->tx_packets = 0;
 297         net->tx_zcopy_err = 0;
 298 }
 299
 300 static void vhost_net_tx_err(struct vhost_net *net)
 301 {
 302         ++net->tx_zcopy_err;
 303 }
 304
 305 static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
 306 {
 307         /* TX flush waits for outstanding DMAs to be done.
 308          * Don't start new DMAs.
 309          */
 310         return !net->tx_flush &&
 311                 net->tx_packets / 64 >= net->tx_zcopy_err;
 312 }
 313
 314 static bool vhost_sock_zcopy(struct socket *sock)
 315 {
 316         return unlikely(experimental_zcopytx) &&
 317                 sock_flag(sock->sk, SOCK_ZEROCOPY);
 318 }
 319
 320 /* In case of DMA done not in order in lower device driver for some reason.
 321  * upend_idx is used to track end of used idx, done_idx is used to track head
 322  * of used idx. Once lower device DMA done contiguously, we will signal KVM
 323  * guest used idx.
 324  */
 325 static void vhost_zerocopy_signal_used(struct vhost_net *net,
 326                                        struct vhost_virtqueue *vq)
 327 {
 328         struct vhost_net_virtqueue *nvq =
 329                 container_of(vq, struct vhost_net_virtqueue, vq);
 330         int i, add;
 331         int j = 0;
 332
 333         for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
 334                 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN)
 335                         vhost_net_tx_err(net);
 336                 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) {
 337                         vq->heads[i].len = VHOST_DMA_CLEAR_LEN;
 338                         ++j;
 339                 } else
 340                         break;
 341         }
 342         while (j) {
 343                 add = min(UIO_MAXIOV - nvq->done_idx, j);
 344                 vhost_add_used_and_signal_n(vq->dev, vq,
 345                                             &vq->heads[nvq->done_idx], add);
 346                 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
 347                 j -= add;
 348         }
 349 }
 350
 351 static void vhost_zerocopy_callback(struct ubuf_info *ubuf, bool success)
 352 {
 353         struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;
 354         struct vhost_virtqueue *vq = ubufs->vq;
 355         int cnt;
 356
 357         rcu_read_lock_bh();
 358
 359         /* set len to mark this desc buffers done DMA */
 360         vq->heads[ubuf->desc].len = success ?
 361                 VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN;
 362         cnt = vhost_net_ubuf_put(ubufs);
 363
 364         /*
 365          * Trigger polling thread if guest stopped submitting new buffers:
 366          * in this case, the refcount after decrement will eventually reach 1.
 367          * We also trigger polling periodically after each 16 packets
 368          * (the value 16 here is more or less arbitrary, it's tuned to trigger
 369          * less than 10% of times).
 370          */
 371         if (cnt <= 1 || !(cnt % 16))
 372                 vhost_poll_queue(&vq->poll);
 373
 374         rcu_read_unlock_bh();
 375 }
 376
 377 static inline unsigned long busy_clock(void)
 378 {
 379         return local_clock() >> 10;
 380 }
 381
 382 static bool vhost_can_busy_poll(struct vhost_dev *dev,
 383                                 unsigned long endtime)
 384 {
 385         return likely(!need_resched()) &&
 386                likely(!time_after(busy_clock(), endtime)) &&
 387                likely(!signal_pending(current)) &&
 388                !vhost_has_work(dev);
 389 }
 390
 391 static void vhost_net_disable_vq(struct vhost_net *n,
 392                                  struct vhost_virtqueue *vq)
 393 {
 394         struct vhost_net_virtqueue *nvq =
 395                 container_of(vq, struct vhost_net_virtqueue, vq);
 396         struct vhost_poll *poll = n->poll + (nvq - n->vqs);
 397         if (!vq->private_data)
 398                 return;
 399         vhost_poll_stop(poll);
 400 }
 401
 402 static int vhost_net_enable_vq(struct vhost_net *n,
 403                                 struct vhost_virtqueue *vq)
 404 {
 405         struct vhost_net_virtqueue *nvq =
 406                 container_of(vq, struct vhost_net_virtqueue, vq);
 407         struct vhost_poll *poll = n->poll + (nvq - n->vqs);
 408         struct socket *sock;
 409
 410         sock = vq->private_data;
 411         if (!sock)
 412                 return 0;
 413
 414         return vhost_poll_start(poll, sock->file);
 415 }
 416
 417 static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
 418                                     struct vhost_virtqueue *vq,
 419                                     struct iovec iov[], unsigned int iov_size,
 420                                     unsigned int *out_num, unsigned int *in_num)
 421 {
 422         unsigned long uninitialized_var(endtime);
 423         int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
 424                                   out_num, in_num, NULL, NULL);
 425
 426         if (r == vq->num && vq->busyloop_timeout) {
 427                 preempt_disable();
 428                 endtime = busy_clock() + vq->busyloop_timeout;
 429                 while (vhost_can_busy_poll(vq->dev, endtime) &&
 430                        vhost_vq_avail_empty(vq->dev, vq))
 431                         cpu_relax();
 432                 preempt_enable();
 433                 r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
 434                                       out_num, in_num, NULL, NULL);
 435         }
 436
 437         return r;
 438 }
 439
 440 static bool vhost_exceeds_maxpend(struct vhost_net *net)
 441 {
 442         struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
 443         struct vhost_virtqueue *vq = &nvq->vq;
 444
 445         return (nvq->upend_idx + vq->num - VHOST_MAX_PEND) % UIO_MAXIOV
 446                 == nvq->done_idx;
 447 }
 448
 449 /* Expects to be always run from workqueue - which acts as
 450  * read-size critical section for our kind of RCU. */
 451 static void handle_tx(struct vhost_net *net)
 452 {
 453         struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
 454         struct vhost_virtqueue *vq = &nvq->vq;
 455         unsigned out, in;
 456         int head;
 457         struct msghdr msg = {
 458                 .msg_name = NULL,
 459                 .msg_namelen = 0,
 460                 .msg_control = NULL,
 461                 .msg_controllen = 0,
 462                 .msg_flags = MSG_DONTWAIT,
 463         };
 464         size_t len, total_len = 0;
 465         int err;
 466         size_t hdr_size;
 467         struct socket *sock;
 468         struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
 469         struct ubuf_info *ubuf;
 470         bool zcopy, zcopy_used;
 471         int sent_pkts = 0;
 472
 473         mutex_lock(&vq->mutex);
 474         sock = vq->private_data;
 475         if (!sock)
 476                 goto out;
 477
 478         if (!vq_iotlb_prefetch(vq))
 479                 goto out;
 480
 481         vhost_disable_notify(&net->dev, vq);
 482
 483         hdr_size = nvq->vhost_hlen;
 484         zcopy = nvq->ubufs;
 485
 486         do {
 487                 /* Release DMAs done buffers first */
 488                 if (zcopy)
 489                         vhost_zerocopy_signal_used(net, vq);
 490
 491                 /* If more outstanding DMAs, queue the work.
 492                  * Handle upend_idx wrap around
 493                  */
 494                 if (unlikely(vhost_exceeds_maxpend(net)))
 495                         break;
 496
 497                 head = vhost_net_tx_get_vq_desc(net, vq, vq->iov,
 498                                                 ARRAY_SIZE(vq->iov),
 499                                                 &out, &in);
 500                 /* On error, stop handling until the next kick. */
 501                 if (unlikely(head < 0))
 502                         break;
 503                 /* Nothing new?  Wait for eventfd to tell us they refilled. */
 504                 if (head == vq->num) {
 505                         if (unlikely(vhost_enable_notify(&net->dev, vq))) {
 506                                 vhost_disable_notify(&net->dev, vq);
 507                                 continue;
 508                         }
 509                         break;
 510                 }
 511                 if (in) {
 512                         vq_err(vq, "Unexpected descriptor format for TX: "
 513                                "out %d, int %d\n", out, in);
 514                         break;
 515                 }
 516                 /* Skip header. TODO: support TSO. */
 517                 len = iov_length(vq->iov, out);
 518                 iov_iter_init(&msg.msg_iter, WRITE, vq->iov, out, len);
 519                 iov_iter_advance(&msg.msg_iter, hdr_size);
 520                 /* Sanity check */
 521                 if (!msg_data_left(&msg)) {
 522                         vq_err(vq, "Unexpected header len for TX: "
 523                                "%zd expected %zd\n",
 524                                len, hdr_size);
 525                         break;
 526                 }
 527                 len = msg_data_left(&msg);
 528
 529                 zcopy_used = zcopy && len >= VHOST_GOODCOPY_LEN
 530                                    && (nvq->upend_idx + 1) % UIO_MAXIOV !=
 531                                       nvq->done_idx
 532                                    && vhost_net_tx_select_zcopy(net);
 533
 534                 /* use msg_control to pass vhost zerocopy ubuf info to skb */
 535                 if (zcopy_used) {
 536                         ubuf = nvq->ubuf_info + nvq->upend_idx;
 537                         vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
 538                         vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
 539                         ubuf->callback = vhost_zerocopy_callback;
 540                         ubuf->ctx = nvq->ubufs;
 541                         ubuf->desc = nvq->upend_idx;
 542                         refcount_set(&ubuf->refcnt, 1);
 543                         msg.msg_control = ubuf;
 544                         msg.msg_controllen = sizeof(ubuf);
 545                         ubufs = nvq->ubufs;
 546                         atomic_inc(&ubufs->refcount);
 547                         nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
 548                 } else {
 549                         msg.msg_control = NULL;
 550                         ubufs = NULL;
 551                 }
 552                 total_len += len;
 553                 if (total_len < VHOST_NET_WEIGHT &&
 554                     !vhost_vq_avail_empty(&net->dev, vq) &&
 555                     likely(!vhost_exceeds_maxpend(net))) {
 556                         msg.msg_flags |= MSG_MORE;
 557                 } else {
 558                         msg.msg_flags &= ~MSG_MORE;
 559                 }
 560
 561                 /* TODO: Check specific error and bomb out unless ENOBUFS? */
 562                 err = sock->ops->sendmsg(sock, &msg, len);
 563                 if (unlikely(err < 0)) {
 564                         if (zcopy_used) {
 565                                 if (vq->heads[ubuf->desc].len == VHOST_DMA_IN_PROGRESS)
 566                                         vhost_net_ubuf_put(ubufs);
 567                                 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
 568                                         % UIO_MAXIOV;
 569                         }
 570                         vhost_discard_vq_desc(vq, 1);
 571                         break;
 572                 }
 573                 if (err != len)
 574                         pr_debug("Truncated TX packet: "
 575                                  " len %d != %zd\n", err, len);
 576                 if (!zcopy_used)
 577                         vhost_add_used_and_signal(&net->dev, vq, head, 0);
 578                 else
 579                         vhost_zerocopy_signal_used(net, vq);
 580                 vhost_net_tx_packet(net);
 581         } while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len)));
 582 out:
 583         mutex_unlock(&vq->mutex);
 584 }
 585
 586 static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk)
 587 {
 588         struct sk_buff *head;
 589         int len = 0;
 590         unsigned long flags;
 591
 592         if (rvq->rx_array)
 593                 return vhost_net_buf_peek(rvq);
 594
 595         spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
 596         head = skb_peek(&sk->sk_receive_queue);
 597         if (likely(head)) {
 598                 len = head->len;
 599                 if (skb_vlan_tag_present(head))
 600                         len += VLAN_HLEN;
 601         }
 602
 603         spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags);
 604         return len;
 605 }
 606
 607 static int sk_has_rx_data(struct sock *sk)
 608 {
 609         struct socket *sock = sk->sk_socket;
 610
 611         if (sock->ops->peek_len)
 612                 return sock->ops->peek_len(sock);
 613
 614         return skb_queue_empty(&sk->sk_receive_queue);
 615 }
 616
 617 static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk)
 618 {
 619         struct vhost_net_virtqueue *rvq = &net->vqs[VHOST_NET_VQ_RX];
 620         struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
 621         struct vhost_virtqueue *vq = &nvq->vq;
 622         unsigned long uninitialized_var(endtime);
 623         int len = peek_head_len(rvq, sk);
 624
 625         if (!len && vq->busyloop_timeout) {
 626                 /* Both tx vq and rx socket were polled here */
 627                 mutex_lock_nested(&vq->mutex, 1);
 628                 vhost_disable_notify(&net->dev, vq);
 629
 630                 preempt_disable();
 631                 endtime = busy_clock() + vq->busyloop_timeout;
 632
 633                 while (vhost_can_busy_poll(&net->dev, endtime) &&
 634                        !sk_has_rx_data(sk) &&
 635                        vhost_vq_avail_empty(&net->dev, vq))
 636                         cpu_relax();
 637
 638                 preempt_enable();
 639
 640                 if (!vhost_vq_avail_empty(&net->dev, vq))
 641                         vhost_poll_queue(&vq->poll);
 642                 else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
 643                         vhost_disable_notify(&net->dev, vq);
 644                         vhost_poll_queue(&vq->poll);
 645                 }
 646
 647                 mutex_unlock(&vq->mutex);
 648
 649                 len = peek_head_len(rvq, sk);
 650         }
 651
 652         return len;
 653 }
 654
 655 /* This is a multi-buffer version of vhost_get_desc, that works if
 656  *      vq has read descriptors only.
 657  * @vq          - the relevant virtqueue
 658  * @datalen     - data length we'll be reading
 659  * @iovcount    - returned count of io vectors we fill
 660  * @log         - vhost log
 661  * @log_num     - log offset
 662  * @quota       - headcount quota, 1 for big buffer
 663  *      returns number of buffer heads allocated, negative on error
 664  */
 665 static int get_rx_bufs(struct vhost_virtqueue *vq,
 666                        struct vring_used_elem *heads,
 667                        int datalen,
 668                        unsigned *iovcount,
 669                        struct vhost_log *log,
 670                        unsigned *log_num,
 671                        unsigned int quota)
 672 {
 673         unsigned int out, in;
 674         int seg = 0;
 675         int headcount = 0;
 676         unsigned d;
 677         int r, nlogs = 0;
 678         /* len is always initialized before use since we are always called with
 679          * datalen > 0.
 680          */
 681         u32 uninitialized_var(len);
 682
 683         while (datalen > 0 && headcount < quota) {
 684                 if (unlikely(seg >= UIO_MAXIOV)) {
 685                         r = -ENOBUFS;
 686                         goto err;
 687                 }
 688                 r = vhost_get_vq_desc(vq, vq->iov + seg,
 689                                       ARRAY_SIZE(vq->iov) - seg, &out,
 690                                       &in, log, log_num);
 691                 if (unlikely(r < 0))
 692                         goto err;
 693
 694                 d = r;
 695                 if (d == vq->num) {
 696                         r = 0;
 697                         goto err;
 698                 }
 699                 if (unlikely(out || in <= 0)) {
 700                         vq_err(vq, "unexpected descriptor format for RX: "
 701                                 "out %d, in %d\n", out, in);
 702                         r = -EINVAL;
 703                         goto err;
 704                 }
 705                 if (unlikely(log)) {
 706                         nlogs += *log_num;
 707                         log += *log_num;
 708                 }
 709                 heads[headcount].id = cpu_to_vhost32(vq, d);
 710                 len = iov_length(vq->iov + seg, in);
 711                 heads[headcount].len = cpu_to_vhost32(vq, len);
 712                 datalen -= len;
 713                 ++headcount;
 714                 seg += in;
 715         }
 716         heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen);
 717         *iovcount = seg;
 718         if (unlikely(log))
 719                 *log_num = nlogs;
 720
 721         /* Detect overrun */
 722         if (unlikely(datalen > 0)) {
 723                 r = UIO_MAXIOV + 1;
 724                 goto err;
 725         }
 726         return headcount;
 727 err:
 728         vhost_discard_vq_desc(vq, headcount);
 729         return r;
 730 }
 731
 732 /* Expects to be always run from workqueue - which acts as
 733  * read-size critical section for our kind of RCU. */
 734 static void handle_rx(struct vhost_net *net)
 735 {
 736         struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
 737         struct vhost_virtqueue *vq = &nvq->vq;
 738         unsigned uninitialized_var(in), log;
 739         struct vhost_log *vq_log;
 740         struct msghdr msg = {
 741                 .msg_name = NULL,
 742                 .msg_namelen = 0,
 743                 .msg_control = NULL, /* FIXME: get and handle RX aux data. */
 744                 .msg_controllen = 0,
 745                 .msg_flags = MSG_DONTWAIT,
 746         };
 747         struct virtio_net_hdr hdr = {
 748                 .flags = 0,
 749                 .gso_type = VIRTIO_NET_HDR_GSO_NONE
 750         };
 751         size_t total_len = 0;
 752         int err, mergeable;
 753         s16 headcount;
 754         size_t vhost_hlen, sock_hlen;
 755         size_t vhost_len, sock_len;
 756         struct socket *sock;
 757         struct iov_iter fixup;
 758         __virtio16 num_buffers;
 759         int recv_pkts = 0;
 760
 761         mutex_lock_nested(&vq->mutex, 0);
 762         sock = vq->private_data;
 763         if (!sock)
 764                 goto out;
 765
 766         if (!vq_iotlb_prefetch(vq))
 767                 goto out;
 768
 769         vhost_disable_notify(&net->dev, vq);
 770         vhost_net_disable_vq(net, vq);
 771
 772         vhost_hlen = nvq->vhost_hlen;
 773         sock_hlen = nvq->sock_hlen;
 774
 775         vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
 776                 vq->log : NULL;
 777         mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
 778
 779         do {
 780                 sock_len = vhost_net_rx_peek_head_len(net, sock->sk);
 781
 782                 if (!sock_len)
 783                         break;
 784                 sock_len += sock_hlen;
 785                 vhost_len = sock_len + vhost_hlen;
 786                 headcount = get_rx_bufs(vq, vq->heads, vhost_len,
 787                                         &in, vq_log, &log,
 788                                         likely(mergeable) ? UIO_MAXIOV : 1);
 789                 /* On error, stop handling until the next kick. */
 790                 if (unlikely(headcount < 0))
 791                         goto out;
 792                 /* OK, now we need to know about added descriptors. */
 793                 if (!headcount) {
 794                         if (unlikely(vhost_enable_notify(&net->dev, vq))) {
 795                                 /* They have slipped one in as we were
 796                                  * doing that: check again. */
 797                                 vhost_disable_notify(&net->dev, vq);
 798                                 continue;
 799                         }
 800                         /* Nothing new?  Wait for eventfd to tell us
 801                          * they refilled. */
 802                         goto out;
 803                 }
 804                 if (nvq->rx_array)
 805                         msg.msg_control = vhost_net_buf_consume(&nvq->rxq);
 806                 /* On overrun, truncate and discard */
 807                 if (unlikely(headcount > UIO_MAXIOV)) {
 808                         iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
 809                         err = sock->ops->recvmsg(sock, &msg,
 810                                                  1, MSG_DONTWAIT | MSG_TRUNC);
 811                         pr_debug("Discarded rx packet: len %zd\n", sock_len);
 812                         continue;
 813                 }
 814                 /* We don't need to be notified again. */
 815                 iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len);
 816                 fixup = msg.msg_iter;
 817                 if (unlikely((vhost_hlen))) {
 818                         /* We will supply the header ourselves
 819                          * TODO: support TSO.
 820                          */
 821                         iov_iter_advance(&msg.msg_iter, vhost_hlen);
 822                 }
 823                 err = sock->ops->recvmsg(sock, &msg,
 824                                          sock_len, MSG_DONTWAIT | MSG_TRUNC);
 825                 /* Userspace might have consumed the packet meanwhile:
 826                  * it's not supposed to do this usually, but might be hard
 827                  * to prevent. Discard data we got (if any) and keep going. */
 828                 if (unlikely(err != sock_len)) {
 829                         pr_debug("Discarded rx packet: "
 830                                  " len %d, expected %zd\n", err, sock_len);
 831                         vhost_discard_vq_desc(vq, headcount);
 832                         continue;
 833                 }
 834                 /* Supply virtio_net_hdr if VHOST_NET_F_VIRTIO_NET_HDR */
 835                 if (unlikely(vhost_hlen)) {
 836                         if (copy_to_iter(&hdr, sizeof(hdr),
 837                                          &fixup) != sizeof(hdr)) {
 838                                 vq_err(vq, "Unable to write vnet_hdr "
 839                                        "at addr %p\n", vq->iov->iov_base);
 840                                 goto out;
 841                         }
 842                 } else {
 843                         /* Header came from socket; we'll need to patch
 844                          * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF
 845                          */
 846                         iov_iter_advance(&fixup, sizeof(hdr));
 847                 }
 848                 /* TODO: Should check and handle checksum. */
 849
 850                 num_buffers = cpu_to_vhost16(vq, headcount);
 851                 if (likely(mergeable) &&
 852                     copy_to_iter(&num_buffers, sizeof num_buffers,
 853                                  &fixup) != sizeof num_buffers) {
 854                         vq_err(vq, "Failed num_buffers write");
 855                         vhost_discard_vq_desc(vq, headcount);
 856                         goto out;
 857                 }
 858                 vhost_add_used_and_signal_n(&net->dev, vq, vq->heads,
 859                                             headcount);
 860                 if (unlikely(vq_log))
 861                         vhost_log_write(vq, vq_log, log, vhost_len,
 862                                         vq->iov, in);
 863                 total_len += vhost_len;
 864         } while (likely(!vhost_exceeds_weight(vq, ++recv_pkts, total_len)));
 865
 866         vhost_net_enable_vq(net, vq);
 867 out:
 868         mutex_unlock(&vq->mutex);
 869 }
 870
 871 static void handle_tx_kick(struct vhost_work *work)
 872 {
 873         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
 874                                                   poll.work);
 875         struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
 876
 877         handle_tx(net);
 878 }
 879
 880 static void handle_rx_kick(struct vhost_work *work)
 881 {
 882         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
 883                                                   poll.work);
 884         struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
 885
 886         handle_rx(net);
 887 }
 888
 889 static void handle_tx_net(struct vhost_work *work)
 890 {
 891         struct vhost_net *net = container_of(work, struct vhost_net,
 892                                              poll[VHOST_NET_VQ_TX].work);
 893         handle_tx(net);
 894 }
 895
 896 static void handle_rx_net(struct vhost_work *work)
 897 {
 898         struct vhost_net *net = container_of(work, struct vhost_net,
 899                                              poll[VHOST_NET_VQ_RX].work);
 900         handle_rx(net);
 901 }
 902
 903 static int vhost_net_open(struct inode *inode, struct file *f)
 904 {
 905         struct vhost_net *n;
 906         struct vhost_dev *dev;
 907         struct vhost_virtqueue **vqs;
 908         struct sk_buff **queue;
 909         int i;
 910
 911         n = kvmalloc(sizeof *n, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
 912         if (!n)
 913                 return -ENOMEM;
 914         vqs = kmalloc(VHOST_NET_VQ_MAX * sizeof(*vqs), GFP_KERNEL);
 915         if (!vqs) {
 916                 kvfree(n);
 917                 return -ENOMEM;
 918         }
 919
 920         queue = kmalloc_array(VHOST_RX_BATCH, sizeof(struct sk_buff *),
 921                               GFP_KERNEL);
 922         if (!queue) {
 923                 kfree(vqs);
 924                 kvfree(n);
 925                 return -ENOMEM;
 926         }
 927         n->vqs[VHOST_NET_VQ_RX].rxq.queue = queue;
 928
 929         dev = &n->dev;
 930         vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq;
 931         vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq;
 932         n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;
 933         n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;
 934         for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
 935                 n->vqs[i].ubufs = NULL;
 936                 n->vqs[i].ubuf_info = NULL;
 937                 n->vqs[i].upend_idx = 0;
 938                 n->vqs[i].done_idx = 0;
 939                 n->vqs[i].vhost_hlen = 0;
 940                 n->vqs[i].sock_hlen = 0;
 941                 vhost_net_buf_init(&n->vqs[i].rxq);
 942         }
 943         vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX,
 944                        VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT);
 945
 946         vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev);
 947         vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev);
 948
 949         f->private_data = n;
 950
 951         return 0;
 952 }
 953
 954 static struct socket *vhost_net_stop_vq(struct vhost_net *n,
 955                                         struct vhost_virtqueue *vq)
 956 {
 957         struct socket *sock;
 958         struct vhost_net_virtqueue *nvq =
 959                 container_of(vq, struct vhost_net_virtqueue, vq);
 960
 961         mutex_lock(&vq->mutex);
 962         sock = vq->private_data;
 963         vhost_net_disable_vq(n, vq);
 964         vq->private_data = NULL;
 965         vhost_net_buf_unproduce(nvq);
 966         mutex_unlock(&vq->mutex);
 967         return sock;
 968 }
 969
 970 static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
 971                            struct socket **rx_sock)
 972 {
 973         *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq);
 974         *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq);
 975 }
 976
 977 static void vhost_net_flush_vq(struct vhost_net *n, int index)
 978 {
 979         vhost_poll_flush(n->poll + index);
 980         vhost_poll_flush(&n->vqs[index].vq.poll);
 981 }
 982
 983 static void vhost_net_flush(struct vhost_net *n)
 984 {
 985         vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
 986         vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
 987         if (n->vqs[VHOST_NET_VQ_TX].ubufs) {
 988                 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
 989                 n->tx_flush = true;
 990                 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
 991                 /* Wait for all lower device DMAs done. */
 992                 vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs);
 993                 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
 994                 n->tx_flush = false;
 995                 atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1);
 996                 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
 997         }
 998 }
 999
1000 static int vhost_net_release(struct inode *inode, struct file *f)
1001 {
1002         struct vhost_net *n = f->private_data;
1003         struct socket *tx_sock;
1004         struct socket *rx_sock;
1005
1006         vhost_net_stop(n, &tx_sock, &rx_sock);
1007         vhost_net_flush(n);
1008         vhost_dev_stop(&n->dev);
1009         vhost_dev_cleanup(&n->dev, false);
1010         vhost_net_vq_reset(n);
1011         if (tx_sock)
1012                 sockfd_put(tx_sock);
1013         if (rx_sock)
1014                 sockfd_put(rx_sock);
1015         /* Make sure no callbacks are outstanding */
1016         synchronize_rcu_bh();
1017         /* We do an extra flush before freeing memory,
1018          * since jobs can re-queue themselves. */
1019         vhost_net_flush(n);
1020         kfree(n->vqs[VHOST_NET_VQ_RX].rxq.queue);
1021         kfree(n->dev.vqs);
1022         kvfree(n);
1023         return 0;
1024 }
1025
1026 static struct socket *get_raw_socket(int fd)
1027 {
1028         int r;
1029         struct socket *sock = sockfd_lookup(fd, &r);
1030
1031         if (!sock)
1032                 return ERR_PTR(-ENOTSOCK);
1033
1034         /* Parameter checking */
1035         if (sock->sk->sk_type != SOCK_RAW) {
1036                 r = -ESOCKTNOSUPPORT;
1037                 goto err;
1038         }
1039
1040         if (sock->sk->sk_family != AF_PACKET) {
1041                 r = -EPFNOSUPPORT;
1042                 goto err;
1043         }
1044         return sock;
1045 err:
1046         sockfd_put(sock);
1047         return ERR_PTR(r);
1048 }
1049
1050 static struct skb_array *get_tap_skb_array(int fd)
1051 {
1052         struct skb_array *array;
1053         struct file *file = fget(fd);
1054
1055         if (!file)
1056                 return NULL;
1057         array = tun_get_skb_array(file);
1058         if (!IS_ERR(array))
1059                 goto out;
1060         array = tap_get_skb_array(file);
1061         if (!IS_ERR(array))
1062                 goto out;
1063         array = NULL;
1064 out:
1065         fput(file);
1066         return array;
1067 }
1068
1069 static struct socket *get_tap_socket(int fd)
1070 {
1071         struct file *file = fget(fd);
1072         struct socket *sock;
1073
1074         if (!file)
1075                 return ERR_PTR(-EBADF);
1076         sock = tun_get_socket(file);
1077         if (!IS_ERR(sock))
1078                 return sock;
1079         sock = tap_get_socket(file);
1080         if (IS_ERR(sock))
1081                 fput(file);
1082         return sock;
1083 }
1084
1085 static struct socket *get_socket(int fd)
1086 {
1087         struct socket *sock;
1088
1089         /* special case to disable backend */
1090         if (fd == -1)
1091                 return NULL;
1092         sock = get_raw_socket(fd);
1093         if (!IS_ERR(sock))
1094                 return sock;
1095         sock = get_tap_socket(fd);
1096         if (!IS_ERR(sock))
1097                 return sock;
1098         return ERR_PTR(-ENOTSOCK);
1099 }
1100
1101 static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
1102 {
1103         struct socket *sock, *oldsock;
1104         struct vhost_virtqueue *vq;
1105         struct vhost_net_virtqueue *nvq;
1106         struct vhost_net_ubuf_ref *ubufs, *oldubufs = NULL;
1107         int r;
1108
1109         mutex_lock(&n->dev.mutex);
1110         r = vhost_dev_check_owner(&n->dev);
1111         if (r)
1112                 goto err;
1113
1114         if (index >= VHOST_NET_VQ_MAX) {
1115                 r = -ENOBUFS;
1116                 goto err;
1117         }
1118         vq = &n->vqs[index].vq;
1119         nvq = &n->vqs[index];
1120         mutex_lock(&vq->mutex);
1121
1122         /* Verify that ring has been setup correctly. */
1123         if (!vhost_vq_access_ok(vq)) {
1124                 r = -EFAULT;
1125                 goto err_vq;
1126         }
1127         sock = get_socket(fd);
1128         if (IS_ERR(sock)) {
1129                 r = PTR_ERR(sock);
1130                 goto err_vq;
1131         }
1132
1133         /* start polling new socket */
1134         oldsock = vq->private_data;
1135         if (sock != oldsock) {
1136                 ubufs = vhost_net_ubuf_alloc(vq,
1137                                              sock && vhost_sock_zcopy(sock));
1138                 if (IS_ERR(ubufs)) {
1139                         r = PTR_ERR(ubufs);
1140                         goto err_ubufs;
1141                 }
1142
1143                 vhost_net_disable_vq(n, vq);
1144                 vq->private_data = sock;
1145                 vhost_net_buf_unproduce(nvq);
1146                 if (index == VHOST_NET_VQ_RX)
1147                         nvq->rx_array = get_tap_skb_array(fd);
1148                 r = vhost_vq_init_access(vq);
1149                 if (r)
1150                         goto err_used;
1151                 r = vhost_net_enable_vq(n, vq);
1152                 if (r)
1153                         goto err_used;
1154
1155                 oldubufs = nvq->ubufs;
1156                 nvq->ubufs = ubufs;
1157
1158                 n->tx_packets = 0;
1159                 n->tx_zcopy_err = 0;
1160                 n->tx_flush = false;
1161         }
1162
1163         mutex_unlock(&vq->mutex);
1164
1165         if (oldubufs) {
1166                 vhost_net_ubuf_put_wait_and_free(oldubufs);
1167                 mutex_lock(&vq->mutex);
1168                 vhost_zerocopy_signal_used(n, vq);
1169                 mutex_unlock(&vq->mutex);
1170         }
1171
1172         if (oldsock) {
1173                 vhost_net_flush_vq(n, index);
1174                 sockfd_put(oldsock);
1175         }
1176
1177         mutex_unlock(&n->dev.mutex);
1178         return 0;
1179
1180 err_used:
1181         vq->private_data = oldsock;
1182         vhost_net_enable_vq(n, vq);
1183         if (ubufs)
1184                 vhost_net_ubuf_put_wait_and_free(ubufs);
1185 err_ubufs:
1186         if (sock)
1187                 sockfd_put(sock);
1188 err_vq:
1189         mutex_unlock(&vq->mutex);
1190 err:
1191         mutex_unlock(&n->dev.mutex);
1192         return r;
1193 }
1194
1195 static long vhost_net_reset_owner(struct vhost_net *n)
1196 {
1197         struct socket *tx_sock = NULL;
1198         struct socket *rx_sock = NULL;
1199         long err;
1200         struct vhost_umem *umem;
1201
1202         mutex_lock(&n->dev.mutex);
1203         err = vhost_dev_check_owner(&n->dev);
1204         if (err)
1205                 goto done;
1206         umem = vhost_dev_reset_owner_prepare();
1207         if (!umem) {
1208                 err = -ENOMEM;
1209                 goto done;
1210         }
1211         vhost_net_stop(n, &tx_sock, &rx_sock);
1212         vhost_net_flush(n);
1213         vhost_dev_stop(&n->dev);
1214         vhost_dev_reset_owner(&n->dev, umem);
1215         vhost_net_vq_reset(n);
1216 done:
1217         mutex_unlock(&n->dev.mutex);
1218         if (tx_sock)
1219                 sockfd_put(tx_sock);
1220         if (rx_sock)
1221                 sockfd_put(rx_sock);
1222         return err;
1223 }
1224
1225 static int vhost_net_set_features(struct vhost_net *n, u64 features)
1226 {
1227         size_t vhost_hlen, sock_hlen, hdr_len;
1228         int i;
1229
1230         hdr_len = (features & ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
1231                                (1ULL << VIRTIO_F_VERSION_1))) ?
1232                         sizeof(struct virtio_net_hdr_mrg_rxbuf) :
1233                         sizeof(struct virtio_net_hdr);
1234         if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
1235                 /* vhost provides vnet_hdr */
1236                 vhost_hlen = hdr_len;
1237                 sock_hlen = 0;
1238         } else {
1239                 /* socket provides vnet_hdr */
1240                 vhost_hlen = 0;
1241                 sock_hlen = hdr_len;
1242         }
1243         mutex_lock(&n->dev.mutex);
1244         if ((features & (1 << VHOST_F_LOG_ALL)) &&
1245             !vhost_log_access_ok(&n->dev))
1246                 goto out_unlock;
1247
1248         if ((features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) {
1249                 if (vhost_init_device_iotlb(&n->dev, true))
1250                         goto out_unlock;
1251         }
1252
1253         for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
1254                 mutex_lock(&n->vqs[i].vq.mutex);
1255                 n->vqs[i].vq.acked_features = features;
1256                 n->vqs[i].vhost_hlen = vhost_hlen;
1257                 n->vqs[i].sock_hlen = sock_hlen;
1258                 mutex_unlock(&n->vqs[i].vq.mutex);
1259         }
1260         mutex_unlock(&n->dev.mutex);
1261         return 0;
1262
1263 out_unlock:
1264         mutex_unlock(&n->dev.mutex);
1265         return -EFAULT;
1266 }
1267
1268 static long vhost_net_set_owner(struct vhost_net *n)
1269 {
1270         int r;
1271
1272         mutex_lock(&n->dev.mutex);
1273         if (vhost_dev_has_owner(&n->dev)) {
1274                 r = -EBUSY;
1275                 goto out;
1276         }
1277         r = vhost_net_set_ubuf_info(n);
1278         if (r)
1279                 goto out;
1280         r = vhost_dev_set_owner(&n->dev);
1281         if (r)
1282                 vhost_net_clear_ubuf_info(n);
1283         vhost_net_flush(n);
1284 out:
1285         mutex_unlock(&n->dev.mutex);
1286         return r;
1287 }
1288
1289 static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
1290                             unsigned long arg)
1291 {
1292         struct vhost_net *n = f->private_data;
1293         void __user *argp = (void __user *)arg;
1294         u64 __user *featurep = argp;
1295         struct vhost_vring_file backend;
1296         u64 features;
1297         int r;
1298
1299         switch (ioctl) {
1300         case VHOST_NET_SET_BACKEND:
1301                 if (copy_from_user(&backend, argp, sizeof backend))
1302                         return -EFAULT;
1303                 return vhost_net_set_backend(n, backend.index, backend.fd);
1304         case VHOST_GET_FEATURES:
1305                 features = VHOST_NET_FEATURES;
1306                 if (copy_to_user(featurep, &features, sizeof features))
1307                         return -EFAULT;
1308                 return 0;
1309         case VHOST_SET_FEATURES:
1310                 if (copy_from_user(&features, featurep, sizeof features))
1311                         return -EFAULT;
1312                 if (features & ~VHOST_NET_FEATURES)
1313                         return -EOPNOTSUPP;
1314                 return vhost_net_set_features(n, features);
1315         case VHOST_RESET_OWNER:
1316                 return vhost_net_reset_owner(n);
1317         case VHOST_SET_OWNER:
1318                 return vhost_net_set_owner(n);
1319         default:
1320                 mutex_lock(&n->dev.mutex);
1321                 r = vhost_dev_ioctl(&n->dev, ioctl, argp);
1322                 if (r == -ENOIOCTLCMD)
1323                         r = vhost_vring_ioctl(&n->dev, ioctl, argp);
1324                 else
1325                         vhost_net_flush(n);
1326                 mutex_unlock(&n->dev.mutex);
1327                 return r;
1328         }
1329 }
1330
1331 #ifdef CONFIG_COMPAT
1332 static long vhost_net_compat_ioctl(struct file *f, unsigned int ioctl,
1333                                    unsigned long arg)
1334 {
1335         return vhost_net_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
1336 }
1337 #endif
1338
1339 static ssize_t vhost_net_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
1340 {
1341         struct file *file = iocb->ki_filp;
1342         struct vhost_net *n = file->private_data;
1343         struct vhost_dev *dev = &n->dev;
1344         int noblock = file->f_flags & O_NONBLOCK;
1345
1346         return vhost_chr_read_iter(dev, to, noblock);
1347 }
1348
1349 static ssize_t vhost_net_chr_write_iter(struct kiocb *iocb,
1350                                         struct iov_iter *from)
1351 {
1352         struct file *file = iocb->ki_filp;
1353         struct vhost_net *n = file->private_data;
1354         struct vhost_dev *dev = &n->dev;
1355
1356         return vhost_chr_write_iter(dev, from);
1357 }
1358
1359 static unsigned int vhost_net_chr_poll(struct file *file, poll_table *wait)
1360 {
1361         struct vhost_net *n = file->private_data;
1362         struct vhost_dev *dev = &n->dev;
1363
1364         return vhost_chr_poll(file, dev, wait);
1365 }
1366
1367 static const struct file_operations vhost_net_fops = {
1368         .owner          = THIS_MODULE,
1369         .release        = vhost_net_release,
1370         .read_iter      = vhost_net_chr_read_iter,
1371         .write_iter     = vhost_net_chr_write_iter,
1372         .poll           = vhost_net_chr_poll,
1373         .unlocked_ioctl = vhost_net_ioctl,
1374 #ifdef CONFIG_COMPAT
1375         .compat_ioctl   = vhost_net_compat_ioctl,
1376 #endif
1377         .open           = vhost_net_open,
1378         .llseek         = noop_llseek,
1379 };
1380
1381 static struct miscdevice vhost_net_misc = {
1382         .minor = VHOST_NET_MINOR,
1383         .name = "vhost-net",
1384         .fops = &vhost_net_fops,
1385 };
1386
1387 static int vhost_net_init(void)
1388 {
1389         if (experimental_zcopytx)
1390                 vhost_net_enable_zcopy(VHOST_NET_VQ_TX);
1391         return misc_register(&vhost_net_misc);
1392 }
1393 module_init(vhost_net_init);
1394
1395 static void vhost_net_exit(void)
1396 {
1397         misc_deregister(&vhost_net_misc);
1398 }
1399 module_exit(vhost_net_exit);
1400
1401 MODULE_VERSION("0.0.1");
1402 MODULE_LICENSE("GPL v2");
1403 MODULE_AUTHOR("Michael S. Tsirkin");
1404 MODULE_DESCRIPTION("Host kernel accelerator for virtio net");
1405 MODULE_ALIAS_MISCDEV(VHOST_NET_MINOR);
1406 MODULE_ALIAS("devname:vhost-net");