sys/dev/virtio/network/if_vtnet.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO network devices. */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #ifdef HAVE_KERNEL_OPTION_HEADERS
  33 #include "opt_device_polling.h"
  34 #endif
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/sockio.h>
  40 #include <sys/mbuf.h>
  41 #include <sys/malloc.h>
  42 #include <sys/module.h>
  43 #include <sys/socket.h>
  44 #include <sys/sysctl.h>
  45 #include <sys/random.h>
  46 #include <sys/sglist.h>
  47 #include <sys/lock.h>
  48 #include <sys/mutex.h>
  49
  50 #include <vm/uma.h>
  51
  52 #include <net/ethernet.h>
  53 #include <net/if.h>
  54 #include <net/if_arp.h>
  55 #include <net/if_dl.h>
  56 #include <net/if_types.h>
  57 #include <net/if_media.h>
  58 #include <net/if_vlan_var.h>
  59
  60 #include <net/bpf.h>
  61
  62 #include <netinet/in_systm.h>
  63 #include <netinet/in.h>
  64 #include <netinet/ip.h>
  65 #include <netinet/ip6.h>
  66 #include <netinet/udp.h>
  67 #include <netinet/tcp.h>
  68 #include <netinet/sctp.h>
  69
  70 #include <machine/bus.h>
  71 #include <machine/resource.h>
  72 #include <sys/bus.h>
  73 #include <sys/rman.h>
  74
  75 #include <dev/virtio/virtio.h>
  76 #include <dev/virtio/virtqueue.h>
  77 #include <dev/virtio/network/virtio_net.h>
  78 #include <dev/virtio/network/if_vtnetvar.h>
  79
  80 #include "virtio_if.h"
  81
  82 static int      vtnet_modevent(module_t, int, void *);
  83
  84 static int      vtnet_probe(device_t);
  85 static int      vtnet_attach(device_t);
  86 static int      vtnet_detach(device_t);
  87 static int      vtnet_suspend(device_t);
  88 static int      vtnet_resume(device_t);
  89 static int      vtnet_shutdown(device_t);
  90 static int      vtnet_config_change(device_t);
  91
  92 static void     vtnet_negotiate_features(struct vtnet_softc *);
  93 static int      vtnet_alloc_virtqueues(struct vtnet_softc *);
  94 static void     vtnet_get_hwaddr(struct vtnet_softc *);
  95 static void     vtnet_set_hwaddr(struct vtnet_softc *);
  96 static int      vtnet_is_link_up(struct vtnet_softc *);
  97 static void     vtnet_update_link_status(struct vtnet_softc *);
  98 static void     vtnet_watchdog(struct vtnet_softc *);
  99 static int      vtnet_change_mtu(struct vtnet_softc *, int);
 100 static int      vtnet_ioctl(struct ifnet *, u_long, caddr_t);
 101
 102 static int      vtnet_init_rx_vq(struct vtnet_softc *);
 103 static void     vtnet_free_rx_mbufs(struct vtnet_softc *);
 104 static void     vtnet_free_tx_mbufs(struct vtnet_softc *);
 105 static void     vtnet_free_ctrl_vq(struct vtnet_softc *);
 106
 107 #ifdef DEVICE_POLLING
 108 static poll_handler_t vtnet_poll;
 109 #endif
 110
 111 static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
 112                     struct mbuf **);
 113 static int      vtnet_replace_rxbuf(struct vtnet_softc *,
 114                     struct mbuf *, int);
 115 static int      vtnet_newbuf(struct vtnet_softc *);
 116 static void     vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
 117 static void     vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
 118 static int      vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
 119 static void     vtnet_vlan_tag_remove(struct mbuf *);
 120 static int      vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
 121                     struct virtio_net_hdr *);
 122 static int      vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
 123 static int      vtnet_rxeof(struct vtnet_softc *, int, int *);
 124 static void     vtnet_rx_vq_intr(void *);
 125
 126 static void     vtnet_txeof(struct vtnet_softc *);
 127 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
 128                     struct virtio_net_hdr *);
 129 static int      vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
 130                     struct vtnet_tx_header *);
 131 static int      vtnet_encap(struct vtnet_softc *, struct mbuf **);
 132 static void     vtnet_start_locked(struct ifnet *);
 133 static void     vtnet_start(struct ifnet *);
 134 static void     vtnet_tick(void *);
 135 static void     vtnet_tx_vq_intr(void *);
 136
 137 static void     vtnet_stop(struct vtnet_softc *);
 138 static int      vtnet_reinit(struct vtnet_softc *);
 139 static void     vtnet_init_locked(struct vtnet_softc *);
 140 static void     vtnet_init(void *);
 141
 142 static void     vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
 143                     struct sglist *, int, int);
 144
 145 static void     vtnet_rx_filter(struct vtnet_softc *sc);
 146 static int      vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
 147 static int      vtnet_set_promisc(struct vtnet_softc *, int);
 148 static int      vtnet_set_allmulti(struct vtnet_softc *, int);
 149 static void     vtnet_rx_filter_mac(struct vtnet_softc *);
 150
 151 static int      vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
 152 static void     vtnet_rx_filter_vlan(struct vtnet_softc *);
 153 static void     vtnet_set_vlan_filter(struct vtnet_softc *, int, uint16_t);
 154 static void     vtnet_register_vlan(void *, struct ifnet *, uint16_t);
 155 static void     vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
 156
 157 static int      vtnet_ifmedia_upd(struct ifnet *);
 158 static void     vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 159
 160 static void     vtnet_add_statistics(struct vtnet_softc *);
 161
 162 static int      vtnet_enable_rx_intr(struct vtnet_softc *);
 163 static int      vtnet_enable_tx_intr(struct vtnet_softc *);
 164 static void     vtnet_disable_rx_intr(struct vtnet_softc *);
 165 static void     vtnet_disable_tx_intr(struct vtnet_softc *);
 166
 167 /* Tunables. */
 168 static int vtnet_csum_disable = 0;
 169 TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
 170 static int vtnet_tso_disable = 0;
 171 TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
 172 static int vtnet_lro_disable = 0;
 173 TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
 174
 175 /*
 176  * Reducing the number of transmit completed interrupts can
 177  * improve performance. To do so, the define below keeps the
 178  * Tx vq interrupt disabled and adds calls to vtnet_txeof()
 179  * in the start and watchdog paths. The price to pay for this
 180  * is the m_free'ing of transmitted mbufs may be delayed until
 181  * the watchdog fires.
 182  */
 183 #define VTNET_TX_INTR_MODERATION
 184
 185 static uma_zone_t vtnet_tx_header_zone;
 186
 187 static struct virtio_feature_desc vtnet_feature_desc[] = {
 188         { VIRTIO_NET_F_CSUM,            "TxChecksum"    },
 189         { VIRTIO_NET_F_GUEST_CSUM,      "RxChecksum"    },
 190         { VIRTIO_NET_F_MAC,             "MacAddress"    },
 191         { VIRTIO_NET_F_GSO,             "TxAllGSO"      },
 192         { VIRTIO_NET_F_GUEST_TSO4,      "RxTSOv4"       },
 193         { VIRTIO_NET_F_GUEST_TSO6,      "RxTSOv6"       },
 194         { VIRTIO_NET_F_GUEST_ECN,       "RxECN"         },
 195         { VIRTIO_NET_F_GUEST_UFO,       "RxUFO"         },
 196         { VIRTIO_NET_F_HOST_TSO4,       "TxTSOv4"       },
 197         { VIRTIO_NET_F_HOST_TSO6,       "TxTSOv6"       },
 198         { VIRTIO_NET_F_HOST_ECN,        "TxTSOECN"      },
 199         { VIRTIO_NET_F_HOST_UFO,        "TxUFO"         },
 200         { VIRTIO_NET_F_MRG_RXBUF,       "MrgRxBuf"      },
 201         { VIRTIO_NET_F_STATUS,          "Status"        },
 202         { VIRTIO_NET_F_CTRL_VQ,         "ControlVq"     },
 203         { VIRTIO_NET_F_CTRL_RX,         "RxMode"        },
 204         { VIRTIO_NET_F_CTRL_VLAN,       "VLanFilter"    },
 205         { VIRTIO_NET_F_CTRL_RX_EXTRA,   "RxModeExtra"   },
 206
 207         { 0, NULL }
 208 };
 209
 210 static device_method_t vtnet_methods[] = {
 211         /* Device methods. */
 212         DEVMETHOD(device_probe,         vtnet_probe),
 213         DEVMETHOD(device_attach,        vtnet_attach),
 214         DEVMETHOD(device_detach,        vtnet_detach),
 215         DEVMETHOD(device_suspend,       vtnet_suspend),
 216         DEVMETHOD(device_resume,        vtnet_resume),
 217         DEVMETHOD(device_shutdown,      vtnet_shutdown),
 218
 219         /* VirtIO methods. */
 220         DEVMETHOD(virtio_config_change, vtnet_config_change),
 221
 222         DEVMETHOD_END
 223 };
 224
 225 static driver_t vtnet_driver = {
 226         "vtnet",
 227         vtnet_methods,
 228         sizeof(struct vtnet_softc)
 229 };
 230 static devclass_t vtnet_devclass;
 231
 232 DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass,
 233     vtnet_modevent, 0);
 234 MODULE_VERSION(vtnet, 1);
 235 MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
 236
 237 static int
 238 vtnet_modevent(module_t mod, int type, void *unused)
 239 {
 240         int error;
 241
 242         error = 0;
 243
 244         switch (type) {
 245         case MOD_LOAD:
 246                 vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr",
 247                     sizeof(struct vtnet_tx_header),
 248                     NULL, NULL, NULL, NULL, 0, 0);
 249                 break;
 250         case MOD_QUIESCE:
 251         case MOD_UNLOAD:
 252                 if (uma_zone_get_cur(vtnet_tx_header_zone) > 0)
 253                         error = EBUSY;
 254                 else if (type == MOD_UNLOAD) {
 255                         uma_zdestroy(vtnet_tx_header_zone);
 256                         vtnet_tx_header_zone = NULL;
 257                 }
 258                 break;
 259         case MOD_SHUTDOWN:
 260                 break;
 261         default:
 262                 error = EOPNOTSUPP;
 263                 break;
 264         }
 265
 266         return (error);
 267 }
 268
 269 static int
 270 vtnet_probe(device_t dev)
 271 {
 272
 273         if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
 274                 return (ENXIO);
 275
 276         device_set_desc(dev, "VirtIO Networking Adapter");
 277
 278         return (BUS_PROBE_DEFAULT);
 279 }
 280
 281 static int
 282 vtnet_attach(device_t dev)
 283 {
 284         struct vtnet_softc *sc;
 285         struct ifnet *ifp;
 286         int tx_size, error;
 287
 288         sc = device_get_softc(dev);
 289         sc->vtnet_dev = dev;
 290
 291         VTNET_LOCK_INIT(sc);
 292         callout_init_mtx(&sc->vtnet_tick_ch, VTNET_MTX(sc), 0);
 293
 294         ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
 295             vtnet_ifmedia_sts);
 296         ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
 297         ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
 298
 299         vtnet_add_statistics(sc);
 300
 301         virtio_set_feature_desc(dev, vtnet_feature_desc);
 302         vtnet_negotiate_features(sc);
 303
 304         if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
 305                 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
 306                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 307         } else
 308                 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
 309
 310         sc->vtnet_rx_mbuf_size = MCLBYTES;
 311         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
 312
 313         if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
 314                 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
 315
 316                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX)) {
 317                         sc->vtnet_mac_filter = malloc(
 318                             sizeof(struct vtnet_mac_filter), M_DEVBUF,
 319                             M_NOWAIT | M_ZERO);
 320                         if (sc->vtnet_mac_filter == NULL) {
 321                                 device_printf(dev,
 322                                     "cannot allocate mac filter table\n");
 323                                 error = ENOMEM;
 324                                 goto fail;
 325                         }
 326
 327                         sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
 328                 }
 329
 330                 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
 331                         sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
 332         }
 333
 334         vtnet_get_hwaddr(sc);
 335
 336         error = vtnet_alloc_virtqueues(sc);
 337         if (error) {
 338                 device_printf(dev, "cannot allocate virtqueues\n");
 339                 goto fail;
 340         }
 341
 342         ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
 343         if (ifp == NULL) {
 344                 device_printf(dev, "cannot allocate ifnet structure\n");
 345                 error = ENOSPC;
 346                 goto fail;
 347         }
 348
 349         ifp->if_softc = sc;
 350         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 351         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 352         ifp->if_init = vtnet_init;
 353         ifp->if_start = vtnet_start;
 354         ifp->if_ioctl = vtnet_ioctl;
 355
 356         sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq);
 357         sc->vtnet_rx_process_limit = sc->vtnet_rx_size;
 358
 359         tx_size = virtqueue_size(sc->vtnet_tx_vq);
 360         sc->vtnet_tx_size = tx_size;
 361         IFQ_SET_MAXLEN(&ifp->if_snd, tx_size - 1);
 362         ifp->if_snd.ifq_drv_maxlen = tx_size - 1;
 363         IFQ_SET_READY(&ifp->if_snd);
 364
 365         ether_ifattach(ifp, sc->vtnet_hwaddr);
 366
 367         if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS))
 368                 ifp->if_capabilities |= IFCAP_LINKSTATE;
 369
 370         /* Tell the upper layer(s) we support long frames. */
 371         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 372         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 373
 374         if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
 375                 ifp->if_capabilities |= IFCAP_TXCSUM;
 376
 377                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
 378                         ifp->if_capabilities |= IFCAP_TSO4;
 379                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
 380                         ifp->if_capabilities |= IFCAP_TSO6;
 381                 if (ifp->if_capabilities & IFCAP_TSO)
 382                         ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
 383
 384                 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
 385                         sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
 386         }
 387
 388         if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
 389                 ifp->if_capabilities |= IFCAP_RXCSUM;
 390
 391                 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
 392                     virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
 393                         ifp->if_capabilities |= IFCAP_LRO;
 394         }
 395
 396         if (ifp->if_capabilities & IFCAP_HWCSUM) {
 397                 /*
 398                  * VirtIO does not support VLAN tagging, but we can fake
 399                  * it by inserting and removing the 802.1Q header during
 400                  * transmit and receive. We are then able to do checksum
 401                  * offloading of VLAN frames.
 402                  */
 403                 ifp->if_capabilities |=
 404                     IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
 405         }
 406
 407         ifp->if_capenable = ifp->if_capabilities;
 408
 409         /*
 410          * Capabilities after here are not enabled by default.
 411          */
 412
 413         if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
 414                 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 415
 416                 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 417                     vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
 418                 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 419                     vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
 420         }
 421
 422 #ifdef DEVICE_POLLING
 423         ifp->if_capabilities |= IFCAP_POLLING;
 424 #endif
 425
 426         error = virtio_setup_intr(dev, INTR_TYPE_NET);
 427         if (error) {
 428                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 429                 ether_ifdetach(ifp);
 430                 goto fail;
 431         }
 432
 433         /*
 434          * Device defaults to promiscuous mode for backwards
 435          * compatibility. Turn it off if possible.
 436          */
 437         if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
 438                 VTNET_LOCK(sc);
 439                 if (vtnet_set_promisc(sc, 0) != 0) {
 440                         ifp->if_flags |= IFF_PROMISC;
 441                         device_printf(dev,
 442                             "cannot disable promiscuous mode\n");
 443                 }
 444                 VTNET_UNLOCK(sc);
 445         } else
 446                 ifp->if_flags |= IFF_PROMISC;
 447
 448 fail:
 449         if (error)
 450                 vtnet_detach(dev);
 451
 452         return (error);
 453 }
 454
 455 static int
 456 vtnet_detach(device_t dev)
 457 {
 458         struct vtnet_softc *sc;
 459         struct ifnet *ifp;
 460
 461         sc = device_get_softc(dev);
 462         ifp = sc->vtnet_ifp;
 463
 464         KASSERT(mtx_initialized(VTNET_MTX(sc)),
 465             ("vtnet mutex not initialized"));
 466
 467 #ifdef DEVICE_POLLING
 468         if (ifp != NULL && ifp->if_capenable & IFCAP_POLLING)
 469                 ether_poll_deregister(ifp);
 470 #endif
 471
 472         if (device_is_attached(dev)) {
 473                 VTNET_LOCK(sc);
 474                 vtnet_stop(sc);
 475                 VTNET_UNLOCK(sc);
 476
 477                 callout_drain(&sc->vtnet_tick_ch);
 478
 479                 ether_ifdetach(ifp);
 480         }
 481
 482         if (sc->vtnet_vlan_attach != NULL) {
 483                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
 484                 sc->vtnet_vlan_attach = NULL;
 485         }
 486         if (sc->vtnet_vlan_detach != NULL) {
 487                 EVENTHANDLER_DEREGISTER(vlan_unconfg, sc->vtnet_vlan_detach);
 488                 sc->vtnet_vlan_detach = NULL;
 489         }
 490
 491         if (sc->vtnet_mac_filter != NULL) {
 492                 free(sc->vtnet_mac_filter, M_DEVBUF);
 493                 sc->vtnet_mac_filter = NULL;
 494         }
 495
 496         if (ifp != NULL) {
 497                 if_free(ifp);
 498                 sc->vtnet_ifp = NULL;
 499         }
 500
 501         if (sc->vtnet_rx_vq != NULL)
 502                 vtnet_free_rx_mbufs(sc);
 503         if (sc->vtnet_tx_vq != NULL)
 504                 vtnet_free_tx_mbufs(sc);
 505         if (sc->vtnet_ctrl_vq != NULL)
 506                 vtnet_free_ctrl_vq(sc);
 507
 508         ifmedia_removeall(&sc->vtnet_media);
 509         VTNET_LOCK_DESTROY(sc);
 510
 511         return (0);
 512 }
 513
 514 static int
 515 vtnet_suspend(device_t dev)
 516 {
 517         struct vtnet_softc *sc;
 518
 519         sc = device_get_softc(dev);
 520
 521         VTNET_LOCK(sc);
 522         vtnet_stop(sc);
 523         sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
 524         VTNET_UNLOCK(sc);
 525
 526         return (0);
 527 }
 528
 529 static int
 530 vtnet_resume(device_t dev)
 531 {
 532         struct vtnet_softc *sc;
 533         struct ifnet *ifp;
 534
 535         sc = device_get_softc(dev);
 536         ifp = sc->vtnet_ifp;
 537
 538         VTNET_LOCK(sc);
 539         if (ifp->if_flags & IFF_UP)
 540                 vtnet_init_locked(sc);
 541         sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
 542         VTNET_UNLOCK(sc);
 543
 544         return (0);
 545 }
 546
 547 static int
 548 vtnet_shutdown(device_t dev)
 549 {
 550
 551         /*
 552          * Suspend already does all of what we need to
 553          * do here; we just never expect to be resumed.
 554          */
 555         return (vtnet_suspend(dev));
 556 }
 557
 558 static int
 559 vtnet_config_change(device_t dev)
 560 {
 561         struct vtnet_softc *sc;
 562
 563         sc = device_get_softc(dev);
 564
 565         VTNET_LOCK(sc);
 566         vtnet_update_link_status(sc);
 567         VTNET_UNLOCK(sc);
 568
 569         return (0);
 570 }
 571
 572 static void
 573 vtnet_negotiate_features(struct vtnet_softc *sc)
 574 {
 575         device_t dev;
 576         uint64_t mask, features;
 577
 578         dev = sc->vtnet_dev;
 579         mask = 0;
 580
 581         if (vtnet_csum_disable)
 582                 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
 583
 584         /*
 585          * TSO and LRO are only available when their corresponding
 586          * checksum offload feature is also negotiated.
 587          */
 588
 589         if (vtnet_csum_disable || vtnet_tso_disable)
 590                 mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
 591                     VIRTIO_NET_F_HOST_ECN;
 592
 593         if (vtnet_csum_disable || vtnet_lro_disable)
 594                 mask |= VTNET_LRO_FEATURES;
 595
 596         features = VTNET_FEATURES & ~mask;
 597 #ifdef VTNET_TX_INTR_MODERATION
 598         features |= VIRTIO_F_NOTIFY_ON_EMPTY;
 599 #endif
 600         sc->vtnet_features = virtio_negotiate_features(dev, features);
 601
 602         if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0 &&
 603             virtio_with_feature(dev, VTNET_LRO_FEATURES)) {
 604                 /*
 605                  * LRO without mergeable buffers requires special care. This
 606                  * is not ideal because every receive buffer must be large
 607                  * enough to hold the maximum TCP packet, the Ethernet header,
 608                  * and the vtnet_rx_header. This requires up to 34 descriptors
 609                  * when using MCLBYTES clusters. If we do not have indirect
 610                  * descriptors, LRO is disabled since the virtqueue will not
 611                  * be able to contain very many receive buffers.
 612                  */
 613                 if (virtio_with_feature(dev,
 614                     VIRTIO_RING_F_INDIRECT_DESC) == 0) {
 615                         device_printf(dev,
 616                             "LRO disabled due to lack of both mergeable "
 617                             "buffers and indirect descriptors\n");
 618
 619                         sc->vtnet_features = virtio_negotiate_features(dev,
 620                             features & ~VTNET_LRO_FEATURES);
 621                 } else
 622                         sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
 623         }
 624 }
 625
 626 static int
 627 vtnet_alloc_virtqueues(struct vtnet_softc *sc)
 628 {
 629         device_t dev;
 630         struct vq_alloc_info vq_info[3];
 631         int nvqs, rxsegs;
 632
 633         dev = sc->vtnet_dev;
 634         nvqs = 2;
 635
 636         /*
 637          * Indirect descriptors are not needed for the Rx
 638          * virtqueue when mergeable buffers are negotiated.
 639          * The header is placed inline with the data, not
 640          * in a separate descriptor, and mbuf clusters are
 641          * always physically contiguous.
 642          */
 643         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
 644                 rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ?
 645                     VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
 646         } else
 647                 rxsegs = 0;
 648
 649         VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs,
 650             vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq,
 651             "%s receive", device_get_nameunit(dev));
 652
 653         VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS,
 654             vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq,
 655             "%s transmit", device_get_nameunit(dev));
 656
 657         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
 658                 nvqs++;
 659
 660                 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL,
 661                     &sc->vtnet_ctrl_vq, "%s control",
 662                     device_get_nameunit(dev));
 663         }
 664
 665         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 666 }
 667
 668 static void
 669 vtnet_get_hwaddr(struct vtnet_softc *sc)
 670 {
 671         device_t dev;
 672
 673         dev = sc->vtnet_dev;
 674
 675         if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
 676                 virtio_read_device_config(dev,
 677                     offsetof(struct virtio_net_config, mac),
 678                     sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 679         } else {
 680                 /* Generate random locally administered unicast address. */
 681                 sc->vtnet_hwaddr[0] = 0xB2;
 682                 arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0);
 683
 684                 vtnet_set_hwaddr(sc);
 685         }
 686 }
 687
 688 static void
 689 vtnet_set_hwaddr(struct vtnet_softc *sc)
 690 {
 691         device_t dev;
 692
 693         dev = sc->vtnet_dev;
 694
 695         virtio_write_device_config(dev,
 696             offsetof(struct virtio_net_config, mac),
 697             sc->vtnet_hwaddr, ETHER_ADDR_LEN);
 698 }
 699
 700 static int
 701 vtnet_is_link_up(struct vtnet_softc *sc)
 702 {
 703         device_t dev;
 704         struct ifnet *ifp;
 705         uint16_t status;
 706
 707         dev = sc->vtnet_dev;
 708         ifp = sc->vtnet_ifp;
 709
 710         VTNET_LOCK_ASSERT(sc);
 711
 712         if ((ifp->if_capenable & IFCAP_LINKSTATE) == 0)
 713                 return (1);
 714
 715         status = virtio_read_dev_config_2(dev,
 716             offsetof(struct virtio_net_config, status));
 717
 718         return ((status & VIRTIO_NET_S_LINK_UP) != 0);
 719 }
 720
 721 static void
 722 vtnet_update_link_status(struct vtnet_softc *sc)
 723 {
 724         struct ifnet *ifp;
 725         int link;
 726
 727         ifp = sc->vtnet_ifp;
 728
 729         link = vtnet_is_link_up(sc);
 730
 731         if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
 732                 sc->vtnet_flags |= VTNET_FLAG_LINK;
 733                 if_link_state_change(ifp, LINK_STATE_UP);
 734                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 735                         vtnet_start_locked(ifp);
 736         } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
 737                 sc->vtnet_flags &= ~VTNET_FLAG_LINK;
 738                 if_link_state_change(ifp, LINK_STATE_DOWN);
 739         }
 740 }
 741
 742 static void
 743 vtnet_watchdog(struct vtnet_softc *sc)
 744 {
 745         struct ifnet *ifp;
 746
 747         ifp = sc->vtnet_ifp;
 748
 749 #ifdef VTNET_TX_INTR_MODERATION
 750         vtnet_txeof(sc);
 751 #endif
 752
 753         if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer)
 754                 return;
 755
 756         if_printf(ifp, "watchdog timeout -- resetting\n");
 757 #ifdef VTNET_DEBUG
 758         virtqueue_dump(sc->vtnet_tx_vq);
 759 #endif
 760         ifp->if_oerrors++;
 761         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 762         vtnet_init_locked(sc);
 763 }
 764
 765 static int
 766 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 767 {
 768         struct vtnet_softc *sc;
 769         struct ifreq *ifr;
 770         int reinit, mask, error;
 771
 772         sc = ifp->if_softc;
 773         ifr = (struct ifreq *) data;
 774         reinit = 0;
 775         error = 0;
 776
 777         switch (cmd) {
 778         case SIOCSIFMTU:
 779                 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
 780                         error = EINVAL;
 781                 else if (ifp->if_mtu != ifr->ifr_mtu) {
 782                         VTNET_LOCK(sc);
 783                         error = vtnet_change_mtu(sc, ifr->ifr_mtu);
 784                         VTNET_UNLOCK(sc);
 785                 }
 786                 break;
 787
 788         case SIOCSIFFLAGS:
 789                 VTNET_LOCK(sc);
 790                 if ((ifp->if_flags & IFF_UP) == 0) {
 791                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 792                                 vtnet_stop(sc);
 793                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 794                         if ((ifp->if_flags ^ sc->vtnet_if_flags) &
 795                             (IFF_PROMISC | IFF_ALLMULTI)) {
 796                                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
 797                                         vtnet_rx_filter(sc);
 798                                 else
 799                                         error = ENOTSUP;
 800                         }
 801                 } else
 802                         vtnet_init_locked(sc);
 803
 804                 if (error == 0)
 805                         sc->vtnet_if_flags = ifp->if_flags;
 806                 VTNET_UNLOCK(sc);
 807                 break;
 808
 809         case SIOCADDMULTI:
 810         case SIOCDELMULTI:
 811                 VTNET_LOCK(sc);
 812                 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
 813                     (ifp->if_drv_flags & IFF_DRV_RUNNING))
 814                         vtnet_rx_filter_mac(sc);
 815                 VTNET_UNLOCK(sc);
 816                 break;
 817
 818         case SIOCSIFMEDIA:
 819         case SIOCGIFMEDIA:
 820                 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
 821                 break;
 822
 823         case SIOCSIFCAP:
 824                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 825
 826 #ifdef DEVICE_POLLING
 827                 if (mask & IFCAP_POLLING) {
 828                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
 829                                 error = ether_poll_register(vtnet_poll, ifp);
 830                                 if (error)
 831                                         break;
 832
 833                                 VTNET_LOCK(sc);
 834                                 vtnet_disable_rx_intr(sc);
 835                                 vtnet_disable_tx_intr(sc);
 836                                 ifp->if_capenable |= IFCAP_POLLING;
 837                                 VTNET_UNLOCK(sc);
 838                         } else {
 839                                 error = ether_poll_deregister(ifp);
 840
 841                                 /* Enable interrupts even in error case. */
 842                                 VTNET_LOCK(sc);
 843                                 vtnet_enable_tx_intr(sc);
 844                                 vtnet_enable_rx_intr(sc);
 845                                 ifp->if_capenable &= ~IFCAP_POLLING;
 846                                 VTNET_UNLOCK(sc);
 847                         }
 848                 }
 849 #endif
 850                 VTNET_LOCK(sc);
 851
 852                 if (mask & IFCAP_TXCSUM) {
 853                         ifp->if_capenable ^= IFCAP_TXCSUM;
 854                         if (ifp->if_capenable & IFCAP_TXCSUM)
 855                                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
 856                         else
 857                                 ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
 858                 }
 859
 860                 if (mask & IFCAP_TSO4) {
 861                         ifp->if_capenable ^= IFCAP_TSO4;
 862                         if (ifp->if_capenable & IFCAP_TSO4)
 863                                 ifp->if_hwassist |= CSUM_TSO;
 864                         else
 865                                 ifp->if_hwassist &= ~CSUM_TSO;
 866                 }
 867
 868                 if (mask & IFCAP_RXCSUM) {
 869                         ifp->if_capenable ^= IFCAP_RXCSUM;
 870                         reinit = 1;
 871                 }
 872
 873                 if (mask & IFCAP_LRO) {
 874                         ifp->if_capenable ^= IFCAP_LRO;
 875                         reinit = 1;
 876                 }
 877
 878                 if (mask & IFCAP_VLAN_HWFILTER) {
 879                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 880                         reinit = 1;
 881                 }
 882
 883                 if (mask & IFCAP_VLAN_HWTSO)
 884                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 885
 886                 if (mask & IFCAP_VLAN_HWTAGGING)
 887                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 888
 889                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 890                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 891                         vtnet_init_locked(sc);
 892                 }
 893                 VLAN_CAPABILITIES(ifp);
 894
 895                 VTNET_UNLOCK(sc);
 896                 break;
 897
 898         default:
 899                 error = ether_ioctl(ifp, cmd, data);
 900                 break;
 901         }
 902
 903         VTNET_LOCK_ASSERT_NOTOWNED(sc);
 904
 905         return (error);
 906 }
 907
 908 static int
 909 vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
 910 {
 911         struct ifnet *ifp;
 912         int new_frame_size, clsize;
 913
 914         ifp = sc->vtnet_ifp;
 915
 916         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
 917                 new_frame_size = sizeof(struct vtnet_rx_header) +
 918                     sizeof(struct ether_vlan_header) + new_mtu;
 919
 920                 if (new_frame_size > MJUM9BYTES)
 921                         return (EINVAL);
 922
 923                 if (new_frame_size <= MCLBYTES)
 924                         clsize = MCLBYTES;
 925                 else
 926                         clsize = MJUM9BYTES;
 927         } else {
 928                 new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
 929                     sizeof(struct ether_vlan_header) + new_mtu;
 930
 931                 if (new_frame_size <= MCLBYTES)
 932                         clsize = MCLBYTES;
 933                 else
 934                         clsize = MJUMPAGESIZE;
 935         }
 936
 937         sc->vtnet_rx_mbuf_size = clsize;
 938         sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
 939         KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
 940             ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
 941
 942         ifp->if_mtu = new_mtu;
 943
 944         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 945                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 946                 vtnet_init_locked(sc);
 947         }
 948
 949         return (0);
 950 }
 951
 952 static int
 953 vtnet_init_rx_vq(struct vtnet_softc *sc)
 954 {
 955         struct virtqueue *vq;
 956         int nbufs, error;
 957
 958         vq = sc->vtnet_rx_vq;
 959         nbufs = 0;
 960         error = ENOSPC;
 961
 962         while (!virtqueue_full(vq)) {
 963                 if ((error = vtnet_newbuf(sc)) != 0)
 964                         break;
 965                 nbufs++;
 966         }
 967
 968         if (nbufs > 0) {
 969                 virtqueue_notify(vq);
 970
 971                 /*
 972                  * EMSGSIZE signifies the virtqueue did not have enough
 973                  * entries available to hold the last mbuf. This is not
 974                  * an error. We should not get ENOSPC since we check if
 975                  * the virtqueue is full before attempting to add a
 976                  * buffer.
 977                  */
 978                 if (error == EMSGSIZE)
 979                         error = 0;
 980         }
 981
 982         return (error);
 983 }
 984
 985 static void
 986 vtnet_free_rx_mbufs(struct vtnet_softc *sc)
 987 {
 988         struct virtqueue *vq;
 989         struct mbuf *m;
 990         int last;
 991
 992         vq = sc->vtnet_rx_vq;
 993         last = 0;
 994
 995         while ((m = virtqueue_drain(vq, &last)) != NULL)
 996                 m_freem(m);
 997
 998         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
 999 }
1000
1001 static void
1002 vtnet_free_tx_mbufs(struct vtnet_softc *sc)
1003 {
1004         struct virtqueue *vq;
1005         struct vtnet_tx_header *txhdr;
1006         int last;
1007
1008         vq = sc->vtnet_tx_vq;
1009         last = 0;
1010
1011         while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
1012                 m_freem(txhdr->vth_mbuf);
1013                 uma_zfree(vtnet_tx_header_zone, txhdr);
1014         }
1015
1016         KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
1017 }
1018
1019 static void
1020 vtnet_free_ctrl_vq(struct vtnet_softc *sc)
1021 {
1022
1023         /*
1024          * The control virtqueue is only polled, therefore
1025          * it should already be empty.
1026          */
1027         KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
1028             ("Ctrl Vq not empty"));
1029 }
1030
1031 #ifdef DEVICE_POLLING
1032 static int
1033 vtnet_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1034 {
1035         struct vtnet_softc *sc;
1036         int rx_done;
1037
1038         sc = ifp->if_softc;
1039         rx_done = 0;
1040
1041         VTNET_LOCK(sc);
1042         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1043                 if (cmd == POLL_AND_CHECK_STATUS)
1044                         vtnet_update_link_status(sc);
1045
1046                 if (virtqueue_nused(sc->vtnet_rx_vq) > 0)
1047                         vtnet_rxeof(sc, count, &rx_done);
1048
1049                 vtnet_txeof(sc);
1050                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1051                         vtnet_start_locked(ifp);
1052         }
1053         VTNET_UNLOCK(sc);
1054
1055         return (rx_done);
1056 }
1057 #endif /* DEVICE_POLLING */
1058
1059 static struct mbuf *
1060 vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
1061 {
1062         struct mbuf *m_head, *m_tail, *m;
1063         int i, clsize;
1064
1065         clsize = sc->vtnet_rx_mbuf_size;
1066
1067         m_head = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, clsize);
1068         if (m_head == NULL)
1069                 goto fail;
1070
1071         m_head->m_len = clsize;
1072         m_tail = m_head;
1073
1074         if (nbufs > 1) {
1075                 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
1076                     ("chained Rx mbuf requested without LRO_NOMRG"));
1077
1078                 for (i = 1; i < nbufs; i++) {
1079                         m = m_getjcl(M_NOWAIT, MT_DATA, 0, clsize);
1080                         if (m == NULL)
1081                                 goto fail;
1082
1083                         m->m_len = clsize;
1084                         m_tail->m_next = m;
1085                         m_tail = m;
1086                 }
1087         }
1088
1089         if (m_tailp != NULL)
1090                 *m_tailp = m_tail;
1091
1092         return (m_head);
1093
1094 fail:
1095         sc->vtnet_stats.mbuf_alloc_failed++;
1096         m_freem(m_head);
1097
1098         return (NULL);
1099 }
1100
1101 static int
1102 vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
1103 {
1104         struct mbuf *m, *m_prev;
1105         struct mbuf *m_new, *m_tail;
1106         int len, clsize, nreplace, error;
1107
1108         m = m0;
1109         m_prev = NULL;
1110         len = len0;
1111
1112         m_tail = NULL;
1113         clsize = sc->vtnet_rx_mbuf_size;
1114         nreplace = 0;
1115
1116         KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ||
1117             m->m_next == NULL, ("chained Rx mbuf without LRO_NOMRG"));
1118
1119         /*
1120          * Since LRO_NOMRG mbuf chains are so large, we want to avoid
1121          * allocating an entire chain for each received frame. When
1122          * the received frame's length is less than that of the chain,
1123          * the unused mbufs are reassigned to the new chain.
1124          */
1125         while (len > 0) {
1126                 /*
1127                  * Something is seriously wrong if we received
1128                  * a frame larger than the mbuf chain. Drop it.
1129                  */
1130                 if (m == NULL) {
1131                         sc->vtnet_stats.rx_frame_too_large++;
1132                         return (EMSGSIZE);
1133                 }
1134
1135                 KASSERT(m->m_len == clsize,
1136                     ("mbuf length not expected cluster size: %d",
1137                     m->m_len));
1138
1139                 m->m_len = MIN(m->m_len, len);
1140                 len -= m->m_len;
1141
1142                 m_prev = m;
1143                 m = m->m_next;
1144                 nreplace++;
1145         }
1146
1147         KASSERT(m_prev != NULL, ("m_prev == NULL"));
1148         KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
1149             ("too many replacement mbufs: %d/%d", nreplace,
1150             sc->vtnet_rx_mbuf_count));
1151
1152         m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
1153         if (m_new == NULL) {
1154                 m_prev->m_len = clsize;
1155                 return (ENOBUFS);
1156         }
1157
1158         /*
1159          * Move unused mbufs, if any, from the original chain
1160          * onto the end of the new chain.
1161          */
1162         if (m_prev->m_next != NULL) {
1163                 m_tail->m_next = m_prev->m_next;
1164                 m_prev->m_next = NULL;
1165         }
1166
1167         error = vtnet_enqueue_rxbuf(sc, m_new);
1168         if (error) {
1169                 /*
1170                  * BAD! We could not enqueue the replacement mbuf chain. We
1171                  * must restore the m0 chain to the original state if it was
1172                  * modified so we can subsequently discard it.
1173                  *
1174                  * NOTE: The replacement is suppose to be an identical copy
1175                  * to the one just dequeued so this is an unexpected error.
1176                  */
1177                 sc->vtnet_stats.rx_enq_replacement_failed++;
1178
1179                 if (m_tail->m_next != NULL) {
1180                         m_prev->m_next = m_tail->m_next;
1181                         m_tail->m_next = NULL;
1182                 }
1183
1184                 m_prev->m_len = clsize;
1185                 m_freem(m_new);
1186         }
1187
1188         return (error);
1189 }
1190
1191 static int
1192 vtnet_newbuf(struct vtnet_softc *sc)
1193 {
1194         struct mbuf *m;
1195         int error;
1196
1197         m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
1198         if (m == NULL)
1199                 return (ENOBUFS);
1200
1201         error = vtnet_enqueue_rxbuf(sc, m);
1202         if (error)
1203                 m_freem(m);
1204
1205         return (error);
1206 }
1207
1208 static void
1209 vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
1210 {
1211         struct virtqueue *vq;
1212         struct mbuf *m;
1213
1214         vq = sc->vtnet_rx_vq;
1215
1216         while (--nbufs > 0) {
1217                 if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
1218                         break;
1219                 vtnet_discard_rxbuf(sc, m);
1220         }
1221 }
1222
1223 static void
1224 vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1225 {
1226         int error;
1227
1228         /*
1229          * Requeue the discarded mbuf. This should always be
1230          * successful since it was just dequeued.
1231          */
1232         error = vtnet_enqueue_rxbuf(sc, m);
1233         KASSERT(error == 0, ("cannot requeue discarded mbuf"));
1234 }
1235
1236 static int
1237 vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
1238 {
1239         struct sglist sg;
1240         struct sglist_seg segs[VTNET_MAX_RX_SEGS];
1241         struct vtnet_rx_header *rxhdr;
1242         struct virtio_net_hdr *hdr;
1243         uint8_t *mdata;
1244         int offset, error;
1245
1246         VTNET_LOCK_ASSERT(sc);
1247         KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ||
1248             m->m_next == NULL, ("chained Rx mbuf without LRO_NOMRG"));
1249
1250         sglist_init(&sg, VTNET_MAX_RX_SEGS, segs);
1251
1252         mdata = mtod(m, uint8_t *);
1253         offset = 0;
1254
1255         if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1256                 rxhdr = (struct vtnet_rx_header *) mdata;
1257                 hdr = &rxhdr->vrh_hdr;
1258                 offset += sizeof(struct vtnet_rx_header);
1259
1260                 error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
1261                 KASSERT(error == 0, ("cannot add header to sglist"));
1262         }
1263
1264         error = sglist_append(&sg, mdata + offset, m->m_len - offset);
1265         if (error)
1266                 return (error);
1267
1268         if (m->m_next != NULL) {
1269                 error = sglist_append_mbuf(&sg, m->m_next);
1270                 if (error)
1271                         return (error);
1272         }
1273
1274         return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
1275 }
1276
1277 static void
1278 vtnet_vlan_tag_remove(struct mbuf *m)
1279 {
1280         struct ether_vlan_header *evl;
1281
1282         evl = mtod(m, struct ether_vlan_header *);
1283
1284         m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag);
1285         m->m_flags |= M_VLANTAG;
1286
1287         /* Strip the 802.1Q header. */
1288         bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
1289             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1290         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1291 }
1292
1293 #ifdef notyet
1294 static int
1295 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
1296     struct virtio_net_hdr *hdr)
1297 {
1298         struct ether_header *eh;
1299         struct ether_vlan_header *evh;
1300         struct ip *ip;
1301         struct ip6_hdr *ip6;
1302         struct udphdr *udp;
1303         int ip_offset, csum_start, csum_offset, hlen;
1304         uint16_t eth_type;
1305         uint8_t ip_proto;
1306
1307         /*
1308          * Convert the VirtIO checksum interface to FreeBSD's interface.
1309          * The host only provides us with the offset at which to start
1310          * checksumming, and the offset from that to place the completed
1311          * checksum. While this maps well with how Linux does checksums,
1312          * for FreeBSD, we must parse the received packet in order to set
1313          * the appropriate CSUM_* flags.
1314          */
1315
1316         /*
1317          * Every mbuf added to the receive virtqueue is always at least
1318          * MCLBYTES big, so assume something is amiss if the first mbuf
1319          * does not contain both the Ethernet and protocol headers.
1320          */
1321         ip_offset = sizeof(struct ether_header);
1322         if (m->m_len < ip_offset)
1323                 return (1);
1324
1325         eh = mtod(m, struct ether_header *);
1326         eth_type = ntohs(eh->ether_type);
1327         if (eth_type == ETHERTYPE_VLAN) {
1328                 ip_offset = sizeof(struct ether_vlan_header);
1329                 if (m->m_len < ip_offset)
1330                         return (1);
1331                 evh = mtod(m, struct ether_vlan_header *);
1332                 eth_type = ntohs(evh->evl_proto);
1333         }
1334
1335         switch (eth_type) {
1336         case ETHERTYPE_IP:
1337                 if (m->m_len < ip_offset + sizeof(struct ip))
1338                         return (1);
1339
1340                 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
1341                  /* Sanity check the IP header. */
1342                 if (ip->ip_v != IPVERSION)
1343                         return (1);
1344                 hlen = ip->ip_hl << 2;
1345                 if (hlen < sizeof(struct ip))
1346                         return (1);
1347                 if (ntohs(ip->ip_len) < hlen)
1348                         return (1);
1349                 if (ntohs(ip->ip_len) != (m->m_pkthdr.len - ip_offset))
1350                         return (1);
1351
1352                 ip_proto = ip->ip_p;
1353                 csum_start = ip_offset + hlen;
1354                 break;
1355
1356         case ETHERTYPE_IPV6:
1357                 if (m->m_len < ip_offset + sizeof(struct ip6_hdr))
1358                         return (1);
1359
1360                 /*
1361                  * XXX FreeBSD does not handle any IPv6 checksum offloading
1362                  * at the moment.
1363                  */
1364
1365                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
1366                 /* XXX Assume no extension headers are present. */
1367                 ip_proto = ip6->ip6_nxt;
1368                 csum_start = ip_offset + sizeof(struct ip6_hdr);
1369                 break;
1370
1371         default:
1372                 sc->vtnet_stats.rx_csum_bad_ethtype++;
1373                 return (1);
1374         }
1375
1376         /* Assume checksum begins right after the IP header. */
1377         if (hdr->csum_start != csum_start) {
1378                 sc->vtnet_stats.rx_csum_bad_start++;
1379                 return (1);
1380         }
1381
1382         switch (ip_proto) {
1383         case IPPROTO_TCP:
1384                 csum_offset = offsetof(struct tcphdr, th_sum);
1385                 break;
1386
1387         case IPPROTO_UDP:
1388                 csum_offset = offsetof(struct udphdr, uh_sum);
1389                 break;
1390
1391         case IPPROTO_SCTP:
1392                 csum_offset = offsetof(struct sctphdr, checksum);
1393                 break;
1394
1395         default:
1396                 sc->vtnet_stats.rx_csum_bad_ipproto++;
1397                 return (1);
1398         }
1399
1400         if (hdr->csum_offset != csum_offset) {
1401                 sc->vtnet_stats.rx_csum_bad_offset++;
1402                 return (1);
1403         }
1404
1405         /*
1406          * The IP header checksum is almost certainly valid but I'm
1407          * uncertain if that is guaranteed.
1408          *
1409          * m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID;
1410          */
1411
1412         switch (ip_proto) {
1413         case IPPROTO_UDP:
1414                 if (m->m_len < csum_start + sizeof(struct udphdr))
1415                         return (1);
1416
1417                 udp = (struct udphdr *)(mtod(m, uint8_t *) + csum_start);
1418                 if (udp->uh_sum == 0)
1419                         return (0);
1420
1421                 /* FALLTHROUGH */
1422
1423         case IPPROTO_TCP:
1424                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1425                 m->m_pkthdr.csum_data = 0xFFFF;
1426                 break;
1427
1428         case IPPROTO_SCTP:
1429                 m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
1430                 break;
1431         }
1432
1433         sc->vtnet_stats.rx_csum_offloaded++;
1434
1435         return (0);
1436 }
1437 #endif
1438
1439 /*
1440  * Alternative method of doing receive checksum offloading. Rather
1441  * than parsing the received frame down to the IP header, use the
1442  * csum_offset to determine which CSUM_* flags are appropriate. We
1443  * can get by with doing this only because the checksum offsets are
1444  * unique for the things we care about.
1445  */
1446 static int
1447 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
1448     struct virtio_net_hdr *hdr)
1449 {
1450         struct ether_header *eh;
1451         struct ether_vlan_header *evh;
1452         struct udphdr *udp;
1453         int csum_len;
1454         uint16_t eth_type;
1455
1456         csum_len = hdr->csum_start + hdr->csum_offset;
1457
1458         if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
1459                 return (1);
1460         if (m->m_len < csum_len)
1461                 return (1);
1462
1463         eh = mtod(m, struct ether_header *);
1464         eth_type = ntohs(eh->ether_type);
1465         if (eth_type == ETHERTYPE_VLAN) {
1466                 evh = mtod(m, struct ether_vlan_header *);
1467                 eth_type = ntohs(evh->evl_proto);
1468         }
1469
1470         if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
1471                 sc->vtnet_stats.rx_csum_bad_ethtype++;
1472                 return (1);
1473         }
1474
1475         /* Use the offset to determine the appropriate CSUM_* flags. */
1476         switch (hdr->csum_offset) {
1477         case offsetof(struct udphdr, uh_sum):
1478                 if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
1479                         return (1);
1480                 udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
1481                 if (udp->uh_sum == 0)
1482                         return (0);
1483
1484                 /* FALLTHROUGH */
1485
1486         case offsetof(struct tcphdr, th_sum):
1487                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1488                 m->m_pkthdr.csum_data = 0xFFFF;
1489                 break;
1490
1491         case offsetof(struct sctphdr, checksum):
1492                 m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
1493                 break;
1494
1495         default:
1496                 sc->vtnet_stats.rx_csum_bad_offset++;
1497                 return (1);
1498         }
1499
1500         sc->vtnet_stats.rx_csum_offloaded++;
1501
1502         return (0);
1503 }
1504
1505 static int
1506 vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
1507 {
1508         struct ifnet *ifp;
1509         struct virtqueue *vq;
1510         struct mbuf *m, *m_tail;
1511         int len;
1512
1513         ifp = sc->vtnet_ifp;
1514         vq = sc->vtnet_rx_vq;
1515         m_tail = m_head;
1516
1517         while (--nbufs > 0) {
1518                 m = virtqueue_dequeue(vq, &len);
1519                 if (m == NULL) {
1520                         ifp->if_ierrors++;
1521                         goto fail;
1522                 }
1523
1524                 if (vtnet_newbuf(sc) != 0) {
1525                         ifp->if_iqdrops++;
1526                         vtnet_discard_rxbuf(sc, m);
1527                         if (nbufs > 1)
1528                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1529                         goto fail;
1530                 }
1531
1532                 if (m->m_len < len)
1533                         len = m->m_len;
1534
1535                 m->m_len = len;
1536                 m->m_flags &= ~M_PKTHDR;
1537
1538                 m_head->m_pkthdr.len += len;
1539                 m_tail->m_next = m;
1540                 m_tail = m;
1541         }
1542
1543         return (0);
1544
1545 fail:
1546         sc->vtnet_stats.rx_mergeable_failed++;
1547         m_freem(m_head);
1548
1549         return (1);
1550 }
1551
1552 static int
1553 vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
1554 {
1555         struct virtio_net_hdr lhdr;
1556         struct ifnet *ifp;
1557         struct virtqueue *vq;
1558         struct mbuf *m;
1559         struct ether_header *eh;
1560         struct virtio_net_hdr *hdr;
1561         struct virtio_net_hdr_mrg_rxbuf *mhdr;
1562         int len, deq, nbufs, adjsz, rx_npkts;
1563
1564         ifp = sc->vtnet_ifp;
1565         vq = sc->vtnet_rx_vq;
1566         hdr = &lhdr;
1567         deq = 0;
1568         rx_npkts = 0;
1569
1570         VTNET_LOCK_ASSERT(sc);
1571
1572         while (--count >= 0) {
1573                 m = virtqueue_dequeue(vq, &len);
1574                 if (m == NULL)
1575                         break;
1576                 deq++;
1577
1578                 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
1579                         ifp->if_ierrors++;
1580                         vtnet_discard_rxbuf(sc, m);
1581                         continue;
1582                 }
1583
1584                 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
1585                         nbufs = 1;
1586                         adjsz = sizeof(struct vtnet_rx_header);
1587                         /*
1588                          * Account for our pad between the header and
1589                          * the actual start of the frame.
1590                          */
1591                         len += VTNET_RX_HEADER_PAD;
1592                 } else {
1593                         mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
1594                         nbufs = mhdr->num_buffers;
1595                         adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1596                 }
1597
1598                 if (vtnet_replace_rxbuf(sc, m, len) != 0) {
1599                         ifp->if_iqdrops++;
1600                         vtnet_discard_rxbuf(sc, m);
1601                         if (nbufs > 1)
1602                                 vtnet_discard_merged_rxbuf(sc, nbufs);
1603                         continue;
1604                 }
1605
1606                 m->m_pkthdr.len = len;
1607                 m->m_pkthdr.rcvif = ifp;
1608                 m->m_pkthdr.csum_flags = 0;
1609
1610                 if (nbufs > 1) {
1611                         if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
1612                                 continue;
1613                 }
1614
1615                 ifp->if_ipackets++;
1616
1617                 /*
1618                  * Save copy of header before we strip it. For both mergeable
1619                  * and non-mergeable, the VirtIO header is placed first in the
1620                  * mbuf's data. We no longer need num_buffers, so always use a
1621                  * virtio_net_hdr.
1622                  */
1623                 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
1624                 m_adj(m, adjsz);
1625
1626                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1627                         eh = mtod(m, struct ether_header *);
1628                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1629                                 vtnet_vlan_tag_remove(m);
1630
1631                                 /*
1632                                  * With the 802.1Q header removed, update the
1633                                  * checksum starting location accordingly.
1634                                  */
1635                                 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1636                                         hdr->csum_start -=
1637                                             ETHER_VLAN_ENCAP_LEN;
1638                         }
1639                 }
1640
1641                 if (ifp->if_capenable & IFCAP_RXCSUM &&
1642                     hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1643                         if (vtnet_rx_csum(sc, m, hdr) != 0)
1644                                 sc->vtnet_stats.rx_csum_failed++;
1645                 }
1646
1647                 VTNET_UNLOCK(sc);
1648                 rx_npkts++;
1649                 (*ifp->if_input)(ifp, m);
1650                 VTNET_LOCK(sc);
1651
1652                 /*
1653                  * The interface may have been stopped while we were
1654                  * passing the packet up the network stack.
1655                  */
1656                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1657                         break;
1658         }
1659
1660         if (deq > 0)
1661                 virtqueue_notify(vq);
1662
1663         if (rx_npktsp != NULL)
1664                 *rx_npktsp = rx_npkts;
1665
1666         return (count > 0 ? 0 : EAGAIN);
1667 }
1668
1669 static void
1670 vtnet_rx_vq_intr(void *xsc)
1671 {
1672         struct vtnet_softc *sc;
1673         struct ifnet *ifp;
1674         int more;
1675
1676         sc = xsc;
1677         ifp = sc->vtnet_ifp;
1678
1679 again:
1680         VTNET_LOCK(sc);
1681
1682 #ifdef DEVICE_POLLING
1683         if (ifp->if_capenable & IFCAP_POLLING) {
1684                 VTNET_UNLOCK(sc);
1685                 return;
1686         }
1687 #endif
1688
1689         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1690                 vtnet_enable_rx_intr(sc);
1691                 VTNET_UNLOCK(sc);
1692                 return;
1693         }
1694
1695         more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
1696         if (more || vtnet_enable_rx_intr(sc) != 0) {
1697                 if (!more)
1698                         vtnet_disable_rx_intr(sc);
1699                 sc->vtnet_stats.rx_task_rescheduled++;
1700                 VTNET_UNLOCK(sc);
1701                 goto again;
1702         }
1703
1704         VTNET_UNLOCK(sc);
1705 }
1706
1707 static void
1708 vtnet_txeof(struct vtnet_softc *sc)
1709 {
1710         struct virtqueue *vq;
1711         struct ifnet *ifp;
1712         struct vtnet_tx_header *txhdr;
1713         int deq;
1714
1715         vq = sc->vtnet_tx_vq;
1716         ifp = sc->vtnet_ifp;
1717         deq = 0;
1718
1719         VTNET_LOCK_ASSERT(sc);
1720
1721         while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
1722                 deq++;
1723                 ifp->if_opackets++;
1724                 m_freem(txhdr->vth_mbuf);
1725                 uma_zfree(vtnet_tx_header_zone, txhdr);
1726         }
1727
1728         if (deq > 0) {
1729                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1730                 if (virtqueue_empty(vq))
1731                         sc->vtnet_watchdog_timer = 0;
1732         }
1733 }
1734
1735 static struct mbuf *
1736 vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
1737     struct virtio_net_hdr *hdr)
1738 {
1739         struct ifnet *ifp;
1740         struct ether_header *eh;
1741         struct ether_vlan_header *evh;
1742         struct ip *ip;
1743         struct ip6_hdr *ip6;
1744         struct tcphdr *tcp;
1745         int ip_offset;
1746         uint16_t eth_type, csum_start;
1747         uint8_t ip_proto, gso_type;
1748
1749         ifp = sc->vtnet_ifp;
1750
1751         ip_offset = sizeof(struct ether_header);
1752         if (m->m_len < ip_offset) {
1753                 if ((m = m_pullup(m, ip_offset)) == NULL)
1754                         return (NULL);
1755         }
1756
1757         eh = mtod(m, struct ether_header *);
1758         eth_type = ntohs(eh->ether_type);
1759         if (eth_type == ETHERTYPE_VLAN) {
1760                 ip_offset = sizeof(struct ether_vlan_header);
1761                 if (m->m_len < ip_offset) {
1762                         if ((m = m_pullup(m, ip_offset)) == NULL)
1763                                 return (NULL);
1764                 }
1765                 evh = mtod(m, struct ether_vlan_header *);
1766                 eth_type = ntohs(evh->evl_proto);
1767         }
1768
1769         switch (eth_type) {
1770         case ETHERTYPE_IP:
1771                 if (m->m_len < ip_offset + sizeof(struct ip)) {
1772                         m = m_pullup(m, ip_offset + sizeof(struct ip));
1773                         if (m == NULL)
1774                                 return (NULL);
1775                 }
1776
1777                 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
1778                 ip_proto = ip->ip_p;
1779                 csum_start = ip_offset + (ip->ip_hl << 2);
1780                 gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1781                 break;
1782
1783         case ETHERTYPE_IPV6:
1784                 if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
1785                         m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
1786                         if (m == NULL)
1787                                 return (NULL);
1788                 }
1789
1790                 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
1791                 /*
1792                  * XXX Assume no extension headers are present. Presently,
1793                  * this will always be true in the case of TSO, and FreeBSD
1794                  * does not perform checksum offloading of IPv6 yet.
1795                  */
1796                 ip_proto = ip6->ip6_nxt;
1797                 csum_start = ip_offset + sizeof(struct ip6_hdr);
1798                 gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1799                 break;
1800
1801         default:
1802                 return (m);
1803         }
1804
1805         if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
1806                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1807                 hdr->csum_start = csum_start;
1808                 hdr->csum_offset = m->m_pkthdr.csum_data;
1809
1810                 sc->vtnet_stats.tx_csum_offloaded++;
1811         }
1812
1813         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1814                 if (ip_proto != IPPROTO_TCP)
1815                         return (m);
1816
1817                 if (m->m_len < csum_start + sizeof(struct tcphdr)) {
1818                         m = m_pullup(m, csum_start + sizeof(struct tcphdr));
1819                         if (m == NULL)
1820                                 return (NULL);
1821                 }
1822
1823                 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
1824                 hdr->gso_type = gso_type;
1825                 hdr->hdr_len = csum_start + (tcp->th_off << 2);
1826                 hdr->gso_size = m->m_pkthdr.tso_segsz;
1827
1828                 if (tcp->th_flags & TH_CWR) {
1829                         /*
1830                          * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
1831                          * ECN support is only configurable globally with the
1832                          * net.inet.tcp.ecn.enable sysctl knob.
1833                          */
1834                         if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
1835                                 if_printf(ifp, "TSO with ECN not supported "
1836                                     "by host\n");
1837                                 m_freem(m);
1838                                 return (NULL);
1839                         }
1840
1841                         hdr->flags |= VIRTIO_NET_HDR_GSO_ECN;
1842                 }
1843
1844                 sc->vtnet_stats.tx_tso_offloaded++;
1845         }
1846
1847         return (m);
1848 }
1849
1850 static int
1851 vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
1852     struct vtnet_tx_header *txhdr)
1853 {
1854         struct sglist sg;
1855         struct sglist_seg segs[VTNET_MAX_TX_SEGS];
1856         struct virtqueue *vq;
1857         struct mbuf *m;
1858         int collapsed, error;
1859
1860         vq = sc->vtnet_tx_vq;
1861         m = *m_head;
1862         collapsed = 0;
1863
1864         sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
1865         error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
1866         KASSERT(error == 0 && sg.sg_nseg == 1,
1867             ("%s: cannot add header to sglist error %d", __func__, error));
1868
1869 again:
1870         error = sglist_append_mbuf(&sg, m);
1871         if (error) {
1872                 if (collapsed)
1873                         goto fail;
1874
1875                 m = m_collapse(m, M_NOWAIT, VTNET_MAX_TX_SEGS - 1);
1876                 if (m == NULL)
1877                         goto fail;
1878
1879                 *m_head = m;
1880                 collapsed = 1;
1881                 goto again;
1882         }
1883
1884         txhdr->vth_mbuf = m;
1885
1886         return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0));
1887
1888 fail:
1889         m_freem(*m_head);
1890         *m_head = NULL;
1891
1892         return (ENOBUFS);
1893 }
1894
1895 static int
1896 vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
1897 {
1898         struct vtnet_tx_header *txhdr;
1899         struct virtio_net_hdr *hdr;
1900         struct mbuf *m;
1901         int error;
1902
1903         m = *m_head;
1904         M_ASSERTPKTHDR(m);
1905
1906         txhdr = uma_zalloc(vtnet_tx_header_zone, M_NOWAIT | M_ZERO);
1907         if (txhdr == NULL) {
1908                 *m_head = NULL;
1909                 m_freem(m);
1910                 return (ENOMEM);
1911         }
1912
1913         /*
1914          * Always use the non-mergeable header to simplify things. When
1915          * the mergeable feature is negotiated, the num_buffers field
1916          * must be set to zero. We use vtnet_hdr_size later to enqueue
1917          * the correct header size to the host.
1918          */
1919         hdr = &txhdr->vth_uhdr.hdr;
1920
1921         if (m->m_flags & M_VLANTAG) {
1922                 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1923                 if ((*m_head = m) == NULL) {
1924                         error = ENOBUFS;
1925                         goto fail;
1926                 }
1927                 m->m_flags &= ~M_VLANTAG;
1928         }
1929
1930         if (m->m_pkthdr.csum_flags != 0) {
1931                 m = vtnet_tx_offload(sc, m, hdr);
1932                 if ((*m_head = m) == NULL) {
1933                         error = ENOBUFS;
1934                         goto fail;
1935                 }
1936         }
1937
1938         error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
1939 fail:
1940         if (error)
1941                 uma_zfree(vtnet_tx_header_zone, txhdr);
1942
1943         return (error);
1944 }
1945
1946 static void
1947 vtnet_start(struct ifnet *ifp)
1948 {
1949         struct vtnet_softc *sc;
1950
1951         sc = ifp->if_softc;
1952
1953         VTNET_LOCK(sc);
1954         vtnet_start_locked(ifp);
1955         VTNET_UNLOCK(sc);
1956 }
1957
1958 static void
1959 vtnet_start_locked(struct ifnet *ifp)
1960 {
1961         struct vtnet_softc *sc;
1962         struct virtqueue *vq;
1963         struct mbuf *m0;
1964         int enq;
1965
1966         sc = ifp->if_softc;
1967         vq = sc->vtnet_tx_vq;
1968         enq = 0;
1969
1970         VTNET_LOCK_ASSERT(sc);
1971
1972         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1973             IFF_DRV_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
1974                 return;
1975
1976 #ifdef VTNET_TX_INTR_MODERATION
1977         if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
1978                 vtnet_txeof(sc);
1979 #endif
1980
1981         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
1982                 if (virtqueue_full(vq)) {
1983                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1984                         break;
1985                 }
1986
1987                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
1988                 if (m0 == NULL)
1989                         break;
1990
1991                 if (vtnet_encap(sc, &m0) != 0) {
1992                         if (m0 == NULL)
1993                                 break;
1994                         IFQ_DRV_PREPEND(&ifp->if_snd, m0);
1995                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1996                         break;
1997                 }
1998
1999                 enq++;
2000                 ETHER_BPF_MTAP(ifp, m0);
2001         }
2002
2003         if (enq > 0) {
2004                 virtqueue_notify(vq);
2005                 sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT;
2006         }
2007 }
2008
2009 static void
2010 vtnet_tick(void *xsc)
2011 {
2012         struct vtnet_softc *sc;
2013
2014         sc = xsc;
2015
2016         VTNET_LOCK_ASSERT(sc);
2017 #ifdef VTNET_DEBUG
2018         virtqueue_dump(sc->vtnet_rx_vq);
2019         virtqueue_dump(sc->vtnet_tx_vq);
2020 #endif
2021
2022         vtnet_watchdog(sc);
2023         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2024 }
2025
2026 static void
2027 vtnet_tx_vq_intr(void *xsc)
2028 {
2029         struct vtnet_softc *sc;
2030         struct ifnet *ifp;
2031
2032         sc = xsc;
2033         ifp = sc->vtnet_ifp;
2034
2035 again:
2036         VTNET_LOCK(sc);
2037
2038 #ifdef DEVICE_POLLING
2039         if (ifp->if_capenable & IFCAP_POLLING) {
2040                 VTNET_UNLOCK(sc);
2041                 return;
2042         }
2043 #endif
2044
2045         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2046                 vtnet_enable_tx_intr(sc);
2047                 VTNET_UNLOCK(sc);
2048                 return;
2049         }
2050
2051         vtnet_txeof(sc);
2052
2053         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2054                 vtnet_start_locked(ifp);
2055
2056         if (vtnet_enable_tx_intr(sc) != 0) {
2057                 vtnet_disable_tx_intr(sc);
2058                 sc->vtnet_stats.tx_task_rescheduled++;
2059                 VTNET_UNLOCK(sc);
2060                 goto again;
2061         }
2062
2063         VTNET_UNLOCK(sc);
2064 }
2065
2066 static void
2067 vtnet_stop(struct vtnet_softc *sc)
2068 {
2069         device_t dev;
2070         struct ifnet *ifp;
2071
2072         dev = sc->vtnet_dev;
2073         ifp = sc->vtnet_ifp;
2074
2075         VTNET_LOCK_ASSERT(sc);
2076
2077         sc->vtnet_watchdog_timer = 0;
2078         callout_stop(&sc->vtnet_tick_ch);
2079         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2080
2081         vtnet_disable_rx_intr(sc);
2082         vtnet_disable_tx_intr(sc);
2083
2084         /*
2085          * Stop the host VirtIO adapter. Note this will reset the host
2086          * adapter's state back to the pre-initialized state, so in
2087          * order to make the device usable again, we must drive it
2088          * through virtio_reinit() and virtio_reinit_complete().
2089          */
2090         virtio_stop(dev);
2091
2092         sc->vtnet_flags &= ~VTNET_FLAG_LINK;
2093
2094         vtnet_free_rx_mbufs(sc);
2095         vtnet_free_tx_mbufs(sc);
2096 }
2097
2098 static int
2099 vtnet_reinit(struct vtnet_softc *sc)
2100 {
2101         struct ifnet *ifp;
2102         uint64_t features;
2103
2104         ifp = sc->vtnet_ifp;
2105         features = sc->vtnet_features;
2106
2107         /*
2108          * Re-negotiate with the host, removing any disabled receive
2109          * features. Transmit features are disabled only on our side
2110          * via if_capenable and if_hwassist.
2111          */
2112
2113         if (ifp->if_capabilities & IFCAP_RXCSUM) {
2114                 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
2115                         features &= ~VIRTIO_NET_F_GUEST_CSUM;
2116         }
2117
2118         if (ifp->if_capabilities & IFCAP_LRO) {
2119                 if ((ifp->if_capenable & IFCAP_LRO) == 0)
2120                         features &= ~VTNET_LRO_FEATURES;
2121         }
2122
2123         if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
2124                 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
2125                         features &= ~VIRTIO_NET_F_CTRL_VLAN;
2126         }
2127
2128         return (virtio_reinit(sc->vtnet_dev, features));
2129 }
2130
2131 static void
2132 vtnet_init_locked(struct vtnet_softc *sc)
2133 {
2134         device_t dev;
2135         struct ifnet *ifp;
2136         int error;
2137
2138         dev = sc->vtnet_dev;
2139         ifp = sc->vtnet_ifp;
2140
2141         VTNET_LOCK_ASSERT(sc);
2142
2143         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2144                 return;
2145
2146         /* Stop host's adapter, cancel any pending I/O. */
2147         vtnet_stop(sc);
2148
2149         /* Reinitialize the host device. */
2150         error = vtnet_reinit(sc);
2151         if (error) {
2152                 device_printf(dev,
2153                     "reinitialization failed, stopping device...\n");
2154                 vtnet_stop(sc);
2155                 return;
2156         }
2157
2158         /* Update host with assigned MAC address. */
2159         bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
2160         vtnet_set_hwaddr(sc);
2161
2162         ifp->if_hwassist = 0;
2163         if (ifp->if_capenable & IFCAP_TXCSUM)
2164                 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
2165         if (ifp->if_capenable & IFCAP_TSO4)
2166                 ifp->if_hwassist |= CSUM_TSO;
2167
2168         error = vtnet_init_rx_vq(sc);
2169         if (error) {
2170                 device_printf(dev,
2171                     "cannot allocate mbufs for Rx virtqueue\n");
2172                 vtnet_stop(sc);
2173                 return;
2174         }
2175
2176         if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
2177                 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
2178                         /* Restore promiscuous and all-multicast modes. */
2179                         vtnet_rx_filter(sc);
2180
2181                         /* Restore filtered MAC addresses. */
2182                         vtnet_rx_filter_mac(sc);
2183                 }
2184
2185                 /* Restore VLAN filters. */
2186                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2187                         vtnet_rx_filter_vlan(sc);
2188         }
2189
2190 #ifdef DEVICE_POLLING
2191         if (ifp->if_capenable & IFCAP_POLLING) {
2192                 vtnet_disable_rx_intr(sc);
2193                 vtnet_disable_tx_intr(sc);
2194         } else
2195 #endif
2196         {
2197                 vtnet_enable_rx_intr(sc);
2198                 vtnet_enable_tx_intr(sc);
2199         }
2200
2201         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2202         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2203
2204         virtio_reinit_complete(dev);
2205
2206         vtnet_update_link_status(sc);
2207         callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
2208 }
2209
2210 static void
2211 vtnet_init(void *xsc)
2212 {
2213         struct vtnet_softc *sc;
2214
2215         sc = xsc;
2216
2217         VTNET_LOCK(sc);
2218         vtnet_init_locked(sc);
2219         VTNET_UNLOCK(sc);
2220 }
2221
2222 static void
2223 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
2224     struct sglist *sg, int readable, int writable)
2225 {
2226         struct virtqueue *vq;
2227         void *c;
2228
2229         vq = sc->vtnet_ctrl_vq;
2230
2231         VTNET_LOCK_ASSERT(sc);
2232         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
2233             ("no control virtqueue"));
2234         KASSERT(virtqueue_empty(vq),
2235             ("control command already enqueued"));
2236
2237         if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
2238                 return;
2239
2240         virtqueue_notify(vq);
2241
2242         /*
2243          * Poll until the command is complete. Previously, we would
2244          * sleep until the control virtqueue interrupt handler woke
2245          * us up, but dropping the VTNET_MTX leads to serialization
2246          * difficulties.
2247          *
2248          * Furthermore, it appears QEMU/KVM only allocates three MSIX
2249          * vectors. Two of those vectors are needed for the Rx and Tx
2250          * virtqueues. We do not support sharing both a Vq and config
2251          * changed notification on the same MSIX vector.
2252          */
2253         c = virtqueue_poll(vq, NULL);
2254         KASSERT(c == cookie, ("unexpected control command response"));
2255 }
2256
2257 static void
2258 vtnet_rx_filter(struct vtnet_softc *sc)
2259 {
2260         device_t dev;
2261         struct ifnet *ifp;
2262
2263         dev = sc->vtnet_dev;
2264         ifp = sc->vtnet_ifp;
2265
2266         VTNET_LOCK_ASSERT(sc);
2267         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2268             ("CTRL_RX feature not negotiated"));
2269
2270         if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
2271                 device_printf(dev, "cannot %s promiscuous mode\n",
2272                     ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
2273
2274         if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
2275                 device_printf(dev, "cannot %s all-multicast mode\n",
2276                     ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
2277 }
2278
2279 static int
2280 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
2281 {
2282         struct virtio_net_ctrl_hdr hdr;
2283         struct sglist_seg segs[3];
2284         struct sglist sg;
2285         uint8_t onoff, ack;
2286         int error;
2287
2288         if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
2289                 return (ENOTSUP);
2290
2291         error = 0;
2292
2293         hdr.class = VIRTIO_NET_CTRL_RX;
2294         hdr.cmd = cmd;
2295         onoff = !!on;
2296         ack = VIRTIO_NET_ERR;
2297
2298         sglist_init(&sg, 3, segs);
2299         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2300         error |= sglist_append(&sg, &onoff, sizeof(uint8_t));
2301         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2302         KASSERT(error == 0 && sg.sg_nseg == 3,
2303             ("error adding Rx filter message to sglist"));
2304
2305         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2306
2307         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2308 }
2309
2310 static int
2311 vtnet_set_promisc(struct vtnet_softc *sc, int on)
2312 {
2313
2314         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
2315 }
2316
2317 static int
2318 vtnet_set_allmulti(struct vtnet_softc *sc, int on)
2319 {
2320
2321         return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
2322 }
2323
2324 static void
2325 vtnet_rx_filter_mac(struct vtnet_softc *sc)
2326 {
2327         struct virtio_net_ctrl_hdr hdr;
2328         struct vtnet_mac_filter *filter;
2329         struct sglist_seg segs[4];
2330         struct sglist sg;
2331         struct ifnet *ifp;
2332         struct ifaddr *ifa;
2333         struct ifmultiaddr *ifma;
2334         int ucnt, mcnt, promisc, allmulti, error;
2335         uint8_t ack;
2336
2337         ifp = sc->vtnet_ifp;
2338         filter = sc->vtnet_mac_filter;
2339         ucnt = 0;
2340         mcnt = 0;
2341         promisc = 0;
2342         allmulti = 0;
2343         error = 0;
2344
2345         VTNET_LOCK_ASSERT(sc);
2346         KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
2347             ("CTRL_RX feature not negotiated"));
2348
2349         /* Unicast MAC addresses: */
2350         if_addr_rlock(ifp);
2351         TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
2352                 if (ifa->ifa_addr->sa_family != AF_LINK)
2353                         continue;
2354                 else if (ucnt == VTNET_MAX_MAC_ENTRIES)
2355                         break;
2356
2357                 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
2358                     &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
2359                 ucnt++;
2360         }
2361         if_addr_runlock(ifp);
2362
2363         if (ucnt >= VTNET_MAX_MAC_ENTRIES) {
2364                 promisc = 1;
2365                 filter->vmf_unicast.nentries = 0;
2366
2367                 if_printf(ifp, "more than %d MAC addresses assigned, "
2368                     "falling back to promiscuous mode\n",
2369                     VTNET_MAX_MAC_ENTRIES);
2370         } else
2371                 filter->vmf_unicast.nentries = ucnt;
2372
2373         /* Multicast MAC addresses: */
2374         if_maddr_rlock(ifp);
2375         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2376                 if (ifma->ifma_addr->sa_family != AF_LINK)
2377                         continue;
2378                 else if (mcnt == VTNET_MAX_MAC_ENTRIES)
2379                         break;
2380
2381                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2382                     &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
2383                 mcnt++;
2384         }
2385         if_maddr_runlock(ifp);
2386
2387         if (mcnt >= VTNET_MAX_MAC_ENTRIES) {
2388                 allmulti = 1;
2389                 filter->vmf_multicast.nentries = 0;
2390
2391                 if_printf(ifp, "more than %d multicast MAC addresses "
2392                     "assigned, falling back to all-multicast mode\n",
2393                     VTNET_MAX_MAC_ENTRIES);
2394         } else
2395                 filter->vmf_multicast.nentries = mcnt;
2396
2397         if (promisc && allmulti)
2398                 goto out;
2399
2400         hdr.class = VIRTIO_NET_CTRL_MAC;
2401         hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
2402         ack = VIRTIO_NET_ERR;
2403
2404         sglist_init(&sg, 4, segs);
2405         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2406         error |= sglist_append(&sg, &filter->vmf_unicast,
2407             sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN);
2408         error |= sglist_append(&sg, &filter->vmf_multicast,
2409             sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN);
2410         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2411         KASSERT(error == 0 && sg.sg_nseg == 4,
2412             ("error adding MAC filtering message to sglist"));
2413
2414         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2415
2416         if (ack != VIRTIO_NET_OK)
2417                 if_printf(ifp, "error setting host MAC filter table\n");
2418
2419 out:
2420         if (promisc)
2421                 if (vtnet_set_promisc(sc, 1) != 0)
2422                         if_printf(ifp, "cannot enable promiscuous mode\n");
2423         if (allmulti)
2424                 if (vtnet_set_allmulti(sc, 1) != 0)
2425                         if_printf(ifp, "cannot enable all-multicast mode\n");
2426 }
2427
2428 static int
2429 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2430 {
2431         struct virtio_net_ctrl_hdr hdr;
2432         struct sglist_seg segs[3];
2433         struct sglist sg;
2434         uint8_t ack;
2435         int error;
2436
2437         hdr.class = VIRTIO_NET_CTRL_VLAN;
2438         hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
2439         ack = VIRTIO_NET_ERR;
2440         error = 0;
2441
2442         sglist_init(&sg, 3, segs);
2443         error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
2444         error |= sglist_append(&sg, &tag, sizeof(uint16_t));
2445         error |= sglist_append(&sg, &ack, sizeof(uint8_t));
2446         KASSERT(error == 0 && sg.sg_nseg == 3,
2447             ("error adding VLAN control message to sglist"));
2448
2449         vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
2450
2451         return (ack == VIRTIO_NET_OK ? 0 : EIO);
2452 }
2453
2454 static void
2455 vtnet_rx_filter_vlan(struct vtnet_softc *sc)
2456 {
2457         device_t dev;
2458         uint32_t w, mask;
2459         uint16_t tag;
2460         int i, nvlans, error;
2461
2462         VTNET_LOCK_ASSERT(sc);
2463         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2464             ("VLAN_FILTER feature not negotiated"));
2465
2466         dev = sc->vtnet_dev;
2467         nvlans = sc->vtnet_nvlans;
2468         error = 0;
2469
2470         /* Enable filtering for each configured VLAN. */
2471         for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
2472                 w = sc->vtnet_vlan_shadow[i];
2473                 for (mask = 1, tag = i * 32; w != 0; mask <<= 1, tag++) {
2474                         if ((w & mask) != 0) {
2475                                 w &= ~mask;
2476                                 nvlans--;
2477                                 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0)
2478                                         error++;
2479                         }
2480                 }
2481         }
2482
2483         KASSERT(nvlans == 0, ("VLAN count incorrect"));
2484         if (error)
2485                 device_printf(dev, "cannot restore VLAN filter table\n");
2486 }
2487
2488 static void
2489 vtnet_set_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
2490 {
2491         struct ifnet *ifp;
2492         int idx, bit;
2493
2494         KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
2495             ("VLAN_FILTER feature not negotiated"));
2496
2497         if ((tag == 0) || (tag > 4095))
2498                 return;
2499
2500         ifp = sc->vtnet_ifp;
2501         idx = (tag >> 5) & 0x7F;
2502         bit = tag & 0x1F;
2503
2504         VTNET_LOCK(sc);
2505
2506         /* Update shadow VLAN table. */
2507         if (add) {
2508                 sc->vtnet_nvlans++;
2509                 sc->vtnet_vlan_shadow[idx] |= (1 << bit);
2510         } else {
2511                 sc->vtnet_nvlans--;
2512                 sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
2513         }
2514
2515         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
2516                 if (vtnet_exec_vlan_filter(sc, add, tag) != 0) {
2517                         device_printf(sc->vtnet_dev,
2518                             "cannot %s VLAN %d %s the host filter table\n",
2519                             add ? "add" : "remove", tag,
2520                             add ? "to" : "from");
2521                 }
2522         }
2523
2524         VTNET_UNLOCK(sc);
2525 }
2526
2527 static void
2528 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2529 {
2530
2531         if (ifp->if_softc != arg)
2532                 return;
2533
2534         vtnet_set_vlan_filter(arg, 1, tag);
2535 }
2536
2537 static void
2538 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2539 {
2540
2541         if (ifp->if_softc != arg)
2542                 return;
2543
2544         vtnet_set_vlan_filter(arg, 0, tag);
2545 }
2546
2547 static int
2548 vtnet_ifmedia_upd(struct ifnet *ifp)
2549 {
2550         struct vtnet_softc *sc;
2551         struct ifmedia *ifm;
2552
2553         sc = ifp->if_softc;
2554         ifm = &sc->vtnet_media;
2555
2556         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2557                 return (EINVAL);
2558
2559         return (0);
2560 }
2561
2562 static void
2563 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
2564 {
2565         struct vtnet_softc *sc;
2566
2567         sc = ifp->if_softc;
2568
2569         ifmr->ifm_status = IFM_AVALID;
2570         ifmr->ifm_active = IFM_ETHER;
2571
2572         VTNET_LOCK(sc);
2573         if (vtnet_is_link_up(sc) != 0) {
2574                 ifmr->ifm_status |= IFM_ACTIVE;
2575                 ifmr->ifm_active |= VTNET_MEDIATYPE;
2576         } else
2577                 ifmr->ifm_active |= IFM_NONE;
2578         VTNET_UNLOCK(sc);
2579 }
2580
2581 static void
2582 vtnet_add_statistics(struct vtnet_softc *sc)
2583 {
2584         device_t dev;
2585         struct vtnet_statistics *stats;
2586         struct sysctl_ctx_list *ctx;
2587         struct sysctl_oid *tree;
2588         struct sysctl_oid_list *child;
2589
2590         dev = sc->vtnet_dev;
2591         stats = &sc->vtnet_stats;
2592         ctx = device_get_sysctl_ctx(dev);
2593         tree = device_get_sysctl_tree(dev);
2594         child = SYSCTL_CHILDREN(tree);
2595
2596         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_failed",
2597             CTLFLAG_RD, &stats->mbuf_alloc_failed,
2598             "Mbuf cluster allocation failures");
2599
2600         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_frame_too_large",
2601             CTLFLAG_RD, &stats->rx_frame_too_large,
2602             "Received frame larger than the mbuf chain");
2603         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
2604             CTLFLAG_RD, &stats->rx_enq_replacement_failed,
2605             "Enqueuing the replacement receive mbuf failed");
2606         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_mergeable_failed",
2607             CTLFLAG_RD, &stats->rx_mergeable_failed,
2608             "Mergeable buffers receive failures");
2609         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
2610             CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
2611             "Received checksum offloaded buffer with unsupported "
2612             "Ethernet type");
2613         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_start",
2614             CTLFLAG_RD, &stats->rx_csum_bad_start,
2615             "Received checksum offloaded buffer with incorrect start offset");
2616         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
2617             CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
2618             "Received checksum offloaded buffer with incorrect IP protocol");
2619         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_offset",
2620             CTLFLAG_RD, &stats->rx_csum_bad_offset,
2621             "Received checksum offloaded buffer with incorrect offset");
2622         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_failed",
2623             CTLFLAG_RD, &stats->rx_csum_failed,
2624             "Received buffer checksum offload failed");
2625         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_offloaded",
2626             CTLFLAG_RD, &stats->rx_csum_offloaded,
2627             "Received buffer checksum offload succeeded");
2628         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_task_rescheduled",
2629             CTLFLAG_RD, &stats->rx_task_rescheduled,
2630             "Times the receive interrupt task rescheduled itself");
2631
2632         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_csum_offloaded",
2633             CTLFLAG_RD, &stats->tx_csum_offloaded,
2634             "Offloaded checksum of transmitted buffer");
2635         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_tso_offloaded",
2636             CTLFLAG_RD, &stats->tx_tso_offloaded,
2637             "Segmentation offload of transmitted buffer");
2638         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_csum_bad_ethtype",
2639             CTLFLAG_RD, &stats->tx_csum_bad_ethtype,
2640             "Aborted transmit of checksum offloaded buffer with unknown "
2641             "Ethernet type");
2642         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_tso_bad_ethtype",
2643             CTLFLAG_RD, &stats->tx_tso_bad_ethtype,
2644             "Aborted transmit of TSO buffer with unknown Ethernet type");
2645         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_task_rescheduled",
2646             CTLFLAG_RD, &stats->tx_task_rescheduled,
2647             "Times the transmit interrupt task rescheduled itself");
2648 }
2649
2650 static int
2651 vtnet_enable_rx_intr(struct vtnet_softc *sc)
2652 {
2653
2654         return (virtqueue_enable_intr(sc->vtnet_rx_vq));
2655 }
2656
2657 static void
2658 vtnet_disable_rx_intr(struct vtnet_softc *sc)
2659 {
2660
2661         virtqueue_disable_intr(sc->vtnet_rx_vq);
2662 }
2663
2664 static int
2665 vtnet_enable_tx_intr(struct vtnet_softc *sc)
2666 {
2667
2668 #ifdef VTNET_TX_INTR_MODERATION
2669         return (0);
2670 #else
2671         return (virtqueue_enable_intr(sc->vtnet_tx_vq));
2672 #endif
2673 }
2674
2675 static void
2676 vtnet_disable_tx_intr(struct vtnet_softc *sc)
2677 {
2678
2679         virtqueue_disable_intr(sc->vtnet_tx_vq);
2680 }