sys/dev/vmware/vmxnet3/if_vmx.c

   1 /*-
   2  * Copyright (c) 2013 Tsubai Masanari
   3  * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
   4  *
   5  * Permission to use, copy, modify, and distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  16  *
  17  * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $
  18  */
  19
  20 /* Driver for VMware vmxnet3 virtual ethernet devices. */
  21
  22 #include <sys/cdefs.h>
  23 __FBSDID("$FreeBSD$");
  24
  25 #include <sys/param.h>
  26 #include <sys/systm.h>
  27 #include <sys/eventhandler.h>
  28 #include <sys/kernel.h>
  29 #include <sys/endian.h>
  30 #include <sys/sockio.h>
  31 #include <sys/mbuf.h>
  32 #include <sys/malloc.h>
  33 #include <sys/module.h>
  34 #include <sys/socket.h>
  35 #include <sys/sysctl.h>
  36 #include <sys/smp.h>
  37 #include <sys/taskqueue.h>
  38 #include <vm/vm.h>
  39 #include <vm/pmap.h>
  40
  41 #include <net/ethernet.h>
  42 #include <net/if.h>
  43 #include <net/if_var.h>
  44 #include <net/if_arp.h>
  45 #include <net/if_dl.h>
  46 #include <net/if_types.h>
  47 #include <net/if_media.h>
  48 #include <net/if_vlan_var.h>
  49
  50 #include <net/bpf.h>
  51
  52 #include <netinet/in_systm.h>
  53 #include <netinet/in.h>
  54 #include <netinet/ip.h>
  55 #include <netinet/ip6.h>
  56 #include <netinet6/ip6_var.h>
  57 #include <netinet/udp.h>
  58 #include <netinet/tcp.h>
  59
  60 #include <machine/in_cksum.h>
  61
  62 #include <machine/bus.h>
  63 #include <machine/resource.h>
  64 #include <sys/bus.h>
  65 #include <sys/rman.h>
  66
  67 #include <dev/pci/pcireg.h>
  68 #include <dev/pci/pcivar.h>
  69
  70 #include "if_vmxreg.h"
  71 #include "if_vmxvar.h"
  72
  73 #include "opt_inet.h"
  74 #include "opt_inet6.h"
  75
  76 #ifdef VMXNET3_FAILPOINTS
  77 #include <sys/fail.h>
  78 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0,
  79     "vmxnet3 fail points");
  80 #define VMXNET3_FP      _debug_fail_point_vmxnet3
  81 #endif
  82
  83 static int      vmxnet3_probe(device_t);
  84 static int      vmxnet3_attach(device_t);
  85 static int      vmxnet3_detach(device_t);
  86 static int      vmxnet3_shutdown(device_t);
  87
  88 static int      vmxnet3_alloc_resources(struct vmxnet3_softc *);
  89 static void     vmxnet3_free_resources(struct vmxnet3_softc *);
  90 static int      vmxnet3_check_version(struct vmxnet3_softc *);
  91 static void     vmxnet3_initial_config(struct vmxnet3_softc *);
  92 static void     vmxnet3_check_multiqueue(struct vmxnet3_softc *);
  93
  94 static int      vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
  95 static int      vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
  96 static int      vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
  97 static int      vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int,
  98                     struct vmxnet3_interrupt *);
  99 static int      vmxnet3_alloc_intr_resources(struct vmxnet3_softc *);
 100 static int      vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
 101 static int      vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
 102 static int      vmxnet3_setup_interrupts(struct vmxnet3_softc *);
 103 static int      vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
 104
 105 static void     vmxnet3_free_interrupt(struct vmxnet3_softc *,
 106                     struct vmxnet3_interrupt *);
 107 static void     vmxnet3_free_interrupts(struct vmxnet3_softc *);
 108
 109 #ifndef VMXNET3_LEGACY_TX
 110 static int      vmxnet3_alloc_taskqueue(struct vmxnet3_softc *);
 111 static void     vmxnet3_start_taskqueue(struct vmxnet3_softc *);
 112 static void     vmxnet3_drain_taskqueue(struct vmxnet3_softc *);
 113 static void     vmxnet3_free_taskqueue(struct vmxnet3_softc *);
 114 #endif
 115
 116 static int      vmxnet3_init_rxq(struct vmxnet3_softc *, int);
 117 static int      vmxnet3_init_txq(struct vmxnet3_softc *, int);
 118 static int      vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
 119 static void     vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
 120 static void     vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
 121 static void     vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);
 122
 123 static int      vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
 124 static void     vmxnet3_free_shared_data(struct vmxnet3_softc *);
 125 static int      vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
 126 static void     vmxnet3_free_txq_data(struct vmxnet3_softc *);
 127 static int      vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
 128 static void     vmxnet3_free_rxq_data(struct vmxnet3_softc *);
 129 static int      vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
 130 static void     vmxnet3_free_queue_data(struct vmxnet3_softc *);
 131 static int      vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
 132 static void     vmxnet3_init_shared_data(struct vmxnet3_softc *);
 133 static void     vmxnet3_init_hwassist(struct vmxnet3_softc *);
 134 static void     vmxnet3_reinit_interface(struct vmxnet3_softc *);
 135 static void     vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
 136 static void     vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
 137 static int      vmxnet3_alloc_data(struct vmxnet3_softc *);
 138 static void     vmxnet3_free_data(struct vmxnet3_softc *);
 139 static int      vmxnet3_setup_interface(struct vmxnet3_softc *);
 140
 141 static void     vmxnet3_evintr(struct vmxnet3_softc *);
 142 static void     vmxnet3_txq_eof(struct vmxnet3_txqueue *);
 143 static void     vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
 144 static int      vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
 145 static void     vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
 146                     struct vmxnet3_rxring *, int);
 147 static void     vmxnet3_rxq_eof(struct vmxnet3_rxqueue *);
 148 static void     vmxnet3_legacy_intr(void *);
 149 static void     vmxnet3_txq_intr(void *);
 150 static void     vmxnet3_rxq_intr(void *);
 151 static void     vmxnet3_event_intr(void *);
 152
 153 static void     vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
 154 static void     vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
 155 static void     vmxnet3_stop(struct vmxnet3_softc *);
 156
 157 static void     vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
 158 static int      vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
 159 static int      vmxnet3_reinit_queues(struct vmxnet3_softc *);
 160 static int      vmxnet3_enable_device(struct vmxnet3_softc *);
 161 static void     vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
 162 static int      vmxnet3_reinit(struct vmxnet3_softc *);
 163 static void     vmxnet3_init_locked(struct vmxnet3_softc *);
 164 static void     vmxnet3_init(void *);
 165
 166 static int      vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *,
 167                     int *, int *, int *);
 168 static int      vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **,
 169                     bus_dmamap_t, bus_dma_segment_t [], int *);
 170 static void     vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
 171 static int      vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
 172 #ifdef VMXNET3_LEGACY_TX
 173 static void     vmxnet3_start_locked(struct ifnet *);
 174 static void     vmxnet3_start(struct ifnet *);
 175 #else
 176 static int      vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *,
 177                     struct mbuf *);
 178 static int      vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *);
 179 static void     vmxnet3_txq_tq_deferred(void *, int);
 180 #endif
 181 static void     vmxnet3_txq_start(struct vmxnet3_txqueue *);
 182 static void     vmxnet3_tx_start_all(struct vmxnet3_softc *);
 183
 184 static void     vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
 185                     uint16_t);
 186 static void     vmxnet3_register_vlan(void *, struct ifnet *, uint16_t);
 187 static void     vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t);
 188 static void     vmxnet3_set_rxfilter(struct vmxnet3_softc *);
 189 static int      vmxnet3_change_mtu(struct vmxnet3_softc *, int);
 190 static int      vmxnet3_ioctl(struct ifnet *, u_long, caddr_t);
 191 static uint64_t vmxnet3_get_counter(struct ifnet *, ift_counter);
 192
 193 #ifndef VMXNET3_LEGACY_TX
 194 static void     vmxnet3_qflush(struct ifnet *);
 195 #endif
 196
 197 static int      vmxnet3_watchdog(struct vmxnet3_txqueue *);
 198 static void     vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
 199 static void     vmxnet3_tick(void *);
 200 static void     vmxnet3_link_status(struct vmxnet3_softc *);
 201 static void     vmxnet3_media_status(struct ifnet *, struct ifmediareq *);
 202 static int      vmxnet3_media_change(struct ifnet *);
 203 static void     vmxnet3_set_lladdr(struct vmxnet3_softc *);
 204 static void     vmxnet3_get_lladdr(struct vmxnet3_softc *);
 205
 206 static void     vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *,
 207                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 208 static void     vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *,
 209                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 210 static void     vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *,
 211                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 212 static void     vmxnet3_setup_sysctl(struct vmxnet3_softc *);
 213
 214 static void     vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t,
 215                     uint32_t);
 216 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t);
 217 static void     vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t,
 218                     uint32_t);
 219 static void     vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t);
 220 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t);
 221
 222 static void     vmxnet3_enable_intr(struct vmxnet3_softc *, int);
 223 static void     vmxnet3_disable_intr(struct vmxnet3_softc *, int);
 224 static void     vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
 225 static void     vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
 226
 227 static int      vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t,
 228                     bus_size_t, struct vmxnet3_dma_alloc *);
 229 static void     vmxnet3_dma_free(struct vmxnet3_softc *,
 230                     struct vmxnet3_dma_alloc *);
 231 static int      vmxnet3_tunable_int(struct vmxnet3_softc *,
 232                     const char *, int);
 233
 234 typedef enum {
 235         VMXNET3_BARRIER_RD,
 236         VMXNET3_BARRIER_WR,
 237         VMXNET3_BARRIER_RDWR,
 238 } vmxnet3_barrier_t;
 239
 240 static void     vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
 241
 242 /* Tunables. */
 243 static int vmxnet3_mq_disable = 0;
 244 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable);
 245 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES;
 246 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue);
 247 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES;
 248 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue);
 249 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC;
 250 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc);
 251 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC;
 252 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc);
 253
 254 static device_method_t vmxnet3_methods[] = {
 255         /* Device interface. */
 256         DEVMETHOD(device_probe,         vmxnet3_probe),
 257         DEVMETHOD(device_attach,        vmxnet3_attach),
 258         DEVMETHOD(device_detach,        vmxnet3_detach),
 259         DEVMETHOD(device_shutdown,      vmxnet3_shutdown),
 260
 261         DEVMETHOD_END
 262 };
 263
 264 static driver_t vmxnet3_driver = {
 265         "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc)
 266 };
 267
 268 static devclass_t vmxnet3_devclass;
 269 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0);
 270
 271 MODULE_DEPEND(vmx, pci, 1, 1, 1);
 272 MODULE_DEPEND(vmx, ether, 1, 1, 1);
 273
 274 #define VMXNET3_VMWARE_VENDOR_ID        0x15AD
 275 #define VMXNET3_VMWARE_DEVICE_ID        0x07B0
 276
 277 static int
 278 vmxnet3_probe(device_t dev)
 279 {
 280
 281         if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID &&
 282             pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) {
 283                 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter");
 284                 return (BUS_PROBE_DEFAULT);
 285         }
 286
 287         return (ENXIO);
 288 }
 289
 290 static int
 291 vmxnet3_attach(device_t dev)
 292 {
 293         struct vmxnet3_softc *sc;
 294         int error;
 295
 296         sc = device_get_softc(dev);
 297         sc->vmx_dev = dev;
 298
 299         pci_enable_busmaster(dev);
 300
 301         VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev));
 302         callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0);
 303
 304         vmxnet3_initial_config(sc);
 305
 306         error = vmxnet3_alloc_resources(sc);
 307         if (error)
 308                 goto fail;
 309
 310         error = vmxnet3_check_version(sc);
 311         if (error)
 312                 goto fail;
 313
 314         error = vmxnet3_alloc_rxtx_queues(sc);
 315         if (error)
 316                 goto fail;
 317
 318 #ifndef VMXNET3_LEGACY_TX
 319         error = vmxnet3_alloc_taskqueue(sc);
 320         if (error)
 321                 goto fail;
 322 #endif
 323
 324         error = vmxnet3_alloc_interrupts(sc);
 325         if (error)
 326                 goto fail;
 327
 328         vmxnet3_check_multiqueue(sc);
 329
 330         error = vmxnet3_alloc_data(sc);
 331         if (error)
 332                 goto fail;
 333
 334         error = vmxnet3_setup_interface(sc);
 335         if (error)
 336                 goto fail;
 337
 338         error = vmxnet3_setup_interrupts(sc);
 339         if (error) {
 340                 ether_ifdetach(sc->vmx_ifp);
 341                 device_printf(dev, "could not set up interrupt\n");
 342                 goto fail;
 343         }
 344
 345         vmxnet3_setup_sysctl(sc);
 346 #ifndef VMXNET3_LEGACY_TX
 347         vmxnet3_start_taskqueue(sc);
 348 #endif
 349
 350 fail:
 351         if (error)
 352                 vmxnet3_detach(dev);
 353
 354         return (error);
 355 }
 356
 357 static int
 358 vmxnet3_detach(device_t dev)
 359 {
 360         struct vmxnet3_softc *sc;
 361         struct ifnet *ifp;
 362
 363         sc = device_get_softc(dev);
 364         ifp = sc->vmx_ifp;
 365
 366         if (device_is_attached(dev)) {
 367                 VMXNET3_CORE_LOCK(sc);
 368                 vmxnet3_stop(sc);
 369                 VMXNET3_CORE_UNLOCK(sc);
 370
 371                 callout_drain(&sc->vmx_tick);
 372 #ifndef VMXNET3_LEGACY_TX
 373                 vmxnet3_drain_taskqueue(sc);
 374 #endif
 375
 376                 ether_ifdetach(ifp);
 377         }
 378
 379         if (sc->vmx_vlan_attach != NULL) {
 380                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach);
 381                 sc->vmx_vlan_attach = NULL;
 382         }
 383         if (sc->vmx_vlan_detach != NULL) {
 384                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach);
 385                 sc->vmx_vlan_detach = NULL;
 386         }
 387
 388 #ifndef VMXNET3_LEGACY_TX
 389         vmxnet3_free_taskqueue(sc);
 390 #endif
 391         vmxnet3_free_interrupts(sc);
 392
 393         if (ifp != NULL) {
 394                 if_free(ifp);
 395                 sc->vmx_ifp = NULL;
 396         }
 397
 398         ifmedia_removeall(&sc->vmx_media);
 399
 400         vmxnet3_free_data(sc);
 401         vmxnet3_free_resources(sc);
 402         vmxnet3_free_rxtx_queues(sc);
 403
 404         VMXNET3_CORE_LOCK_DESTROY(sc);
 405
 406         return (0);
 407 }
 408
 409 static int
 410 vmxnet3_shutdown(device_t dev)
 411 {
 412
 413         return (0);
 414 }
 415
 416 static int
 417 vmxnet3_alloc_resources(struct vmxnet3_softc *sc)
 418 {
 419         device_t dev;
 420         int rid;
 421
 422         dev = sc->vmx_dev;
 423
 424         rid = PCIR_BAR(0);
 425         sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 426             RF_ACTIVE);
 427         if (sc->vmx_res0 == NULL) {
 428                 device_printf(dev,
 429                     "could not map BAR0 memory\n");
 430                 return (ENXIO);
 431         }
 432
 433         sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0);
 434         sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0);
 435
 436         rid = PCIR_BAR(1);
 437         sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 438             RF_ACTIVE);
 439         if (sc->vmx_res1 == NULL) {
 440                 device_printf(dev,
 441                     "could not map BAR1 memory\n");
 442                 return (ENXIO);
 443         }
 444
 445         sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1);
 446         sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1);
 447
 448         if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
 449                 rid = PCIR_BAR(2);
 450                 sc->vmx_msix_res = bus_alloc_resource_any(dev,
 451                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 452         }
 453
 454         if (sc->vmx_msix_res == NULL)
 455                 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
 456
 457         return (0);
 458 }
 459
 460 static void
 461 vmxnet3_free_resources(struct vmxnet3_softc *sc)
 462 {
 463         device_t dev;
 464         int rid;
 465
 466         dev = sc->vmx_dev;
 467
 468         if (sc->vmx_res0 != NULL) {
 469                 rid = PCIR_BAR(0);
 470                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0);
 471                 sc->vmx_res0 = NULL;
 472         }
 473
 474         if (sc->vmx_res1 != NULL) {
 475                 rid = PCIR_BAR(1);
 476                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1);
 477                 sc->vmx_res1 = NULL;
 478         }
 479
 480         if (sc->vmx_msix_res != NULL) {
 481                 rid = PCIR_BAR(2);
 482                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
 483                     sc->vmx_msix_res);
 484                 sc->vmx_msix_res = NULL;
 485         }
 486 }
 487
 488 static int
 489 vmxnet3_check_version(struct vmxnet3_softc *sc)
 490 {
 491         device_t dev;
 492         uint32_t version;
 493
 494         dev = sc->vmx_dev;
 495
 496         version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
 497         if ((version & 0x01) == 0) {
 498                 device_printf(dev, "unsupported hardware version %#x\n",
 499                     version);
 500                 return (ENOTSUP);
 501         }
 502         vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
 503
 504         version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
 505         if ((version & 0x01) == 0) {
 506                 device_printf(dev, "unsupported UPT version %#x\n", version);
 507                 return (ENOTSUP);
 508         }
 509         vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
 510
 511         return (0);
 512 }
 513
 514 static int
 515 trunc_powerof2(int val)
 516 {
 517
 518         return (1U << (fls(val) - 1));
 519 }
 520
 521 static void
 522 vmxnet3_initial_config(struct vmxnet3_softc *sc)
 523 {
 524         int nqueue, ndesc;
 525
 526         nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue);
 527         if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1)
 528                 nqueue = VMXNET3_DEF_TX_QUEUES;
 529         if (nqueue > mp_ncpus)
 530                 nqueue = mp_ncpus;
 531         sc->vmx_max_ntxqueues = trunc_powerof2(nqueue);
 532
 533         nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue);
 534         if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1)
 535                 nqueue = VMXNET3_DEF_RX_QUEUES;
 536         if (nqueue > mp_ncpus)
 537                 nqueue = mp_ncpus;
 538         sc->vmx_max_nrxqueues = trunc_powerof2(nqueue);
 539
 540         if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) {
 541                 sc->vmx_max_nrxqueues = 1;
 542                 sc->vmx_max_ntxqueues = 1;
 543         }
 544
 545         ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc);
 546         if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC)
 547                 ndesc = VMXNET3_DEF_TX_NDESC;
 548         if (ndesc & VMXNET3_MASK_TX_NDESC)
 549                 ndesc &= ~VMXNET3_MASK_TX_NDESC;
 550         sc->vmx_ntxdescs = ndesc;
 551
 552         ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc);
 553         if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC)
 554                 ndesc = VMXNET3_DEF_RX_NDESC;
 555         if (ndesc & VMXNET3_MASK_RX_NDESC)
 556                 ndesc &= ~VMXNET3_MASK_RX_NDESC;
 557         sc->vmx_nrxdescs = ndesc;
 558         sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
 559 }
 560
 561 static void
 562 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
 563 {
 564
 565         if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
 566                 goto out;
 567
 568         /* BMV: Just use the maximum configured for now. */
 569         sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
 570         sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;
 571
 572         if (sc->vmx_nrxqueues > 1)
 573                 sc->vmx_flags |= VMXNET3_FLAG_RSS;
 574
 575         return;
 576
 577 out:
 578         sc->vmx_ntxqueues = 1;
 579         sc->vmx_nrxqueues = 1;
 580 }
 581
 582 static int
 583 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
 584 {
 585         device_t dev;
 586         int nmsix, cnt, required;
 587
 588         dev = sc->vmx_dev;
 589
 590         if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
 591                 return (1);
 592
 593         /* Allocate an additional vector for the events interrupt. */
 594         required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1;
 595
 596         nmsix = pci_msix_count(dev);
 597         if (nmsix < required)
 598                 return (1);
 599
 600         cnt = required;
 601         if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
 602                 sc->vmx_nintrs = required;
 603                 return (0);
 604         } else
 605                 pci_release_msi(dev);
 606
 607         /* BMV TODO Fallback to sharing MSIX vectors if possible. */
 608
 609         return (1);
 610 }
 611
 612 static int
 613 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
 614 {
 615         device_t dev;
 616         int nmsi, cnt, required;
 617
 618         dev = sc->vmx_dev;
 619         required = 1;
 620
 621         nmsi = pci_msi_count(dev);
 622         if (nmsi < required)
 623                 return (1);
 624
 625         cnt = required;
 626         if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) {
 627                 sc->vmx_nintrs = 1;
 628                 return (0);
 629         } else
 630                 pci_release_msi(dev);
 631
 632         return (1);
 633 }
 634
 635 static int
 636 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
 637 {
 638
 639         sc->vmx_nintrs = 1;
 640         return (0);
 641 }
 642
 643 static int
 644 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags,
 645     struct vmxnet3_interrupt *intr)
 646 {
 647         struct resource *irq;
 648
 649         irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags);
 650         if (irq == NULL)
 651                 return (ENXIO);
 652
 653         intr->vmxi_irq = irq;
 654         intr->vmxi_rid = rid;
 655
 656         return (0);
 657 }
 658
 659 static int
 660 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc)
 661 {
 662         int i, rid, flags, error;
 663
 664         rid = 0;
 665         flags = RF_ACTIVE;
 666
 667         if (sc->vmx_intr_type == VMXNET3_IT_LEGACY)
 668                 flags |= RF_SHAREABLE;
 669         else
 670                 rid = 1;
 671
 672         for (i = 0; i < sc->vmx_nintrs; i++, rid++) {
 673                 error = vmxnet3_alloc_interrupt(sc, rid, flags,
 674                     &sc->vmx_intrs[i]);
 675                 if (error)
 676                         return (error);
 677         }
 678
 679         return (0);
 680 }
 681
 682 static int
 683 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
 684 {
 685         device_t dev;
 686         struct vmxnet3_txqueue *txq;
 687         struct vmxnet3_rxqueue *rxq;
 688         struct vmxnet3_interrupt *intr;
 689         enum intr_type type;
 690         int i, error;
 691
 692         dev = sc->vmx_dev;
 693         intr = &sc->vmx_intrs[0];
 694         type = INTR_TYPE_NET | INTR_MPSAFE;
 695
 696         for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) {
 697                 txq = &sc->vmx_txq[i];
 698                 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
 699                      vmxnet3_txq_intr, txq, &intr->vmxi_handler);
 700                 if (error)
 701                         return (error);
 702                 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
 703                     "tq%d", i);
 704                 txq->vxtxq_intr_idx = intr->vmxi_rid - 1;
 705         }
 706
 707         for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) {
 708                 rxq = &sc->vmx_rxq[i];
 709                 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
 710                     vmxnet3_rxq_intr, rxq, &intr->vmxi_handler);
 711                 if (error)
 712                         return (error);
 713                 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
 714                     "rq%d", i);
 715                 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1;
 716         }
 717
 718         error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
 719             vmxnet3_event_intr, sc, &intr->vmxi_handler);
 720         if (error)
 721                 return (error);
 722         bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event");
 723         sc->vmx_event_intr_idx = intr->vmxi_rid - 1;
 724
 725         return (0);
 726 }
 727
 728 static int
 729 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
 730 {
 731         struct vmxnet3_interrupt *intr;
 732         int i, error;
 733
 734         intr = &sc->vmx_intrs[0];
 735         error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq,
 736             INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc,
 737             &intr->vmxi_handler);
 738
 739         for (i = 0; i < sc->vmx_ntxqueues; i++)
 740                 sc->vmx_txq[i].vxtxq_intr_idx = 0;
 741         for (i = 0; i < sc->vmx_nrxqueues; i++)
 742                 sc->vmx_rxq[i].vxrxq_intr_idx = 0;
 743         sc->vmx_event_intr_idx = 0;
 744
 745         return (error);
 746 }
 747
 748 static void
 749 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
 750 {
 751         struct vmxnet3_txqueue *txq;
 752         struct vmxnet3_txq_shared *txs;
 753         struct vmxnet3_rxqueue *rxq;
 754         struct vmxnet3_rxq_shared *rxs;
 755         int i;
 756
 757         sc->vmx_ds->evintr = sc->vmx_event_intr_idx;
 758
 759         for (i = 0; i < sc->vmx_ntxqueues; i++) {
 760                 txq = &sc->vmx_txq[i];
 761                 txs = txq->vxtxq_ts;
 762                 txs->intr_idx = txq->vxtxq_intr_idx;
 763         }
 764
 765         for (i = 0; i < sc->vmx_nrxqueues; i++) {
 766                 rxq = &sc->vmx_rxq[i];
 767                 rxs = rxq->vxrxq_rs;
 768                 rxs->intr_idx = rxq->vxrxq_intr_idx;
 769         }
 770 }
 771
 772 static int
 773 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
 774 {
 775         int error;
 776
 777         error = vmxnet3_alloc_intr_resources(sc);
 778         if (error)
 779                 return (error);
 780
 781         switch (sc->vmx_intr_type) {
 782         case VMXNET3_IT_MSIX:
 783                 error = vmxnet3_setup_msix_interrupts(sc);
 784                 break;
 785         case VMXNET3_IT_MSI:
 786         case VMXNET3_IT_LEGACY:
 787                 error = vmxnet3_setup_legacy_interrupt(sc);
 788                 break;
 789         default:
 790                 panic("%s: invalid interrupt type %d", __func__,
 791                     sc->vmx_intr_type);
 792         }
 793
 794         if (error == 0)
 795                 vmxnet3_set_interrupt_idx(sc);
 796
 797         return (error);
 798 }
 799
 800 static int
 801 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
 802 {
 803         device_t dev;
 804         uint32_t config;
 805         int error;
 806
 807         dev = sc->vmx_dev;
 808         config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
 809
 810         sc->vmx_intr_type = config & 0x03;
 811         sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
 812
 813         switch (sc->vmx_intr_type) {
 814         case VMXNET3_IT_AUTO:
 815                 sc->vmx_intr_type = VMXNET3_IT_MSIX;
 816                 /* FALLTHROUGH */
 817         case VMXNET3_IT_MSIX:
 818                 error = vmxnet3_alloc_msix_interrupts(sc);
 819                 if (error == 0)
 820                         break;
 821                 sc->vmx_intr_type = VMXNET3_IT_MSI;
 822                 /* FALLTHROUGH */
 823         case VMXNET3_IT_MSI:
 824                 error = vmxnet3_alloc_msi_interrupts(sc);
 825                 if (error == 0)
 826                         break;
 827                 sc->vmx_intr_type = VMXNET3_IT_LEGACY;
 828                 /* FALLTHROUGH */
 829         case VMXNET3_IT_LEGACY:
 830                 error = vmxnet3_alloc_legacy_interrupts(sc);
 831                 if (error == 0)
 832                         break;
 833                 /* FALLTHROUGH */
 834         default:
 835                 sc->vmx_intr_type = -1;
 836                 device_printf(dev, "cannot allocate any interrupt resources\n");
 837                 return (ENXIO);
 838         }
 839
 840         return (error);
 841 }
 842
 843 static void
 844 vmxnet3_free_interrupt(struct vmxnet3_softc *sc,
 845     struct vmxnet3_interrupt *intr)
 846 {
 847         device_t dev;
 848
 849         dev = sc->vmx_dev;
 850
 851         if (intr->vmxi_handler != NULL) {
 852                 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler);
 853                 intr->vmxi_handler = NULL;
 854         }
 855
 856         if (intr->vmxi_irq != NULL) {
 857                 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid,
 858                     intr->vmxi_irq);
 859                 intr->vmxi_irq = NULL;
 860                 intr->vmxi_rid = -1;
 861         }
 862 }
 863
 864 static void
 865 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
 866 {
 867         int i;
 868
 869         for (i = 0; i < sc->vmx_nintrs; i++)
 870                 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]);
 871
 872         if (sc->vmx_intr_type == VMXNET3_IT_MSI ||
 873             sc->vmx_intr_type == VMXNET3_IT_MSIX)
 874                 pci_release_msi(sc->vmx_dev);
 875 }
 876
 877 #ifndef VMXNET3_LEGACY_TX
 878 static int
 879 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc)
 880 {
 881         device_t dev;
 882
 883         dev = sc->vmx_dev;
 884
 885         sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT,
 886             taskqueue_thread_enqueue, &sc->vmx_tq);
 887         if (sc->vmx_tq == NULL)
 888                 return (ENOMEM);
 889
 890         return (0);
 891 }
 892
 893 static void
 894 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc)
 895 {
 896         device_t dev;
 897         int nthreads, error;
 898
 899         dev = sc->vmx_dev;
 900
 901         /*
 902          * The taskqueue is typically not frequently used, so a dedicated
 903          * thread for each queue is unnecessary.
 904          */
 905         nthreads = MAX(1, sc->vmx_ntxqueues / 2);
 906
 907         /*
 908          * Most drivers just ignore the return value - it only fails
 909          * with ENOMEM so an error is not likely. It is hard for us
 910          * to recover from an error here.
 911          */
 912         error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET,
 913             "%s taskq", device_get_nameunit(dev));
 914         if (error)
 915                 device_printf(dev, "failed to start taskqueue: %d", error);
 916 }
 917
 918 static void
 919 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc)
 920 {
 921         struct vmxnet3_txqueue *txq;
 922         int i;
 923
 924         if (sc->vmx_tq != NULL) {
 925                 for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
 926                         txq = &sc->vmx_txq[i];
 927                         taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask);
 928                 }
 929         }
 930 }
 931
 932 static void
 933 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc)
 934 {
 935         if (sc->vmx_tq != NULL) {
 936                 taskqueue_free(sc->vmx_tq);
 937                 sc->vmx_tq = NULL;
 938         }
 939 }
 940 #endif
 941
 942 static int
 943 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
 944 {
 945         struct vmxnet3_rxqueue *rxq;
 946         struct vmxnet3_rxring *rxr;
 947         int i;
 948
 949         rxq = &sc->vmx_rxq[q];
 950
 951         snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
 952             device_get_nameunit(sc->vmx_dev), q);
 953         mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF);
 954
 955         rxq->vxrxq_sc = sc;
 956         rxq->vxrxq_id = q;
 957
 958         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
 959                 rxr = &rxq->vxrxq_cmd_ring[i];
 960                 rxr->vxrxr_rid = i;
 961                 rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
 962                 rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc *
 963                     sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO);
 964                 if (rxr->vxrxr_rxbuf == NULL)
 965                         return (ENOMEM);
 966
 967                 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
 968         }
 969
 970         return (0);
 971 }
 972
 973 static int
 974 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
 975 {
 976         struct vmxnet3_txqueue *txq;
 977         struct vmxnet3_txring *txr;
 978
 979         txq = &sc->vmx_txq[q];
 980         txr = &txq->vxtxq_cmd_ring;
 981
 982         snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
 983             device_get_nameunit(sc->vmx_dev), q);
 984         mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF);
 985
 986         txq->vxtxq_sc = sc;
 987         txq->vxtxq_id = q;
 988
 989         txr->vxtxr_ndesc = sc->vmx_ntxdescs;
 990         txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc *
 991             sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO);
 992         if (txr->vxtxr_txbuf == NULL)
 993                 return (ENOMEM);
 994
 995         txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
 996
 997 #ifndef VMXNET3_LEGACY_TX
 998         TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq);
 999
1000         txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF,
1001             M_NOWAIT, &txq->vxtxq_mtx);
1002         if (txq->vxtxq_br == NULL)
1003                 return (ENOMEM);
1004 #endif
1005
1006         return (0);
1007 }
1008
1009 static int
1010 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
1011 {
1012         int i, error;
1013
1014         /*
1015          * Only attempt to create multiple queues if MSIX is available. MSIX is
1016          * disabled by default because its apparently broken for devices passed
1017          * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable
1018          * must be set to zero for MSIX. This check prevents us from allocating
1019          * queue structures that we will not use.
1020          */
1021         if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
1022                 sc->vmx_max_nrxqueues = 1;
1023                 sc->vmx_max_ntxqueues = 1;
1024         }
1025
1026         sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) *
1027             sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
1028         sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) *
1029             sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
1030         if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL)
1031                 return (ENOMEM);
1032
1033         for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
1034                 error = vmxnet3_init_rxq(sc, i);
1035                 if (error)
1036                         return (error);
1037         }
1038
1039         for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
1040                 error = vmxnet3_init_txq(sc, i);
1041                 if (error)
1042                         return (error);
1043         }
1044
1045         return (0);
1046 }
1047
1048 static void
1049 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
1050 {
1051         struct vmxnet3_rxring *rxr;
1052         int i;
1053
1054         rxq->vxrxq_sc = NULL;
1055         rxq->vxrxq_id = -1;
1056
1057         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1058                 rxr = &rxq->vxrxq_cmd_ring[i];
1059
1060                 if (rxr->vxrxr_rxbuf != NULL) {
1061                         free(rxr->vxrxr_rxbuf, M_DEVBUF);
1062                         rxr->vxrxr_rxbuf = NULL;
1063                 }
1064         }
1065
1066         if (mtx_initialized(&rxq->vxrxq_mtx) != 0)
1067                 mtx_destroy(&rxq->vxrxq_mtx);
1068 }
1069
1070 static void
1071 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
1072 {
1073         struct vmxnet3_txring *txr;
1074
1075         txr = &txq->vxtxq_cmd_ring;
1076
1077         txq->vxtxq_sc = NULL;
1078         txq->vxtxq_id = -1;
1079
1080 #ifndef VMXNET3_LEGACY_TX
1081         if (txq->vxtxq_br != NULL) {
1082                 buf_ring_free(txq->vxtxq_br, M_DEVBUF);
1083                 txq->vxtxq_br = NULL;
1084         }
1085 #endif
1086
1087         if (txr->vxtxr_txbuf != NULL) {
1088                 free(txr->vxtxr_txbuf, M_DEVBUF);
1089                 txr->vxtxr_txbuf = NULL;
1090         }
1091
1092         if (mtx_initialized(&txq->vxtxq_mtx) != 0)
1093                 mtx_destroy(&txq->vxtxq_mtx);
1094 }
1095
1096 static void
1097 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
1098 {
1099         int i;
1100
1101         if (sc->vmx_rxq != NULL) {
1102                 for (i = 0; i < sc->vmx_max_nrxqueues; i++)
1103                         vmxnet3_destroy_rxq(&sc->vmx_rxq[i]);
1104                 free(sc->vmx_rxq, M_DEVBUF);
1105                 sc->vmx_rxq = NULL;
1106         }
1107
1108         if (sc->vmx_txq != NULL) {
1109                 for (i = 0; i < sc->vmx_max_ntxqueues; i++)
1110                         vmxnet3_destroy_txq(&sc->vmx_txq[i]);
1111                 free(sc->vmx_txq, M_DEVBUF);
1112                 sc->vmx_txq = NULL;
1113         }
1114 }
1115
1116 static int
1117 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
1118 {
1119         device_t dev;
1120         uint8_t *kva;
1121         size_t size;
1122         int i, error;
1123
1124         dev = sc->vmx_dev;
1125
1126         size = sizeof(struct vmxnet3_driver_shared);
1127         error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
1128         if (error) {
1129                 device_printf(dev, "cannot alloc shared memory\n");
1130                 return (error);
1131         }
1132         sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;
1133
1134         size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
1135             sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
1136         error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
1137         if (error) {
1138                 device_printf(dev, "cannot alloc queue shared memory\n");
1139                 return (error);
1140         }
1141         sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
1142         kva = sc->vmx_qs;
1143
1144         for (i = 0; i < sc->vmx_ntxqueues; i++) {
1145                 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
1146                 kva += sizeof(struct vmxnet3_txq_shared);
1147         }
1148         for (i = 0; i < sc->vmx_nrxqueues; i++) {
1149                 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
1150                 kva += sizeof(struct vmxnet3_rxq_shared);
1151         }
1152
1153         if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1154                 size = sizeof(struct vmxnet3_rss_shared);
1155                 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
1156                 if (error) {
1157                         device_printf(dev, "cannot alloc rss shared memory\n");
1158                         return (error);
1159                 }
1160                 sc->vmx_rss =
1161                     (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
1162         }
1163
1164         return (0);
1165 }
1166
1167 static void
1168 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
1169 {
1170
1171         if (sc->vmx_rss != NULL) {
1172                 vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
1173                 sc->vmx_rss = NULL;
1174         }
1175
1176         if (sc->vmx_qs != NULL) {
1177                 vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
1178                 sc->vmx_qs = NULL;
1179         }
1180
1181         if (sc->vmx_ds != NULL) {
1182                 vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
1183                 sc->vmx_ds = NULL;
1184         }
1185 }
1186
1187 static int
1188 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
1189 {
1190         device_t dev;
1191         struct vmxnet3_txqueue *txq;
1192         struct vmxnet3_txring *txr;
1193         struct vmxnet3_comp_ring *txc;
1194         size_t descsz, compsz;
1195         int i, q, error;
1196
1197         dev = sc->vmx_dev;
1198
1199         for (q = 0; q < sc->vmx_ntxqueues; q++) {
1200                 txq = &sc->vmx_txq[q];
1201                 txr = &txq->vxtxq_cmd_ring;
1202                 txc = &txq->vxtxq_comp_ring;
1203
1204                 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
1205                 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);
1206
1207                 error = bus_dma_tag_create(bus_get_dma_tag(dev),
1208                     1, 0,                       /* alignment, boundary */
1209                     BUS_SPACE_MAXADDR,          /* lowaddr */
1210                     BUS_SPACE_MAXADDR,          /* highaddr */
1211                     NULL, NULL,                 /* filter, filterarg */
1212                     VMXNET3_TX_MAXSIZE,         /* maxsize */
1213                     VMXNET3_TX_MAXSEGS,         /* nsegments */
1214                     VMXNET3_TX_MAXSEGSIZE,      /* maxsegsize */
1215                     0,                          /* flags */
1216                     NULL, NULL,                 /* lockfunc, lockarg */
1217                     &txr->vxtxr_txtag);
1218                 if (error) {
1219                         device_printf(dev,
1220                             "unable to create Tx buffer tag for queue %d\n", q);
1221                         return (error);
1222                 }
1223
1224                 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
1225                 if (error) {
1226                         device_printf(dev, "cannot alloc Tx descriptors for "
1227                             "queue %d error %d\n", q, error);
1228                         return (error);
1229                 }
1230                 txr->vxtxr_txd =
1231                     (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;
1232
1233                 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
1234                 if (error) {
1235                         device_printf(dev, "cannot alloc Tx comp descriptors "
1236                            "for queue %d error %d\n", q, error);
1237                         return (error);
1238                 }
1239                 txc->vxcr_u.txcd =
1240                     (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;
1241
1242                 for (i = 0; i < txr->vxtxr_ndesc; i++) {
1243                         error = bus_dmamap_create(txr->vxtxr_txtag, 0,
1244                             &txr->vxtxr_txbuf[i].vtxb_dmamap);
1245                         if (error) {
1246                                 device_printf(dev, "unable to create Tx buf "
1247                                     "dmamap for queue %d idx %d\n", q, i);
1248                                 return (error);
1249                         }
1250                 }
1251         }
1252
1253         return (0);
1254 }
1255
1256 static void
1257 vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
1258 {
1259         device_t dev;
1260         struct vmxnet3_txqueue *txq;
1261         struct vmxnet3_txring *txr;
1262         struct vmxnet3_comp_ring *txc;
1263         struct vmxnet3_txbuf *txb;
1264         int i, q;
1265
1266         dev = sc->vmx_dev;
1267
1268         for (q = 0; q < sc->vmx_ntxqueues; q++) {
1269                 txq = &sc->vmx_txq[q];
1270                 txr = &txq->vxtxq_cmd_ring;
1271                 txc = &txq->vxtxq_comp_ring;
1272
1273                 for (i = 0; i < txr->vxtxr_ndesc; i++) {
1274                         txb = &txr->vxtxr_txbuf[i];
1275                         if (txb->vtxb_dmamap != NULL) {
1276                                 bus_dmamap_destroy(txr->vxtxr_txtag,
1277                                     txb->vtxb_dmamap);
1278                                 txb->vtxb_dmamap = NULL;
1279                         }
1280                 }
1281
1282                 if (txc->vxcr_u.txcd != NULL) {
1283                         vmxnet3_dma_free(sc, &txc->vxcr_dma);
1284                         txc->vxcr_u.txcd = NULL;
1285                 }
1286
1287                 if (txr->vxtxr_txd != NULL) {
1288                         vmxnet3_dma_free(sc, &txr->vxtxr_dma);
1289                         txr->vxtxr_txd = NULL;
1290                 }
1291
1292                 if (txr->vxtxr_txtag != NULL) {
1293                         bus_dma_tag_destroy(txr->vxtxr_txtag);
1294                         txr->vxtxr_txtag = NULL;
1295                 }
1296         }
1297 }
1298
1299 static int
1300 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
1301 {
1302         device_t dev;
1303         struct vmxnet3_rxqueue *rxq;
1304         struct vmxnet3_rxring *rxr;
1305         struct vmxnet3_comp_ring *rxc;
1306         int descsz, compsz;
1307         int i, j, q, error;
1308
1309         dev = sc->vmx_dev;
1310
1311         for (q = 0; q < sc->vmx_nrxqueues; q++) {
1312                 rxq = &sc->vmx_rxq[q];
1313                 rxc = &rxq->vxrxq_comp_ring;
1314                 compsz = 0;
1315
1316                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1317                         rxr = &rxq->vxrxq_cmd_ring[i];
1318
1319                         descsz = rxr->vxrxr_ndesc *
1320                             sizeof(struct vmxnet3_rxdesc);
1321                         compsz += rxr->vxrxr_ndesc *
1322                             sizeof(struct vmxnet3_rxcompdesc);
1323
1324                         error = bus_dma_tag_create(bus_get_dma_tag(dev),
1325                             1, 0,               /* alignment, boundary */
1326                             BUS_SPACE_MAXADDR,  /* lowaddr */
1327                             BUS_SPACE_MAXADDR,  /* highaddr */
1328                             NULL, NULL,         /* filter, filterarg */
1329                             MJUMPAGESIZE,       /* maxsize */
1330                             1,                  /* nsegments */
1331                             MJUMPAGESIZE,       /* maxsegsize */
1332                             0,                  /* flags */
1333                             NULL, NULL,         /* lockfunc, lockarg */
1334                             &rxr->vxrxr_rxtag);
1335                         if (error) {
1336                                 device_printf(dev,
1337                                     "unable to create Rx buffer tag for "
1338                                     "queue %d\n", q);
1339                                 return (error);
1340                         }
1341
1342                         error = vmxnet3_dma_malloc(sc, descsz, 512,
1343                             &rxr->vxrxr_dma);
1344                         if (error) {
1345                                 device_printf(dev, "cannot allocate Rx "
1346                                     "descriptors for queue %d/%d error %d\n",
1347                                     i, q, error);
1348                                 return (error);
1349                         }
1350                         rxr->vxrxr_rxd =
1351                             (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
1352                 }
1353
1354                 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
1355                 if (error) {
1356                         device_printf(dev, "cannot alloc Rx comp descriptors "
1357                             "for queue %d error %d\n", q, error);
1358                         return (error);
1359                 }
1360                 rxc->vxcr_u.rxcd =
1361                     (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;
1362
1363                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1364                         rxr = &rxq->vxrxq_cmd_ring[i];
1365
1366                         error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1367                             &rxr->vxrxr_spare_dmap);
1368                         if (error) {
1369                                 device_printf(dev, "unable to create spare "
1370                                     "dmamap for queue %d/%d error %d\n",
1371                                     q, i, error);
1372                                 return (error);
1373                         }
1374
1375                         for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1376                                 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1377                                     &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
1378                                 if (error) {
1379                                         device_printf(dev, "unable to create "
1380                                             "dmamap for queue %d/%d slot %d "
1381                                             "error %d\n",
1382                                             q, i, j, error);
1383                                         return (error);
1384                                 }
1385                         }
1386                 }
1387         }
1388
1389         return (0);
1390 }
1391
1392 static void
1393 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
1394 {
1395         device_t dev;
1396         struct vmxnet3_rxqueue *rxq;
1397         struct vmxnet3_rxring *rxr;
1398         struct vmxnet3_comp_ring *rxc;
1399         struct vmxnet3_rxbuf *rxb;
1400         int i, j, q;
1401
1402         dev = sc->vmx_dev;
1403
1404         for (q = 0; q < sc->vmx_nrxqueues; q++) {
1405                 rxq = &sc->vmx_rxq[q];
1406                 rxc = &rxq->vxrxq_comp_ring;
1407
1408                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1409                         rxr = &rxq->vxrxq_cmd_ring[i];
1410
1411                         if (rxr->vxrxr_spare_dmap != NULL) {
1412                                 bus_dmamap_destroy(rxr->vxrxr_rxtag,
1413                                     rxr->vxrxr_spare_dmap);
1414                                 rxr->vxrxr_spare_dmap = NULL;
1415                         }
1416
1417                         for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1418                                 rxb = &rxr->vxrxr_rxbuf[j];
1419                                 if (rxb->vrxb_dmamap != NULL) {
1420                                         bus_dmamap_destroy(rxr->vxrxr_rxtag,
1421                                             rxb->vrxb_dmamap);
1422                                         rxb->vrxb_dmamap = NULL;
1423                                 }
1424                         }
1425                 }
1426
1427                 if (rxc->vxcr_u.rxcd != NULL) {
1428                         vmxnet3_dma_free(sc, &rxc->vxcr_dma);
1429                         rxc->vxcr_u.rxcd = NULL;
1430                 }
1431
1432                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1433                         rxr = &rxq->vxrxq_cmd_ring[i];
1434
1435                         if (rxr->vxrxr_rxd != NULL) {
1436                                 vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
1437                                 rxr->vxrxr_rxd = NULL;
1438                         }
1439
1440                         if (rxr->vxrxr_rxtag != NULL) {
1441                                 bus_dma_tag_destroy(rxr->vxrxr_rxtag);
1442                                 rxr->vxrxr_rxtag = NULL;
1443                         }
1444                 }
1445         }
1446 }
1447
1448 static int
1449 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
1450 {
1451         int error;
1452
1453         error = vmxnet3_alloc_txq_data(sc);
1454         if (error)
1455                 return (error);
1456
1457         error = vmxnet3_alloc_rxq_data(sc);
1458         if (error)
1459                 return (error);
1460
1461         return (0);
1462 }
1463
1464 static void
1465 vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
1466 {
1467
1468         if (sc->vmx_rxq != NULL)
1469                 vmxnet3_free_rxq_data(sc);
1470
1471         if (sc->vmx_txq != NULL)
1472                 vmxnet3_free_txq_data(sc);
1473 }
1474
1475 static int
1476 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1477 {
1478         int error;
1479
1480         error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
1481             32, &sc->vmx_mcast_dma);
1482         if (error)
1483                 device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1484         else
1485                 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;
1486
1487         return (error);
1488 }
1489
1490 static void
1491 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1492 {
1493
1494         if (sc->vmx_mcast != NULL) {
1495                 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
1496                 sc->vmx_mcast = NULL;
1497         }
1498 }
1499
1500 static void
1501 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1502 {
1503         struct vmxnet3_driver_shared *ds;
1504         struct vmxnet3_txqueue *txq;
1505         struct vmxnet3_txq_shared *txs;
1506         struct vmxnet3_rxqueue *rxq;
1507         struct vmxnet3_rxq_shared *rxs;
1508         int i;
1509
1510         ds = sc->vmx_ds;
1511
1512         /*
1513          * Initialize fields of the shared data that remains the same across
1514          * reinits. Note the shared data is zero'd when allocated.
1515          */
1516
1517         ds->magic = VMXNET3_REV1_MAGIC;
1518
1519         /* DriverInfo */
1520         ds->version = VMXNET3_DRIVER_VERSION;
1521         ds->guest = VMXNET3_GOS_FREEBSD |
1522 #ifdef __LP64__
1523             VMXNET3_GOS_64BIT;
1524 #else
1525             VMXNET3_GOS_32BIT;
1526 #endif
1527         ds->vmxnet3_revision = 1;
1528         ds->upt_version = 1;
1529
1530         /* Misc. conf */
1531         ds->driver_data = vtophys(sc);
1532         ds->driver_data_len = sizeof(struct vmxnet3_softc);
1533         ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
1534         ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
1535         ds->nrxsg_max = sc->vmx_max_rxsegs;
1536
1537         /* RSS conf */
1538         if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1539                 ds->rss.version = 1;
1540                 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
1541                 ds->rss.len = sc->vmx_rss_dma.dma_size;
1542         }
1543
1544         /* Interrupt control. */
1545         ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1546         ds->nintr = sc->vmx_nintrs;
1547         ds->evintr = sc->vmx_event_intr_idx;
1548         ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1549
1550         for (i = 0; i < sc->vmx_nintrs; i++)
1551                 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1552
1553         /* Receive filter. */
1554         ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
1555         ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;
1556
1557         /* Tx queues */
1558         for (i = 0; i < sc->vmx_ntxqueues; i++) {
1559                 txq = &sc->vmx_txq[i];
1560                 txs = txq->vxtxq_ts;
1561
1562                 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
1563                 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1564                 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
1565                 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1566                 txs->driver_data = vtophys(txq);
1567                 txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1568         }
1569
1570         /* Rx queues */
1571         for (i = 0; i < sc->vmx_nrxqueues; i++) {
1572                 rxq = &sc->vmx_rxq[i];
1573                 rxs = rxq->vxrxq_rs;
1574
1575                 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
1576                 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1577                 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
1578                 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1579                 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
1580                 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1581                 rxs->driver_data = vtophys(rxq);
1582                 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1583         }
1584 }
1585
1586 static void
1587 vmxnet3_init_hwassist(struct vmxnet3_softc *sc)
1588 {
1589         struct ifnet *ifp = sc->vmx_ifp;
1590         uint64_t hwassist;
1591
1592         hwassist = 0;
1593         if (ifp->if_capenable & IFCAP_TXCSUM)
1594                 hwassist |= VMXNET3_CSUM_OFFLOAD;
1595         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1596                 hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6;
1597         if (ifp->if_capenable & IFCAP_TSO4)
1598                 hwassist |= CSUM_IP_TSO;
1599         if (ifp->if_capenable & IFCAP_TSO6)
1600                 hwassist |= CSUM_IP6_TSO;
1601         ifp->if_hwassist = hwassist;
1602 }
1603
1604 static void
1605 vmxnet3_reinit_interface(struct vmxnet3_softc *sc)
1606 {
1607         struct ifnet *ifp;
1608
1609         ifp = sc->vmx_ifp;
1610
1611         /* Use the current MAC address. */
1612         bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN);
1613         vmxnet3_set_lladdr(sc);
1614
1615         vmxnet3_init_hwassist(sc);
1616 }
1617
1618 static void
1619 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
1620 {
1621         /*
1622          * Use the same key as the Linux driver until FreeBSD can do
1623          * RSS (presumably Toeplitz) in software.
1624          */
1625         static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
1626             0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
1627             0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
1628             0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
1629             0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
1630             0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
1631         };
1632
1633         struct vmxnet3_driver_shared *ds;
1634         struct vmxnet3_rss_shared *rss;
1635         int i;
1636
1637         ds = sc->vmx_ds;
1638         rss = sc->vmx_rss;
1639
1640         rss->hash_type =
1641             UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
1642             UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
1643         rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
1644         rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
1645         rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
1646         memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
1647
1648         for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
1649                 rss->ind_table[i] = i % sc->vmx_nrxqueues;
1650 }
1651
1652 static void
1653 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1654 {
1655         struct ifnet *ifp;
1656         struct vmxnet3_driver_shared *ds;
1657
1658         ifp = sc->vmx_ifp;
1659         ds = sc->vmx_ds;
1660
1661         ds->mtu = ifp->if_mtu;
1662         ds->ntxqueue = sc->vmx_ntxqueues;
1663         ds->nrxqueue = sc->vmx_nrxqueues;
1664
1665         ds->upt_features = 0;
1666         if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
1667                 ds->upt_features |= UPT1_F_CSUM;
1668         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1669                 ds->upt_features |= UPT1_F_VLAN;
1670         if (ifp->if_capenable & IFCAP_LRO)
1671                 ds->upt_features |= UPT1_F_LRO;
1672
1673         if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1674                 ds->upt_features |= UPT1_F_RSS;
1675                 vmxnet3_reinit_rss_shared_data(sc);
1676         }
1677
1678         vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
1679         vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1680             (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
1681 }
1682
1683 static int
1684 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1685 {
1686         int error;
1687
1688         error = vmxnet3_alloc_shared_data(sc);
1689         if (error)
1690                 return (error);
1691
1692         error = vmxnet3_alloc_queue_data(sc);
1693         if (error)
1694                 return (error);
1695
1696         error = vmxnet3_alloc_mcast_table(sc);
1697         if (error)
1698                 return (error);
1699
1700         vmxnet3_init_shared_data(sc);
1701
1702         return (0);
1703 }
1704
1705 static void
1706 vmxnet3_free_data(struct vmxnet3_softc *sc)
1707 {
1708
1709         vmxnet3_free_mcast_table(sc);
1710         vmxnet3_free_queue_data(sc);
1711         vmxnet3_free_shared_data(sc);
1712 }
1713
1714 static int
1715 vmxnet3_setup_interface(struct vmxnet3_softc *sc)
1716 {
1717         device_t dev;
1718         struct ifnet *ifp;
1719
1720         dev = sc->vmx_dev;
1721
1722         ifp = sc->vmx_ifp = if_alloc(IFT_ETHER);
1723         if (ifp == NULL) {
1724                 device_printf(dev, "cannot allocate ifnet structure\n");
1725                 return (ENOSPC);
1726         }
1727
1728         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1729 #if __FreeBSD_version < 1000025
1730         ifp->if_baudrate = 1000000000;
1731 #elif __FreeBSD_version < 1100011
1732         if_initbaudrate(ifp, IF_Gbps(10));
1733 #else
1734         ifp->if_baudrate = IF_Gbps(10);
1735 #endif
1736         ifp->if_softc = sc;
1737         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1738         ifp->if_init = vmxnet3_init;
1739         ifp->if_ioctl = vmxnet3_ioctl;
1740         ifp->if_get_counter = vmxnet3_get_counter;
1741         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1742         ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS;
1743         ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE;
1744
1745 #ifdef VMXNET3_LEGACY_TX
1746         ifp->if_start = vmxnet3_start;
1747         ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1;
1748         IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1);
1749         IFQ_SET_READY(&ifp->if_snd);
1750 #else
1751         ifp->if_transmit = vmxnet3_txq_mq_start;
1752         ifp->if_qflush = vmxnet3_qflush;
1753 #endif
1754
1755         vmxnet3_get_lladdr(sc);
1756         ether_ifattach(ifp, sc->vmx_lladdr);
1757
1758         ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM;
1759         ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6;
1760         ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
1761         ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
1762             IFCAP_VLAN_HWCSUM;
1763         ifp->if_capenable = ifp->if_capabilities;
1764
1765         /* These capabilities are not enabled by default. */
1766         ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER;
1767
1768         sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
1769             vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
1770         sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config,
1771             vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
1772
1773         ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change,
1774             vmxnet3_media_status);
1775         ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1776         ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO);
1777
1778         return (0);
1779 }
1780
1781 static void
1782 vmxnet3_evintr(struct vmxnet3_softc *sc)
1783 {
1784         device_t dev;
1785         struct ifnet *ifp;
1786         struct vmxnet3_txq_shared *ts;
1787         struct vmxnet3_rxq_shared *rs;
1788         uint32_t event;
1789         int reset;
1790
1791         dev = sc->vmx_dev;
1792         ifp = sc->vmx_ifp;
1793         reset = 0;
1794
1795         VMXNET3_CORE_LOCK(sc);
1796
1797         /* Clear events. */
1798         event = sc->vmx_ds->event;
1799         vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
1800
1801         if (event & VMXNET3_EVENT_LINK) {
1802                 vmxnet3_link_status(sc);
1803                 if (sc->vmx_link_active != 0)
1804                         vmxnet3_tx_start_all(sc);
1805         }
1806
1807         if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
1808                 reset = 1;
1809                 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
1810                 ts = sc->vmx_txq[0].vxtxq_ts;
1811                 if (ts->stopped != 0)
1812                         device_printf(dev, "Tx queue error %#x\n", ts->error);
1813                 rs = sc->vmx_rxq[0].vxrxq_rs;
1814                 if (rs->stopped != 0)
1815                         device_printf(dev, "Rx queue error %#x\n", rs->error);
1816                 device_printf(dev, "Rx/Tx queue error event ... resetting\n");
1817         }
1818
1819         if (event & VMXNET3_EVENT_DIC)
1820                 device_printf(dev, "device implementation change event\n");
1821         if (event & VMXNET3_EVENT_DEBUG)
1822                 device_printf(dev, "debug event\n");
1823
1824         if (reset != 0) {
1825                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1826                 vmxnet3_init_locked(sc);
1827         }
1828
1829         VMXNET3_CORE_UNLOCK(sc);
1830 }
1831
1832 static void
1833 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq)
1834 {
1835         struct vmxnet3_softc *sc;
1836         struct ifnet *ifp;
1837         struct vmxnet3_txring *txr;
1838         struct vmxnet3_comp_ring *txc;
1839         struct vmxnet3_txcompdesc *txcd;
1840         struct vmxnet3_txbuf *txb;
1841         struct mbuf *m;
1842         u_int sop;
1843
1844         sc = txq->vxtxq_sc;
1845         ifp = sc->vmx_ifp;
1846         txr = &txq->vxtxq_cmd_ring;
1847         txc = &txq->vxtxq_comp_ring;
1848
1849         VMXNET3_TXQ_LOCK_ASSERT(txq);
1850
1851         for (;;) {
1852                 txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
1853                 if (txcd->gen != txc->vxcr_gen)
1854                         break;
1855                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1856
1857                 if (++txc->vxcr_next == txc->vxcr_ndesc) {
1858                         txc->vxcr_next = 0;
1859                         txc->vxcr_gen ^= 1;
1860                 }
1861
1862                 sop = txr->vxtxr_next;
1863                 txb = &txr->vxtxr_txbuf[sop];
1864
1865                 if ((m = txb->vtxb_m) != NULL) {
1866                         bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
1867                             BUS_DMASYNC_POSTWRITE);
1868                         bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
1869
1870                         txq->vxtxq_stats.vmtxs_opackets++;
1871                         txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len;
1872                         if (m->m_flags & M_MCAST)
1873                                 txq->vxtxq_stats.vmtxs_omcasts++;
1874
1875                         m_freem(m);
1876                         txb->vtxb_m = NULL;
1877                 }
1878
1879                 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
1880         }
1881
1882         if (txr->vxtxr_head == txr->vxtxr_next)
1883                 txq->vxtxq_watchdog = 0;
1884 }
1885
1886 static int
1887 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr)
1888 {
1889         struct ifnet *ifp;
1890         struct mbuf *m;
1891         struct vmxnet3_rxdesc *rxd;
1892         struct vmxnet3_rxbuf *rxb;
1893         bus_dma_tag_t tag;
1894         bus_dmamap_t dmap;
1895         bus_dma_segment_t segs[1];
1896         int idx, clsize, btype, flags, nsegs, error;
1897
1898         ifp = sc->vmx_ifp;
1899         tag = rxr->vxrxr_rxtag;
1900         dmap = rxr->vxrxr_spare_dmap;
1901         idx = rxr->vxrxr_fill;
1902         rxd = &rxr->vxrxr_rxd[idx];
1903         rxb = &rxr->vxrxr_rxbuf[idx];
1904
1905 #ifdef VMXNET3_FAILPOINTS
1906         KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS);
1907         if (rxr->vxrxr_rid != 0)
1908                 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS);
1909 #endif
1910
1911         if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) {
1912                 flags = M_PKTHDR;
1913                 clsize = MCLBYTES;
1914                 btype = VMXNET3_BTYPE_HEAD;
1915         } else {
1916 #if __FreeBSD_version < 902001
1917                 /*
1918                  * These mbufs will never be used for the start of a frame.
1919                  * Roughly prior to branching releng/9.2, the load_mbuf_sg()
1920                  * required the mbuf to always be a packet header. Avoid
1921                  * unnecessary mbuf initialization in newer versions where
1922                  * that is not the case.
1923                  */
1924                 flags = M_PKTHDR;
1925 #else
1926                 flags = 0;
1927 #endif
1928                 clsize = MJUMPAGESIZE;
1929                 btype = VMXNET3_BTYPE_BODY;
1930         }
1931
1932         m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize);
1933         if (m == NULL) {
1934                 sc->vmx_stats.vmst_mgetcl_failed++;
1935                 return (ENOBUFS);
1936         }
1937
1938         if (btype == VMXNET3_BTYPE_HEAD) {
1939                 m->m_len = m->m_pkthdr.len = clsize;
1940                 m_adj(m, ETHER_ALIGN);
1941         } else
1942                 m->m_len = clsize;
1943
1944         error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs,
1945             BUS_DMA_NOWAIT);
1946         if (error) {
1947                 m_freem(m);
1948                 sc->vmx_stats.vmst_mbuf_load_failed++;
1949                 return (error);
1950         }
1951         KASSERT(nsegs == 1,
1952             ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
1953 #if __FreeBSD_version < 902001
1954         if (btype == VMXNET3_BTYPE_BODY)
1955                 m->m_flags &= ~M_PKTHDR;
1956 #endif
1957
1958         if (rxb->vrxb_m != NULL) {
1959                 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD);
1960                 bus_dmamap_unload(tag, rxb->vrxb_dmamap);
1961         }
1962
1963         rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
1964         rxb->vrxb_dmamap = dmap;
1965         rxb->vrxb_m = m;
1966
1967         rxd->addr = segs[0].ds_addr;
1968         rxd->len = segs[0].ds_len;
1969         rxd->btype = btype;
1970         rxd->gen = rxr->vxrxr_gen;
1971
1972         vmxnet3_rxr_increment_fill(rxr);
1973         return (0);
1974 }
1975
1976 static void
1977 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
1978     struct vmxnet3_rxring *rxr, int idx)
1979 {
1980         struct vmxnet3_rxdesc *rxd;
1981
1982         rxd = &rxr->vxrxr_rxd[idx];
1983         rxd->gen = rxr->vxrxr_gen;
1984         vmxnet3_rxr_increment_fill(rxr);
1985 }
1986
1987 static void
1988 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
1989 {
1990         struct vmxnet3_softc *sc;
1991         struct vmxnet3_rxring *rxr;
1992         struct vmxnet3_comp_ring *rxc;
1993         struct vmxnet3_rxcompdesc *rxcd;
1994         int idx, eof;
1995
1996         sc = rxq->vxrxq_sc;
1997         rxc = &rxq->vxrxq_comp_ring;
1998
1999         do {
2000                 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2001                 if (rxcd->gen != rxc->vxcr_gen)
2002                         break;          /* Not expected. */
2003                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2004
2005                 if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2006                         rxc->vxcr_next = 0;
2007                         rxc->vxcr_gen ^= 1;
2008                 }
2009
2010                 idx = rxcd->rxd_idx;
2011                 eof = rxcd->eop;
2012                 if (rxcd->qid < sc->vmx_nrxqueues)
2013                         rxr = &rxq->vxrxq_cmd_ring[0];
2014                 else
2015                         rxr = &rxq->vxrxq_cmd_ring[1];
2016                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2017         } while (!eof);
2018 }
2019
2020 static void
2021 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2022 {
2023
2024         if (rxcd->ipv4) {
2025                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2026                 if (rxcd->ipcsum_ok)
2027                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2028         }
2029
2030         if (!rxcd->fragment) {
2031                 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) {
2032                         m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2033                             CSUM_PSEUDO_HDR;
2034                         m->m_pkthdr.csum_data = 0xFFFF;
2035                 }
2036         }
2037 }
2038
2039 static void
2040 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
2041     struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2042 {
2043         struct vmxnet3_softc *sc;
2044         struct ifnet *ifp;
2045
2046         sc = rxq->vxrxq_sc;
2047         ifp = sc->vmx_ifp;
2048
2049         if (rxcd->error) {
2050                 rxq->vxrxq_stats.vmrxs_ierrors++;
2051                 m_freem(m);
2052                 return;
2053         }
2054
2055 #ifdef notyet
2056         switch (rxcd->rss_type) {
2057         case VMXNET3_RCD_RSS_TYPE_IPV4:
2058                 m->m_pkthdr.flowid = rxcd->rss_hash;
2059                 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4);
2060                 break;
2061         case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
2062                 m->m_pkthdr.flowid = rxcd->rss_hash;
2063                 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4);
2064                 break;
2065         case VMXNET3_RCD_RSS_TYPE_IPV6:
2066                 m->m_pkthdr.flowid = rxcd->rss_hash;
2067                 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6);
2068                 break;
2069         case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
2070                 m->m_pkthdr.flowid = rxcd->rss_hash;
2071                 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6);
2072                 break;
2073         default: /* VMXNET3_RCD_RSS_TYPE_NONE */
2074                 m->m_pkthdr.flowid = rxq->vxrxq_id;
2075                 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2076                 break;
2077         }
2078 #else
2079         m->m_pkthdr.flowid = rxq->vxrxq_id;
2080         M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2081 #endif
2082
2083         if (!rxcd->no_csum)
2084                 vmxnet3_rx_csum(rxcd, m);
2085         if (rxcd->vlan) {
2086                 m->m_flags |= M_VLANTAG;
2087                 m->m_pkthdr.ether_vtag = rxcd->vtag;
2088         }
2089
2090         rxq->vxrxq_stats.vmrxs_ipackets++;
2091         rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len;
2092
2093         VMXNET3_RXQ_UNLOCK(rxq);
2094         (*ifp->if_input)(ifp, m);
2095         VMXNET3_RXQ_LOCK(rxq);
2096 }
2097
2098 static void
2099 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq)
2100 {
2101         struct vmxnet3_softc *sc;
2102         struct ifnet *ifp;
2103         struct vmxnet3_rxring *rxr;
2104         struct vmxnet3_comp_ring *rxc;
2105         struct vmxnet3_rxdesc *rxd;
2106         struct vmxnet3_rxcompdesc *rxcd;
2107         struct mbuf *m, *m_head, *m_tail;
2108         int idx, length;
2109
2110         sc = rxq->vxrxq_sc;
2111         ifp = sc->vmx_ifp;
2112         rxc = &rxq->vxrxq_comp_ring;
2113
2114         VMXNET3_RXQ_LOCK_ASSERT(rxq);
2115
2116         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2117                 return;
2118
2119         m_head = rxq->vxrxq_mhead;
2120         rxq->vxrxq_mhead = NULL;
2121         m_tail = rxq->vxrxq_mtail;
2122         rxq->vxrxq_mtail = NULL;
2123         MPASS(m_head == NULL || m_tail != NULL);
2124
2125         for (;;) {
2126                 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2127                 if (rxcd->gen != rxc->vxcr_gen) {
2128                         rxq->vxrxq_mhead = m_head;
2129                         rxq->vxrxq_mtail = m_tail;
2130                         break;
2131                 }
2132                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2133
2134                 if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2135                         rxc->vxcr_next = 0;
2136                         rxc->vxcr_gen ^= 1;
2137                 }
2138
2139                 idx = rxcd->rxd_idx;
2140                 length = rxcd->len;
2141                 if (rxcd->qid < sc->vmx_nrxqueues)
2142                         rxr = &rxq->vxrxq_cmd_ring[0];
2143                 else
2144                         rxr = &rxq->vxrxq_cmd_ring[1];
2145                 rxd = &rxr->vxrxr_rxd[idx];
2146
2147                 m = rxr->vxrxr_rxbuf[idx].vrxb_m;
2148                 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf",
2149                     __func__, rxcd->qid, idx));
2150
2151                 /*
2152                  * The host may skip descriptors. We detect this when this
2153                  * descriptor does not match the previous fill index. Catch
2154                  * up with the host now.
2155                  */
2156                 if (__predict_false(rxr->vxrxr_fill != idx)) {
2157                         while (rxr->vxrxr_fill != idx) {
2158                                 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
2159                                     rxr->vxrxr_gen;
2160                                 vmxnet3_rxr_increment_fill(rxr);
2161                         }
2162                 }
2163
2164                 if (rxcd->sop) {
2165                         KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD,
2166                             ("%s: start of frame w/o head buffer", __func__));
2167                         KASSERT(rxr == &rxq->vxrxq_cmd_ring[0],
2168                             ("%s: start of frame not in ring 0", __func__));
2169                         KASSERT((idx % sc->vmx_rx_max_chain) == 0,
2170                             ("%s: start of frame at unexcepted index %d (%d)",
2171                              __func__, idx, sc->vmx_rx_max_chain));
2172                         KASSERT(m_head == NULL,
2173                             ("%s: duplicate start of frame?", __func__));
2174
2175                         if (length == 0) {
2176                                 /* Just ignore this descriptor. */
2177                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2178                                 goto nextp;
2179                         }
2180
2181                         if (vmxnet3_newbuf(sc, rxr) != 0) {
2182                                 rxq->vxrxq_stats.vmrxs_iqdrops++;
2183                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2184                                 if (!rxcd->eop)
2185                                         vmxnet3_rxq_discard_chain(rxq);
2186                                 goto nextp;
2187                         }
2188
2189                         m->m_pkthdr.rcvif = ifp;
2190                         m->m_pkthdr.len = m->m_len = length;
2191                         m->m_pkthdr.csum_flags = 0;
2192                         m_head = m_tail = m;
2193
2194                 } else {
2195                         KASSERT(rxd->btype == VMXNET3_BTYPE_BODY,
2196                             ("%s: non start of frame w/o body buffer", __func__));
2197
2198                         if (m_head == NULL && m_tail == NULL) {
2199                                 /*
2200                                  * This is a continuation of a packet that we
2201                                  * started to drop, but could not drop entirely
2202                                  * because this segment was still owned by the
2203                                  * host.  So, drop the remainder now.
2204                                  */
2205                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2206                                 if (!rxcd->eop)
2207                                         vmxnet3_rxq_discard_chain(rxq);
2208                                 goto nextp;
2209                         }
2210
2211                         KASSERT(m_head != NULL,
2212                             ("%s: frame not started?", __func__));
2213
2214                         if (vmxnet3_newbuf(sc, rxr) != 0) {
2215                                 rxq->vxrxq_stats.vmrxs_iqdrops++;
2216                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2217                                 if (!rxcd->eop)
2218                                         vmxnet3_rxq_discard_chain(rxq);
2219                                 m_freem(m_head);
2220                                 m_head = m_tail = NULL;
2221                                 goto nextp;
2222                         }
2223
2224                         m->m_len = length;
2225                         m_head->m_pkthdr.len += length;
2226                         m_tail->m_next = m;
2227                         m_tail = m;
2228                 }
2229
2230                 if (rxcd->eop) {
2231                         vmxnet3_rxq_input(rxq, rxcd, m_head);
2232                         m_head = m_tail = NULL;
2233
2234                         /* Must recheck after dropping the Rx lock. */
2235                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2236                                 break;
2237                 }
2238
2239 nextp:
2240                 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
2241                         int qid = rxcd->qid;
2242                         bus_size_t r;
2243
2244                         idx = (idx + 1) % rxr->vxrxr_ndesc;
2245                         if (qid >= sc->vmx_nrxqueues) {
2246                                 qid -= sc->vmx_nrxqueues;
2247                                 r = VMXNET3_BAR0_RXH2(qid);
2248                         } else
2249                                 r = VMXNET3_BAR0_RXH1(qid);
2250                         vmxnet3_write_bar0(sc, r, idx);
2251                 }
2252         }
2253 }
2254
2255 static void
2256 vmxnet3_legacy_intr(void *xsc)
2257 {
2258         struct vmxnet3_softc *sc;
2259         struct vmxnet3_rxqueue *rxq;
2260         struct vmxnet3_txqueue *txq;
2261
2262         sc = xsc;
2263         rxq = &sc->vmx_rxq[0];
2264         txq = &sc->vmx_txq[0];
2265
2266         if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
2267                 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
2268                         return;
2269         }
2270         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2271                 vmxnet3_disable_all_intrs(sc);
2272
2273         if (sc->vmx_ds->event != 0)
2274                 vmxnet3_evintr(sc);
2275
2276         VMXNET3_RXQ_LOCK(rxq);
2277         vmxnet3_rxq_eof(rxq);
2278         VMXNET3_RXQ_UNLOCK(rxq);
2279
2280         VMXNET3_TXQ_LOCK(txq);
2281         vmxnet3_txq_eof(txq);
2282         vmxnet3_txq_start(txq);
2283         VMXNET3_TXQ_UNLOCK(txq);
2284
2285         vmxnet3_enable_all_intrs(sc);
2286 }
2287
2288 static void
2289 vmxnet3_txq_intr(void *xtxq)
2290 {
2291         struct vmxnet3_softc *sc;
2292         struct vmxnet3_txqueue *txq;
2293
2294         txq = xtxq;
2295         sc = txq->vxtxq_sc;
2296
2297         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2298                 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx);
2299
2300         VMXNET3_TXQ_LOCK(txq);
2301         vmxnet3_txq_eof(txq);
2302         vmxnet3_txq_start(txq);
2303         VMXNET3_TXQ_UNLOCK(txq);
2304
2305         vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx);
2306 }
2307
2308 static void
2309 vmxnet3_rxq_intr(void *xrxq)
2310 {
2311         struct vmxnet3_softc *sc;
2312         struct vmxnet3_rxqueue *rxq;
2313
2314         rxq = xrxq;
2315         sc = rxq->vxrxq_sc;
2316
2317         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2318                 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
2319
2320         VMXNET3_RXQ_LOCK(rxq);
2321         vmxnet3_rxq_eof(rxq);
2322         VMXNET3_RXQ_UNLOCK(rxq);
2323
2324         vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx);
2325 }
2326
2327 static void
2328 vmxnet3_event_intr(void *xsc)
2329 {
2330         struct vmxnet3_softc *sc;
2331
2332         sc = xsc;
2333
2334         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2335                 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
2336
2337         if (sc->vmx_ds->event != 0)
2338                 vmxnet3_evintr(sc);
2339
2340         vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2341 }
2342
2343 static void
2344 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2345 {
2346         struct vmxnet3_txring *txr;
2347         struct vmxnet3_txbuf *txb;
2348         int i;
2349
2350         txr = &txq->vxtxq_cmd_ring;
2351
2352         for (i = 0; i < txr->vxtxr_ndesc; i++) {
2353                 txb = &txr->vxtxr_txbuf[i];
2354
2355                 if (txb->vtxb_m == NULL)
2356                         continue;
2357
2358                 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
2359                     BUS_DMASYNC_POSTWRITE);
2360                 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
2361                 m_freem(txb->vtxb_m);
2362                 txb->vtxb_m = NULL;
2363         }
2364 }
2365
2366 static void
2367 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2368 {
2369         struct vmxnet3_rxring *rxr;
2370         struct vmxnet3_rxbuf *rxb;
2371         int i, j;
2372
2373         if (rxq->vxrxq_mhead != NULL) {
2374                 m_freem(rxq->vxrxq_mhead);
2375                 rxq->vxrxq_mhead = NULL;
2376                 rxq->vxrxq_mtail = NULL;
2377         }
2378
2379         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
2380                 rxr = &rxq->vxrxq_cmd_ring[i];
2381
2382                 for (j = 0; j < rxr->vxrxr_ndesc; j++) {
2383                         rxb = &rxr->vxrxr_rxbuf[j];
2384
2385                         if (rxb->vrxb_m == NULL)
2386                                 continue;
2387
2388                         bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap,
2389                             BUS_DMASYNC_POSTREAD);
2390                         bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap);
2391                         m_freem(rxb->vrxb_m);
2392                         rxb->vrxb_m = NULL;
2393                 }
2394         }
2395 }
2396
2397 static void
2398 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
2399 {
2400         struct vmxnet3_rxqueue *rxq;
2401         struct vmxnet3_txqueue *txq;
2402         int i;
2403
2404         for (i = 0; i < sc->vmx_nrxqueues; i++) {
2405                 rxq = &sc->vmx_rxq[i];
2406                 VMXNET3_RXQ_LOCK(rxq);
2407                 VMXNET3_RXQ_UNLOCK(rxq);
2408         }
2409
2410         for (i = 0; i < sc->vmx_ntxqueues; i++) {
2411                 txq = &sc->vmx_txq[i];
2412                 VMXNET3_TXQ_LOCK(txq);
2413                 VMXNET3_TXQ_UNLOCK(txq);
2414         }
2415 }
2416
2417 static void
2418 vmxnet3_stop(struct vmxnet3_softc *sc)
2419 {
2420         struct ifnet *ifp;
2421         int q;
2422
2423         ifp = sc->vmx_ifp;
2424         VMXNET3_CORE_LOCK_ASSERT(sc);
2425
2426         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2427         sc->vmx_link_active = 0;
2428         callout_stop(&sc->vmx_tick);
2429
2430         /* Disable interrupts. */
2431         vmxnet3_disable_all_intrs(sc);
2432         vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
2433
2434         vmxnet3_stop_rendezvous(sc);
2435
2436         for (q = 0; q < sc->vmx_ntxqueues; q++)
2437                 vmxnet3_txstop(sc, &sc->vmx_txq[q]);
2438         for (q = 0; q < sc->vmx_nrxqueues; q++)
2439                 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]);
2440
2441         vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
2442 }
2443
2444 static void
2445 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2446 {
2447         struct vmxnet3_txring *txr;
2448         struct vmxnet3_comp_ring *txc;
2449
2450         txr = &txq->vxtxq_cmd_ring;
2451         txr->vxtxr_head = 0;
2452         txr->vxtxr_next = 0;
2453         txr->vxtxr_gen = VMXNET3_INIT_GEN;
2454         bzero(txr->vxtxr_txd,
2455             txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));
2456
2457         txc = &txq->vxtxq_comp_ring;
2458         txc->vxcr_next = 0;
2459         txc->vxcr_gen = VMXNET3_INIT_GEN;
2460         bzero(txc->vxcr_u.txcd,
2461             txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
2462 }
2463
2464 static int
2465 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2466 {
2467         struct ifnet *ifp;
2468         struct vmxnet3_rxring *rxr;
2469         struct vmxnet3_comp_ring *rxc;
2470         int i, populate, idx, frame_size, error;
2471
2472         ifp = sc->vmx_ifp;
2473         frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) +
2474             ifp->if_mtu;
2475
2476         /*
2477          * If the MTU causes us to exceed what a regular sized cluster can
2478          * handle, we allocate a second MJUMPAGESIZE cluster after it in
2479          * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters.
2480          *
2481          * Keep rx_max_chain a divisor of the maximum Rx ring size to make
2482          * our life easier. We do not support changing the ring size after
2483          * the attach.
2484          */
2485         if (frame_size <= MCLBYTES)
2486                 sc->vmx_rx_max_chain = 1;
2487         else
2488                 sc->vmx_rx_max_chain = 2;
2489
2490         /*
2491          * Only populate ring 1 if the configuration will take advantage
2492          * of it. That is either when LRO is enabled or the frame size
2493          * exceeds what ring 0 can contain.
2494          */
2495         if ((ifp->if_capenable & IFCAP_LRO) == 0 &&
2496             frame_size <= MCLBYTES + MJUMPAGESIZE)
2497                 populate = 1;
2498         else
2499                 populate = VMXNET3_RXRINGS_PERQ;
2500
2501         for (i = 0; i < populate; i++) {
2502                 rxr = &rxq->vxrxq_cmd_ring[i];
2503                 rxr->vxrxr_fill = 0;
2504                 rxr->vxrxr_gen = VMXNET3_INIT_GEN;
2505                 bzero(rxr->vxrxr_rxd,
2506                     rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2507
2508                 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
2509                         error = vmxnet3_newbuf(sc, rxr);
2510                         if (error)
2511                                 return (error);
2512                 }
2513         }
2514
2515         for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
2516                 rxr = &rxq->vxrxq_cmd_ring[i];
2517                 rxr->vxrxr_fill = 0;
2518                 rxr->vxrxr_gen = 0;
2519                 bzero(rxr->vxrxr_rxd,
2520                     rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2521         }
2522
2523         rxc = &rxq->vxrxq_comp_ring;
2524         rxc->vxcr_next = 0;
2525         rxc->vxcr_gen = VMXNET3_INIT_GEN;
2526         bzero(rxc->vxcr_u.rxcd,
2527             rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));
2528
2529         return (0);
2530 }
2531
2532 static int
2533 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
2534 {
2535         device_t dev;
2536         int q, error;
2537
2538         dev = sc->vmx_dev;
2539
2540         for (q = 0; q < sc->vmx_ntxqueues; q++)
2541                 vmxnet3_txinit(sc, &sc->vmx_txq[q]);
2542
2543         for (q = 0; q < sc->vmx_nrxqueues; q++) {
2544                 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
2545                 if (error) {
2546                         device_printf(dev, "cannot populate Rx queue %d\n", q);
2547                         return (error);
2548                 }
2549         }
2550
2551         return (0);
2552 }
2553
2554 static int
2555 vmxnet3_enable_device(struct vmxnet3_softc *sc)
2556 {
2557         int q;
2558
2559         if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
2560                 device_printf(sc->vmx_dev, "device enable command failed!\n");
2561                 return (1);
2562         }
2563
2564         /* Reset the Rx queue heads. */
2565         for (q = 0; q < sc->vmx_nrxqueues; q++) {
2566                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
2567                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
2568         }
2569
2570         return (0);
2571 }
2572
2573 static void
2574 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
2575 {
2576         struct ifnet *ifp;
2577
2578         ifp = sc->vmx_ifp;
2579
2580         vmxnet3_set_rxfilter(sc);
2581
2582         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2583                 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter,
2584                     sizeof(sc->vmx_ds->vlan_filter));
2585         else
2586                 bzero(sc->vmx_ds->vlan_filter,
2587                     sizeof(sc->vmx_ds->vlan_filter));
2588         vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
2589 }
2590
2591 static int
2592 vmxnet3_reinit(struct vmxnet3_softc *sc)
2593 {
2594
2595         vmxnet3_reinit_interface(sc);
2596         vmxnet3_reinit_shared_data(sc);
2597
2598         if (vmxnet3_reinit_queues(sc) != 0)
2599                 return (ENXIO);
2600
2601         if (vmxnet3_enable_device(sc) != 0)
2602                 return (ENXIO);
2603
2604         vmxnet3_reinit_rxfilters(sc);
2605
2606         return (0);
2607 }
2608
2609 static void
2610 vmxnet3_init_locked(struct vmxnet3_softc *sc)
2611 {
2612         struct ifnet *ifp;
2613
2614         ifp = sc->vmx_ifp;
2615
2616         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2617                 return;
2618
2619         vmxnet3_stop(sc);
2620
2621         if (vmxnet3_reinit(sc) != 0) {
2622                 vmxnet3_stop(sc);
2623                 return;
2624         }
2625
2626         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2627         vmxnet3_link_status(sc);
2628
2629         vmxnet3_enable_all_intrs(sc);
2630         callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
2631 }
2632
2633 static void
2634 vmxnet3_init(void *xsc)
2635 {
2636         struct vmxnet3_softc *sc;
2637
2638         sc = xsc;
2639
2640         VMXNET3_CORE_LOCK(sc);
2641         vmxnet3_init_locked(sc);
2642         VMXNET3_CORE_UNLOCK(sc);
2643 }
2644
2645 /*
2646  * BMV: Much of this can go away once we finally have offsets in
2647  * the mbuf packet header. Bug andre@.
2648  */
2649 static int
2650 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
2651     int *etype, int *proto, int *start)
2652 {
2653         struct ether_vlan_header *evh;
2654         int offset;
2655 #if defined(INET)
2656         struct ip *ip = NULL;
2657         struct ip iphdr;
2658 #endif
2659 #if defined(INET6)
2660         struct ip6_hdr *ip6 = NULL;
2661         struct ip6_hdr ip6hdr;
2662 #endif
2663
2664         evh = mtod(m, struct ether_vlan_header *);
2665         if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2666                 /* BMV: We should handle nested VLAN tags too. */
2667                 *etype = ntohs(evh->evl_proto);
2668                 offset = sizeof(struct ether_vlan_header);
2669         } else {
2670                 *etype = ntohs(evh->evl_encap_proto);
2671                 offset = sizeof(struct ether_header);
2672         }
2673
2674         switch (*etype) {
2675 #if defined(INET)
2676         case ETHERTYPE_IP:
2677                 if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
2678                         m_copydata(m, offset, sizeof(struct ip),
2679                             (caddr_t) &iphdr);
2680                         ip = &iphdr;
2681                 } else
2682                         ip = mtodo(m, offset);
2683                 *proto = ip->ip_p;
2684                 *start = offset + (ip->ip_hl << 2);
2685                 break;
2686 #endif
2687 #if defined(INET6)
2688         case ETHERTYPE_IPV6:
2689                 if (__predict_false(m->m_len <
2690                     offset + sizeof(struct ip6_hdr))) {
2691                         m_copydata(m, offset, sizeof(struct ip6_hdr),
2692                             (caddr_t) &ip6hdr);
2693                         ip6 = &ip6hdr;
2694                 } else
2695                         ip6 = mtodo(m, offset);
2696                 *proto = -1;
2697                 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
2698                 /* Assert the network stack sent us a valid packet. */
2699                 KASSERT(*start > offset,
2700                     ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
2701                     *start, offset, *proto));
2702                 break;
2703 #endif
2704         default:
2705                 return (EINVAL);
2706         }
2707
2708         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2709                 struct tcphdr *tcp, tcphdr;
2710                 uint16_t sum;
2711
2712                 if (__predict_false(*proto != IPPROTO_TCP)) {
2713                         /* Likely failed to correctly parse the mbuf. */
2714                         return (EINVAL);
2715                 }
2716
2717                 txq->vxtxq_stats.vmtxs_tso++;
2718
2719                 switch (*etype) {
2720 #if defined(INET)
2721                 case ETHERTYPE_IP:
2722                         sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2723                             htons(IPPROTO_TCP));
2724                         break;
2725 #endif
2726 #if defined(INET6)
2727                 case ETHERTYPE_IPV6:
2728                         sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
2729                         break;
2730 #endif
2731                 default:
2732                         sum = 0;
2733                         break;
2734                 }
2735
2736                 if (m->m_len < *start + sizeof(struct tcphdr)) {
2737                         m_copyback(m, *start + offsetof(struct tcphdr, th_sum),
2738                             sizeof(uint16_t), (caddr_t) &sum);
2739                         m_copydata(m, *start, sizeof(struct tcphdr),
2740                             (caddr_t) &tcphdr);
2741                         tcp = &tcphdr;
2742                 } else {
2743                         tcp = mtodo(m, *start);
2744                         tcp->th_sum = sum;
2745                 }
2746
2747                 /*
2748                  * For TSO, the size of the protocol header is also
2749                  * included in the descriptor header size.
2750                  */
2751                 *start += (tcp->th_off << 2);
2752         } else
2753                 txq->vxtxq_stats.vmtxs_csum++;
2754
2755         return (0);
2756 }
2757
2758 static int
2759 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
2760     bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs)
2761 {
2762         struct vmxnet3_txring *txr;
2763         struct mbuf *m;
2764         bus_dma_tag_t tag;
2765         int error;
2766
2767         txr = &txq->vxtxq_cmd_ring;
2768         m = *m0;
2769         tag = txr->vxtxr_txtag;
2770
2771         error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
2772         if (error == 0 || error != EFBIG)
2773                 return (error);
2774
2775         m = m_defrag(m, M_NOWAIT);
2776         if (m != NULL) {
2777                 *m0 = m;
2778                 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
2779         } else
2780                 error = ENOBUFS;
2781
2782         if (error) {
2783                 m_freem(*m0);
2784                 *m0 = NULL;
2785                 txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++;
2786         } else
2787                 txq->vxtxq_sc->vmx_stats.vmst_defragged++;
2788
2789         return (error);
2790 }
2791
2792 static void
2793 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
2794 {
2795         struct vmxnet3_txring *txr;
2796
2797         txr = &txq->vxtxq_cmd_ring;
2798         bus_dmamap_unload(txr->vxtxr_txtag, dmap);
2799 }
2800
2801 static int
2802 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
2803 {
2804         struct vmxnet3_softc *sc;
2805         struct vmxnet3_txring *txr;
2806         struct vmxnet3_txdesc *txd, *sop;
2807         struct mbuf *m;
2808         bus_dmamap_t dmap;
2809         bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS];
2810         int i, gen, nsegs, etype, proto, start, error;
2811
2812         sc = txq->vxtxq_sc;
2813         start = 0;
2814         txd = NULL;
2815         txr = &txq->vxtxq_cmd_ring;
2816         dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
2817
2818         error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs);
2819         if (error)
2820                 return (error);
2821
2822         m = *m0;
2823         M_ASSERTPKTHDR(m);
2824         KASSERT(nsegs <= VMXNET3_TX_MAXSEGS,
2825             ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
2826
2827         if (VMXNET3_TXRING_AVAIL(txr) < nsegs) {
2828                 txq->vxtxq_stats.vmtxs_full++;
2829                 vmxnet3_txq_unload_mbuf(txq, dmap);
2830                 return (ENOSPC);
2831         } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
2832                 error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start);
2833                 if (error) {
2834                         txq->vxtxq_stats.vmtxs_offload_failed++;
2835                         vmxnet3_txq_unload_mbuf(txq, dmap);
2836                         m_freem(m);
2837                         *m0 = NULL;
2838                         return (error);
2839                 }
2840         }
2841
2842         txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m;
2843         sop = &txr->vxtxr_txd[txr->vxtxr_head];
2844         gen = txr->vxtxr_gen ^ 1;       /* Owned by cpu (yet) */
2845
2846         for (i = 0; i < nsegs; i++) {
2847                 txd = &txr->vxtxr_txd[txr->vxtxr_head];
2848
2849                 txd->addr = segs[i].ds_addr;
2850                 txd->len = segs[i].ds_len;
2851                 txd->gen = gen;
2852                 txd->dtype = 0;
2853                 txd->offload_mode = VMXNET3_OM_NONE;
2854                 txd->offload_pos = 0;
2855                 txd->hlen = 0;
2856                 txd->eop = 0;
2857                 txd->compreq = 0;
2858                 txd->vtag_mode = 0;
2859                 txd->vtag = 0;
2860
2861                 if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
2862                         txr->vxtxr_head = 0;
2863                         txr->vxtxr_gen ^= 1;
2864                 }
2865                 gen = txr->vxtxr_gen;
2866         }
2867         txd->eop = 1;
2868         txd->compreq = 1;
2869
2870         if (m->m_flags & M_VLANTAG) {
2871                 sop->vtag_mode = 1;
2872                 sop->vtag = m->m_pkthdr.ether_vtag;
2873         }
2874
2875         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2876                 sop->offload_mode = VMXNET3_OM_TSO;
2877                 sop->hlen = start;
2878                 sop->offload_pos = m->m_pkthdr.tso_segsz;
2879         } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
2880             VMXNET3_CSUM_OFFLOAD_IPV6)) {
2881                 sop->offload_mode = VMXNET3_OM_CSUM;
2882                 sop->hlen = start;
2883                 sop->offload_pos = start + m->m_pkthdr.csum_data;
2884         }
2885
2886         /* Finally, change the ownership. */
2887         vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
2888         sop->gen ^= 1;
2889
2890         txq->vxtxq_ts->npending += nsegs;
2891         if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
2892                 txq->vxtxq_ts->npending = 0;
2893                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
2894                     txr->vxtxr_head);
2895         }
2896
2897         return (0);
2898 }
2899
2900 #ifdef VMXNET3_LEGACY_TX
2901
2902 static void
2903 vmxnet3_start_locked(struct ifnet *ifp)
2904 {
2905         struct vmxnet3_softc *sc;
2906         struct vmxnet3_txqueue *txq;
2907         struct vmxnet3_txring *txr;
2908         struct mbuf *m_head;
2909         int tx, avail;
2910
2911         sc = ifp->if_softc;
2912         txq = &sc->vmx_txq[0];
2913         txr = &txq->vxtxq_cmd_ring;
2914         tx = 0;
2915
2916         VMXNET3_TXQ_LOCK_ASSERT(txq);
2917
2918         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2919             sc->vmx_link_active == 0)
2920                 return;
2921
2922         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
2923                 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2)
2924                         break;
2925
2926                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
2927                 if (m_head == NULL)
2928                         break;
2929
2930                 /* Assume worse case if this mbuf is the head of a chain. */
2931                 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
2932                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2933                         break;
2934                 }
2935
2936                 if (vmxnet3_txq_encap(txq, &m_head) != 0) {
2937                         if (m_head != NULL)
2938                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2939                         break;
2940                 }
2941
2942                 tx++;
2943                 ETHER_BPF_MTAP(ifp, m_head);
2944         }
2945
2946         if (tx > 0)
2947                 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
2948 }
2949
2950 static void
2951 vmxnet3_start(struct ifnet *ifp)
2952 {
2953         struct vmxnet3_softc *sc;
2954         struct vmxnet3_txqueue *txq;
2955
2956         sc = ifp->if_softc;
2957         txq = &sc->vmx_txq[0];
2958
2959         VMXNET3_TXQ_LOCK(txq);
2960         vmxnet3_start_locked(ifp);
2961         VMXNET3_TXQ_UNLOCK(txq);
2962 }
2963
2964 #else /* !VMXNET3_LEGACY_TX */
2965
2966 static int
2967 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m)
2968 {
2969         struct vmxnet3_softc *sc;
2970         struct vmxnet3_txring *txr;
2971         struct buf_ring *br;
2972         struct ifnet *ifp;
2973         int tx, avail, error;
2974
2975         sc = txq->vxtxq_sc;
2976         br = txq->vxtxq_br;
2977         ifp = sc->vmx_ifp;
2978         txr = &txq->vxtxq_cmd_ring;
2979         tx = 0;
2980         error = 0;
2981
2982         VMXNET3_TXQ_LOCK_ASSERT(txq);
2983
2984         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2985             sc->vmx_link_active == 0) {
2986                 if (m != NULL)
2987                         error = drbr_enqueue(ifp, br, m);
2988                 return (error);
2989         }
2990
2991         if (m != NULL) {
2992                 error = drbr_enqueue(ifp, br, m);
2993                 if (error)
2994                         return (error);
2995         }
2996
2997         while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) {
2998                 m = drbr_peek(ifp, br);
2999                 if (m == NULL)
3000                         break;
3001
3002                 /* Assume worse case if this mbuf is the head of a chain. */
3003                 if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
3004                         drbr_putback(ifp, br, m);
3005                         break;
3006                 }
3007
3008                 if (vmxnet3_txq_encap(txq, &m) != 0) {
3009                         if (m != NULL)
3010                                 drbr_putback(ifp, br, m);
3011                         else
3012                                 drbr_advance(ifp, br);
3013                         break;
3014                 }
3015                 drbr_advance(ifp, br);
3016
3017                 tx++;
3018                 ETHER_BPF_MTAP(ifp, m);
3019         }
3020
3021         if (tx > 0)
3022                 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
3023
3024         return (0);
3025 }
3026
3027 static int
3028 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m)
3029 {
3030         struct vmxnet3_softc *sc;
3031         struct vmxnet3_txqueue *txq;
3032         int i, ntxq, error;
3033
3034         sc = ifp->if_softc;
3035         ntxq = sc->vmx_ntxqueues;
3036
3037         /* check if flowid is set */
3038         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
3039                 i = m->m_pkthdr.flowid % ntxq;
3040         else
3041                 i = curcpu % ntxq;
3042
3043         txq = &sc->vmx_txq[i];
3044
3045         if (VMXNET3_TXQ_TRYLOCK(txq) != 0) {
3046                 error = vmxnet3_txq_mq_start_locked(txq, m);
3047                 VMXNET3_TXQ_UNLOCK(txq);
3048         } else {
3049                 error = drbr_enqueue(ifp, txq->vxtxq_br, m);
3050                 taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask);
3051         }
3052
3053         return (error);
3054 }
3055
3056 static void
3057 vmxnet3_txq_tq_deferred(void *xtxq, int pending)
3058 {
3059         struct vmxnet3_softc *sc;
3060         struct vmxnet3_txqueue *txq;
3061
3062         txq = xtxq;
3063         sc = txq->vxtxq_sc;
3064
3065         VMXNET3_TXQ_LOCK(txq);
3066         if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br))
3067                 vmxnet3_txq_mq_start_locked(txq, NULL);
3068         VMXNET3_TXQ_UNLOCK(txq);
3069 }
3070
3071 #endif /* VMXNET3_LEGACY_TX */
3072
3073 static void
3074 vmxnet3_txq_start(struct vmxnet3_txqueue *txq)
3075 {
3076         struct vmxnet3_softc *sc;
3077         struct ifnet *ifp;
3078
3079         sc = txq->vxtxq_sc;
3080         ifp = sc->vmx_ifp;
3081
3082 #ifdef VMXNET3_LEGACY_TX
3083         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3084                 vmxnet3_start_locked(ifp);
3085 #else
3086         if (!drbr_empty(ifp, txq->vxtxq_br))
3087                 vmxnet3_txq_mq_start_locked(txq, NULL);
3088 #endif
3089 }
3090
3091 static void
3092 vmxnet3_tx_start_all(struct vmxnet3_softc *sc)
3093 {
3094         struct vmxnet3_txqueue *txq;
3095         int i;
3096
3097         VMXNET3_CORE_LOCK_ASSERT(sc);
3098
3099         for (i = 0; i < sc->vmx_ntxqueues; i++) {
3100                 txq = &sc->vmx_txq[i];
3101
3102                 VMXNET3_TXQ_LOCK(txq);
3103                 vmxnet3_txq_start(txq);
3104                 VMXNET3_TXQ_UNLOCK(txq);
3105         }
3106 }
3107
3108 static void
3109 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
3110 {
3111         struct ifnet *ifp;
3112         int idx, bit;
3113
3114         ifp = sc->vmx_ifp;
3115         idx = (tag >> 5) & 0x7F;
3116         bit = tag & 0x1F;
3117
3118         if (tag == 0 || tag > 4095)
3119                 return;
3120
3121         VMXNET3_CORE_LOCK(sc);
3122
3123         /* Update our private VLAN bitvector. */
3124         if (add)
3125                 sc->vmx_vlan_filter[idx] |= (1 << bit);
3126         else
3127                 sc->vmx_vlan_filter[idx] &= ~(1 << bit);
3128
3129         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
3130                 if (add)
3131                         sc->vmx_ds->vlan_filter[idx] |= (1 << bit);
3132                 else
3133                         sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit);
3134                 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
3135         }
3136
3137         VMXNET3_CORE_UNLOCK(sc);
3138 }
3139
3140 static void
3141 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3142 {
3143
3144         if (ifp->if_softc == arg)
3145                 vmxnet3_update_vlan_filter(arg, 1, tag);
3146 }
3147
3148 static void
3149 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3150 {
3151
3152         if (ifp->if_softc == arg)
3153                 vmxnet3_update_vlan_filter(arg, 0, tag);
3154 }
3155
3156 static void
3157 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
3158 {
3159         struct ifnet *ifp;
3160         struct vmxnet3_driver_shared *ds;
3161         struct ifmultiaddr *ifma;
3162         u_int mode;
3163
3164         ifp = sc->vmx_ifp;
3165         ds = sc->vmx_ds;
3166
3167         mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST;
3168         if (ifp->if_flags & IFF_PROMISC)
3169                 mode |= VMXNET3_RXMODE_PROMISC;
3170         if (ifp->if_flags & IFF_ALLMULTI)
3171                 mode |= VMXNET3_RXMODE_ALLMULTI;
3172         else {
3173                 int cnt = 0, overflow = 0;
3174
3175                 if_maddr_rlock(ifp);
3176                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3177                         if (ifma->ifma_addr->sa_family != AF_LINK)
3178                                 continue;
3179                         else if (cnt == VMXNET3_MULTICAST_MAX) {
3180                                 overflow = 1;
3181                                 break;
3182                         }
3183
3184                         bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
3185                            &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN);
3186                         cnt++;
3187                 }
3188                 if_maddr_runlock(ifp);
3189
3190                 if (overflow != 0) {
3191                         cnt = 0;
3192                         mode |= VMXNET3_RXMODE_ALLMULTI;
3193                 } else if (cnt > 0)
3194                         mode |= VMXNET3_RXMODE_MCAST;
3195                 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN;
3196         }
3197
3198         ds->rxmode = mode;
3199
3200         vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
3201         vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
3202 }
3203
3204 static int
3205 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu)
3206 {
3207         struct ifnet *ifp;
3208
3209         ifp = sc->vmx_ifp;
3210
3211         if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU)
3212                 return (EINVAL);
3213
3214         ifp->if_mtu = mtu;
3215
3216         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3217                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3218                 vmxnet3_init_locked(sc);
3219         }
3220
3221         return (0);
3222 }
3223
3224 static int
3225 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
3226 {
3227         struct vmxnet3_softc *sc;
3228         struct ifreq *ifr;
3229         int reinit, mask, error;
3230
3231         sc = ifp->if_softc;
3232         ifr = (struct ifreq *) data;
3233         error = 0;
3234
3235         switch (cmd) {
3236         case SIOCSIFMTU:
3237                 if (ifp->if_mtu != ifr->ifr_mtu) {
3238                         VMXNET3_CORE_LOCK(sc);
3239                         error = vmxnet3_change_mtu(sc, ifr->ifr_mtu);
3240                         VMXNET3_CORE_UNLOCK(sc);
3241                 }
3242                 break;
3243
3244         case SIOCSIFFLAGS:
3245                 VMXNET3_CORE_LOCK(sc);
3246                 if (ifp->if_flags & IFF_UP) {
3247                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3248                                 if ((ifp->if_flags ^ sc->vmx_if_flags) &
3249                                     (IFF_PROMISC | IFF_ALLMULTI)) {
3250                                         vmxnet3_set_rxfilter(sc);
3251                                 }
3252                         } else
3253                                 vmxnet3_init_locked(sc);
3254                 } else {
3255                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3256                                 vmxnet3_stop(sc);
3257                 }
3258                 sc->vmx_if_flags = ifp->if_flags;
3259                 VMXNET3_CORE_UNLOCK(sc);
3260                 break;
3261
3262         case SIOCADDMULTI:
3263         case SIOCDELMULTI:
3264                 VMXNET3_CORE_LOCK(sc);
3265                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3266                         vmxnet3_set_rxfilter(sc);
3267                 VMXNET3_CORE_UNLOCK(sc);
3268                 break;
3269
3270         case SIOCSIFMEDIA:
3271         case SIOCGIFMEDIA:
3272                 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd);
3273                 break;
3274
3275         case SIOCSIFCAP:
3276                 VMXNET3_CORE_LOCK(sc);
3277                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3278
3279                 if (mask & IFCAP_TXCSUM)
3280                         ifp->if_capenable ^= IFCAP_TXCSUM;
3281                 if (mask & IFCAP_TXCSUM_IPV6)
3282                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3283                 if (mask & IFCAP_TSO4)
3284                         ifp->if_capenable ^= IFCAP_TSO4;
3285                 if (mask & IFCAP_TSO6)
3286                         ifp->if_capenable ^= IFCAP_TSO6;
3287
3288                 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO |
3289                     IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) {
3290                         /* Changing these features requires us to reinit. */
3291                         reinit = 1;
3292
3293                         if (mask & IFCAP_RXCSUM)
3294                                 ifp->if_capenable ^= IFCAP_RXCSUM;
3295                         if (mask & IFCAP_RXCSUM_IPV6)
3296                                 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3297                         if (mask & IFCAP_LRO)
3298                                 ifp->if_capenable ^= IFCAP_LRO;
3299                         if (mask & IFCAP_VLAN_HWTAGGING)
3300                                 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3301                         if (mask & IFCAP_VLAN_HWFILTER)
3302                                 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3303                 } else
3304                         reinit = 0;
3305
3306                 if (mask & IFCAP_VLAN_HWTSO)
3307                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3308
3309                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3310                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3311                         vmxnet3_init_locked(sc);
3312                 } else {
3313                         vmxnet3_init_hwassist(sc);
3314                 }
3315
3316                 VMXNET3_CORE_UNLOCK(sc);
3317                 VLAN_CAPABILITIES(ifp);
3318                 break;
3319
3320         default:
3321                 error = ether_ioctl(ifp, cmd, data);
3322                 break;
3323         }
3324
3325         VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc);
3326
3327         return (error);
3328 }
3329
3330 #ifndef VMXNET3_LEGACY_TX
3331 static void
3332 vmxnet3_qflush(struct ifnet *ifp)
3333 {
3334         struct vmxnet3_softc *sc;
3335         struct vmxnet3_txqueue *txq;
3336         struct mbuf *m;
3337         int i;
3338
3339         sc = ifp->if_softc;
3340
3341         for (i = 0; i < sc->vmx_ntxqueues; i++) {
3342                 txq = &sc->vmx_txq[i];
3343
3344                 VMXNET3_TXQ_LOCK(txq);
3345                 while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL)
3346                         m_freem(m);
3347                 VMXNET3_TXQ_UNLOCK(txq);
3348         }
3349
3350         if_qflush(ifp);
3351 }
3352 #endif
3353
3354 static int
3355 vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
3356 {
3357         struct vmxnet3_softc *sc;
3358
3359         sc = txq->vxtxq_sc;
3360
3361         VMXNET3_TXQ_LOCK(txq);
3362         if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
3363                 VMXNET3_TXQ_UNLOCK(txq);
3364                 return (0);
3365         }
3366         VMXNET3_TXQ_UNLOCK(txq);
3367
3368         if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n",
3369             txq->vxtxq_id);
3370         return (1);
3371 }
3372
3373 static void
3374 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
3375 {
3376
3377         vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
3378 }
3379
3380 static uint64_t
3381 vmxnet3_get_counter(struct ifnet *ifp, ift_counter cnt)
3382 {
3383         struct vmxnet3_softc *sc;
3384         uint64_t rv;
3385
3386         sc = if_getsoftc(ifp);
3387         rv = 0;
3388
3389         /*
3390          * With the exception of if_ierrors, these ifnet statistics are
3391          * only updated in the driver, so just set them to our accumulated
3392          * values. if_ierrors is updated in ether_input() for malformed
3393          * frames that we should have already discarded.
3394          */
3395         switch (cnt) {
3396         case IFCOUNTER_IPACKETS:
3397                 for (int i = 0; i < sc->vmx_nrxqueues; i++)
3398                         rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ipackets;
3399                 return (rv);
3400         case IFCOUNTER_IQDROPS:
3401                 for (int i = 0; i < sc->vmx_nrxqueues; i++)
3402                         rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_iqdrops;
3403                 return (rv);
3404         case IFCOUNTER_IERRORS:
3405                 for (int i = 0; i < sc->vmx_nrxqueues; i++)
3406                         rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ierrors;
3407                 return (rv);
3408         case IFCOUNTER_OPACKETS:
3409                 for (int i = 0; i < sc->vmx_ntxqueues; i++)
3410                         rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_opackets;
3411                 return (rv);
3412 #ifndef VMXNET3_LEGACY_TX
3413         case IFCOUNTER_OBYTES:
3414                 for (int i = 0; i < sc->vmx_ntxqueues; i++)
3415                         rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_obytes;
3416                 return (rv);
3417         case IFCOUNTER_OMCASTS:
3418                 for (int i = 0; i < sc->vmx_ntxqueues; i++)
3419                         rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_omcasts;
3420                 return (rv);
3421 #endif
3422         default:
3423                 return (if_get_counter_default(ifp, cnt));
3424         }
3425 }
3426
3427 static void
3428 vmxnet3_tick(void *xsc)
3429 {
3430         struct vmxnet3_softc *sc;
3431         struct ifnet *ifp;
3432         int i, timedout;
3433
3434         sc = xsc;
3435         ifp = sc->vmx_ifp;
3436         timedout = 0;
3437
3438         VMXNET3_CORE_LOCK_ASSERT(sc);
3439
3440         vmxnet3_refresh_host_stats(sc);
3441
3442         for (i = 0; i < sc->vmx_ntxqueues; i++)
3443                 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]);
3444
3445         if (timedout != 0) {
3446                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3447                 vmxnet3_init_locked(sc);
3448         } else
3449                 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
3450 }
3451
3452 static int
3453 vmxnet3_link_is_up(struct vmxnet3_softc *sc)
3454 {
3455         uint32_t status;
3456
3457         /* Also update the link speed while here. */
3458         status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
3459         sc->vmx_link_speed = status >> 16;
3460         return !!(status & 0x1);
3461 }
3462
3463 static void
3464 vmxnet3_link_status(struct vmxnet3_softc *sc)
3465 {
3466         struct ifnet *ifp;
3467         int link;
3468
3469         ifp = sc->vmx_ifp;
3470         link = vmxnet3_link_is_up(sc);
3471
3472         if (link != 0 && sc->vmx_link_active == 0) {
3473                 sc->vmx_link_active = 1;
3474                 if_link_state_change(ifp, LINK_STATE_UP);
3475         } else if (link == 0 && sc->vmx_link_active != 0) {
3476                 sc->vmx_link_active = 0;
3477                 if_link_state_change(ifp, LINK_STATE_DOWN);
3478         }
3479 }
3480
3481 static void
3482 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
3483 {
3484         struct vmxnet3_softc *sc;
3485
3486         sc = ifp->if_softc;
3487
3488         ifmr->ifm_status = IFM_AVALID;
3489         ifmr->ifm_active = IFM_ETHER;
3490
3491         VMXNET3_CORE_LOCK(sc);
3492         if (vmxnet3_link_is_up(sc) != 0) {
3493                 ifmr->ifm_status |= IFM_ACTIVE;
3494                 ifmr->ifm_active |= IFM_AUTO;
3495         } else
3496                 ifmr->ifm_active |= IFM_NONE;
3497         VMXNET3_CORE_UNLOCK(sc);
3498 }
3499
3500 static int
3501 vmxnet3_media_change(struct ifnet *ifp)
3502 {
3503
3504         /* Ignore. */
3505         return (0);
3506 }
3507
3508 static void
3509 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
3510 {
3511         uint32_t ml, mh;
3512
3513         ml  = sc->vmx_lladdr[0];
3514         ml |= sc->vmx_lladdr[1] << 8;
3515         ml |= sc->vmx_lladdr[2] << 16;
3516         ml |= sc->vmx_lladdr[3] << 24;
3517         vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
3518
3519         mh  = sc->vmx_lladdr[4];
3520         mh |= sc->vmx_lladdr[5] << 8;
3521         vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
3522 }
3523
3524 static void
3525 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
3526 {
3527         uint32_t ml, mh;
3528
3529         ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
3530         mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
3531
3532         sc->vmx_lladdr[0] = ml;
3533         sc->vmx_lladdr[1] = ml >> 8;
3534         sc->vmx_lladdr[2] = ml >> 16;
3535         sc->vmx_lladdr[3] = ml >> 24;
3536         sc->vmx_lladdr[4] = mh;
3537         sc->vmx_lladdr[5] = mh >> 8;
3538 }
3539
3540 static void
3541 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq,
3542     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3543 {
3544         struct sysctl_oid *node, *txsnode;
3545         struct sysctl_oid_list *list, *txslist;
3546         struct vmxnet3_txq_stats *stats;
3547         struct UPT1_TxStats *txstats;
3548         char namebuf[16];
3549
3550         stats = &txq->vxtxq_stats;
3551         txstats = &txq->vxtxq_ts->stats;
3552
3553         snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id);
3554         node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3555             NULL, "Transmit Queue");
3556         txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node);
3557
3558         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD,
3559             &stats->vmtxs_opackets, "Transmit packets");
3560         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD,
3561             &stats->vmtxs_obytes, "Transmit bytes");
3562         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD,
3563             &stats->vmtxs_omcasts, "Transmit multicasts");
3564         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
3565             &stats->vmtxs_csum, "Transmit checksum offloaded");
3566         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
3567             &stats->vmtxs_tso, "Transmit TCP segmentation offloaded");
3568         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD,
3569             &stats->vmtxs_full, "Transmit ring full");
3570         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD,
3571             &stats->vmtxs_offload_failed, "Transmit checksum offload failed");
3572
3573         /*
3574          * Add statistics reported by the host. These are updated once
3575          * per second.
3576          */
3577         txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3578             NULL, "Host Statistics");
3579         txslist = SYSCTL_CHILDREN(txsnode);
3580         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD,
3581             &txstats->TSO_packets, "TSO packets");
3582         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD,
3583             &txstats->TSO_bytes, "TSO bytes");
3584         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3585             &txstats->ucast_packets, "Unicast packets");
3586         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3587             &txstats->ucast_bytes, "Unicast bytes");
3588         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3589             &txstats->mcast_packets, "Multicast packets");
3590         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3591             &txstats->mcast_bytes, "Multicast bytes");
3592         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD,
3593             &txstats->error, "Errors");
3594         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD,
3595             &txstats->discard, "Discards");
3596 }
3597
3598 static void
3599 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq,
3600     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3601 {
3602         struct sysctl_oid *node, *rxsnode;
3603         struct sysctl_oid_list *list, *rxslist;
3604         struct vmxnet3_rxq_stats *stats;
3605         struct UPT1_RxStats *rxstats;
3606         char namebuf[16];
3607
3608         stats = &rxq->vxrxq_stats;
3609         rxstats = &rxq->vxrxq_rs->stats;
3610
3611         snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id);
3612         node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3613             NULL, "Receive Queue");
3614         rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node);
3615
3616         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD,
3617             &stats->vmrxs_ipackets, "Receive packets");
3618         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD,
3619             &stats->vmrxs_ibytes, "Receive bytes");
3620         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD,
3621             &stats->vmrxs_iqdrops, "Receive drops");
3622         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD,
3623             &stats->vmrxs_ierrors, "Receive errors");
3624
3625         /*
3626          * Add statistics reported by the host. These are updated once
3627          * per second.
3628          */
3629         rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3630             NULL, "Host Statistics");
3631         rxslist = SYSCTL_CHILDREN(rxsnode);
3632         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD,
3633             &rxstats->LRO_packets, "LRO packets");
3634         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD,
3635             &rxstats->LRO_bytes, "LRO bytes");
3636         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3637             &rxstats->ucast_packets, "Unicast packets");
3638         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3639             &rxstats->ucast_bytes, "Unicast bytes");
3640         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3641             &rxstats->mcast_packets, "Multicast packets");
3642         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3643             &rxstats->mcast_bytes, "Multicast bytes");
3644         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD,
3645             &rxstats->bcast_packets, "Broadcast packets");
3646         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD,
3647             &rxstats->bcast_bytes, "Broadcast bytes");
3648         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD,
3649             &rxstats->nobuffer, "No buffer");
3650         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD,
3651             &rxstats->error, "Errors");
3652 }
3653
3654 static void
3655 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc,
3656     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3657 {
3658         struct sysctl_oid *node;
3659         struct sysctl_oid_list *list;
3660         int i;
3661
3662         for (i = 0; i < sc->vmx_ntxqueues; i++) {
3663                 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i];
3664
3665                 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO,
3666                     "debug", CTLFLAG_RD, NULL, "");
3667                 list = SYSCTL_CHILDREN(node);
3668
3669                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD,
3670                     &txq->vxtxq_cmd_ring.vxtxr_head, 0, "");
3671                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD,
3672                     &txq->vxtxq_cmd_ring.vxtxr_next, 0, "");
3673                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD,
3674                     &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, "");
3675                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD,
3676                     &txq->vxtxq_cmd_ring.vxtxr_gen, 0, "");
3677                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3678                     &txq->vxtxq_comp_ring.vxcr_next, 0, "");
3679                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3680                     &txq->vxtxq_comp_ring.vxcr_ndesc, 0,"");
3681                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3682                     &txq->vxtxq_comp_ring.vxcr_gen, 0, "");
3683         }
3684
3685         for (i = 0; i < sc->vmx_nrxqueues; i++) {
3686                 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i];
3687
3688                 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
3689                     "debug", CTLFLAG_RD, NULL, "");
3690                 list = SYSCTL_CHILDREN(node);
3691
3692                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD,
3693                     &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, "");
3694                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
3695                     &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
3696                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
3697                     &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
3698                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD,
3699                     &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, "");
3700                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
3701                     &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
3702                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
3703                     &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
3704                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3705                     &rxq->vxrxq_comp_ring.vxcr_next, 0, "");
3706                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3707                     &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
3708                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3709                     &rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
3710         }
3711 }
3712
3713 static void
3714 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
3715     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3716 {
3717         int i;
3718
3719         for (i = 0; i < sc->vmx_ntxqueues; i++)
3720                 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child);
3721         for (i = 0; i < sc->vmx_nrxqueues; i++)
3722                 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child);
3723
3724         vmxnet3_setup_debug_sysctl(sc, ctx, child);
3725 }
3726
3727 static void
3728 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
3729 {
3730         device_t dev;
3731         struct vmxnet3_statistics *stats;
3732         struct sysctl_ctx_list *ctx;
3733         struct sysctl_oid *tree;
3734         struct sysctl_oid_list *child;
3735
3736         dev = sc->vmx_dev;
3737         ctx = device_get_sysctl_ctx(dev);
3738         tree = device_get_sysctl_tree(dev);
3739         child = SYSCTL_CHILDREN(tree);
3740
3741         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD,
3742             &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues");
3743         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD,
3744             &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues");
3745         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD,
3746             &sc->vmx_ntxqueues, 0, "Number of Tx queues");
3747         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD,
3748             &sc->vmx_nrxqueues, 0, "Number of Rx queues");
3749
3750         stats = &sc->vmx_stats;
3751         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD,
3752             &stats->vmst_defragged, 0, "Tx mbuf chains defragged");
3753         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD,
3754             &stats->vmst_defrag_failed, 0,
3755             "Tx mbuf dropped because defrag failed");
3756         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD,
3757             &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed");
3758         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD,
3759             &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed");
3760
3761         vmxnet3_setup_queue_sysctl(sc, ctx, child);
3762 }
3763
3764 static void
3765 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3766 {
3767
3768         bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
3769 }
3770
3771 static uint32_t
3772 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
3773 {
3774
3775         return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
3776 }
3777
3778 static void
3779 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3780 {
3781
3782         bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
3783 }
3784
3785 static void
3786 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3787 {
3788
3789         vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
3790 }
3791
3792 static uint32_t
3793 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3794 {
3795
3796         vmxnet3_write_cmd(sc, cmd);
3797         bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0,
3798             BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
3799         return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
3800 }
3801
3802 static void
3803 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
3804 {
3805
3806         vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
3807 }
3808
3809 static void
3810 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
3811 {
3812
3813         vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
3814 }
3815
3816 static void
3817 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
3818 {
3819         int i;
3820
3821         sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
3822         for (i = 0; i < sc->vmx_nintrs; i++)
3823                 vmxnet3_enable_intr(sc, i);
3824 }
3825
3826 static void
3827 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
3828 {
3829         int i;
3830
3831         sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
3832         for (i = 0; i < sc->vmx_nintrs; i++)
3833                 vmxnet3_disable_intr(sc, i);
3834 }
3835
3836 static void
3837 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3838 {
3839         bus_addr_t *baddr = arg;
3840
3841         if (error == 0)
3842                 *baddr = segs->ds_addr;
3843 }
3844
3845 static int
3846 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
3847     struct vmxnet3_dma_alloc *dma)
3848 {
3849         device_t dev;
3850         int error;
3851
3852         dev = sc->vmx_dev;
3853         bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3854
3855         error = bus_dma_tag_create(bus_get_dma_tag(dev),
3856             align, 0,           /* alignment, bounds */
3857             BUS_SPACE_MAXADDR,  /* lowaddr */
3858             BUS_SPACE_MAXADDR,  /* highaddr */
3859             NULL, NULL,         /* filter, filterarg */
3860             size,               /* maxsize */
3861             1,                  /* nsegments */
3862             size,               /* maxsegsize */
3863             BUS_DMA_ALLOCNOW,   /* flags */
3864             NULL,               /* lockfunc */
3865             NULL,               /* lockfuncarg */
3866             &dma->dma_tag);
3867         if (error) {
3868                 device_printf(dev, "bus_dma_tag_create failed: %d\n", error);
3869                 goto fail;
3870         }
3871
3872         error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
3873             BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
3874         if (error) {
3875                 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error);
3876                 goto fail;
3877         }
3878
3879         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3880             size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
3881         if (error) {
3882                 device_printf(dev, "bus_dmamap_load failed: %d\n", error);
3883                 goto fail;
3884         }
3885
3886         dma->dma_size = size;
3887
3888 fail:
3889         if (error)
3890                 vmxnet3_dma_free(sc, dma);
3891
3892         return (error);
3893 }
3894
3895 static void
3896 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
3897 {
3898
3899         if (dma->dma_tag != NULL) {
3900                 if (dma->dma_paddr != 0) {
3901                         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3902                             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3903                         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3904                 }
3905
3906                 if (dma->dma_vaddr != NULL) {
3907                         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
3908                             dma->dma_map);
3909                 }
3910
3911                 bus_dma_tag_destroy(dma->dma_tag);
3912         }
3913         bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3914 }
3915
3916 static int
3917 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def)
3918 {
3919         char path[64];
3920
3921         snprintf(path, sizeof(path),
3922             "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob);
3923         TUNABLE_INT_FETCH(path, &def);
3924
3925         return (def);
3926 }
3927
3928 /*
3929  * Since this is a purely paravirtualized device, we do not have
3930  * to worry about DMA coherency. But at times, we must make sure
3931  * both the compiler and CPU do not reorder memory operations.
3932  */
3933 static inline void
3934 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
3935 {
3936
3937         switch (type) {
3938         case VMXNET3_BARRIER_RD:
3939                 rmb();
3940                 break;
3941         case VMXNET3_BARRIER_WR:
3942                 wmb();
3943                 break;
3944         case VMXNET3_BARRIER_RDWR:
3945                 mb();
3946                 break;
3947         default:
3948                 panic("%s: bad barrier type %d", __func__, type);
3949         }
3950 }