sys/dev/vmware/vmxnet3/if_vmx.c

   1 /*-
   2  * Copyright (c) 2013 Tsubai Masanari
   3  * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
   4  *
   5  * Permission to use, copy, modify, and distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  16  *
  17  * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $
  18  */
  19
  20 /* Driver for VMware vmxnet3 virtual ethernet devices. */
  21
  22 #include <sys/cdefs.h>
  23 __FBSDID("$FreeBSD$");
  24
  25 #include <sys/param.h>
  26 #include <sys/systm.h>
  27 #include <sys/kernel.h>
  28 #include <sys/endian.h>
  29 #include <sys/sockio.h>
  30 #include <sys/mbuf.h>
  31 #include <sys/malloc.h>
  32 #include <sys/module.h>
  33 #include <sys/socket.h>
  34 #include <sys/sysctl.h>
  35 #include <vm/vm.h>
  36 #include <vm/pmap.h>
  37
  38 #include <net/ethernet.h>
  39 #include <net/if.h>
  40 #include <net/if_arp.h>
  41 #include <net/if_dl.h>
  42 #include <net/if_types.h>
  43 #include <net/if_media.h>
  44 #include <net/if_vlan_var.h>
  45
  46 #include <net/bpf.h>
  47
  48 #include <netinet/in_systm.h>
  49 #include <netinet/in.h>
  50 #include <netinet/ip.h>
  51 #include <netinet/ip6.h>
  52 #include <netinet6/ip6_var.h>
  53 #include <netinet/udp.h>
  54 #include <netinet/tcp.h>
  55
  56 #include <machine/bus.h>
  57 #include <machine/resource.h>
  58 #include <sys/bus.h>
  59 #include <sys/rman.h>
  60
  61 #include <dev/pci/pcireg.h>
  62 #include <dev/pci/pcivar.h>
  63
  64 #include "if_vmxreg.h"
  65 #include "if_vmxvar.h"
  66
  67 #include "opt_inet.h"
  68 #include "opt_inet6.h"
  69
  70 /* Always enable for now - useful for queue hangs. */
  71 #define VMXNET3_DEBUG_SYSCTL
  72
  73 #ifdef VMXNET3_FAILPOINTS
  74 #include <sys/fail.h>
  75 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0,
  76     "vmxnet3 fail points");
  77 #define VMXNET3_FP      _debug_fail_point_vmxnet3
  78 #endif
  79
  80 static int      vmxnet3_probe(device_t);
  81 static int      vmxnet3_attach(device_t);
  82 static int      vmxnet3_detach(device_t);
  83 static int      vmxnet3_shutdown(device_t);
  84
  85 static int      vmxnet3_alloc_resources(struct vmxnet3_softc *);
  86 static void     vmxnet3_free_resources(struct vmxnet3_softc *);
  87 static int      vmxnet3_check_version(struct vmxnet3_softc *);
  88 static void     vmxnet3_initial_config(struct vmxnet3_softc *);
  89
  90 static int      vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
  91 static int      vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
  92 static int      vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
  93 static int      vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int,
  94                     struct vmxnet3_interrupt *);
  95 static int      vmxnet3_alloc_intr_resources(struct vmxnet3_softc *);
  96 static int      vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
  97 static int      vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
  98 static int      vmxnet3_setup_interrupts(struct vmxnet3_softc *);
  99 static int      vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
 100
 101 static void     vmxnet3_free_interrupt(struct vmxnet3_softc *,
 102                     struct vmxnet3_interrupt *);
 103 static void     vmxnet3_free_interrupts(struct vmxnet3_softc *);
 104
 105 static int      vmxnet3_init_rxq(struct vmxnet3_softc *, int);
 106 static int      vmxnet3_init_txq(struct vmxnet3_softc *, int);
 107 static int      vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
 108 static void     vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
 109 static void     vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
 110 static void     vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);
 111
 112 static int      vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
 113 static void     vmxnet3_free_shared_data(struct vmxnet3_softc *);
 114 static int      vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
 115 static void     vmxnet3_free_txq_data(struct vmxnet3_softc *);
 116 static int      vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
 117 static void     vmxnet3_free_rxq_data(struct vmxnet3_softc *);
 118 static int      vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
 119 static void     vmxnet3_free_queue_data(struct vmxnet3_softc *);
 120 static int      vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
 121 static void     vmxnet3_init_shared_data(struct vmxnet3_softc *);
 122 static void     vmxnet3_reinit_interface(struct vmxnet3_softc *);
 123 static void     vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
 124 static int      vmxnet3_alloc_data(struct vmxnet3_softc *);
 125 static void     vmxnet3_free_data(struct vmxnet3_softc *);
 126 static int      vmxnet3_setup_interface(struct vmxnet3_softc *);
 127
 128 static void     vmxnet3_evintr(struct vmxnet3_softc *);
 129 static void     vmxnet3_txq_eof(struct vmxnet3_txqueue *);
 130 static void     vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
 131 static int      vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
 132 static void     vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
 133                     struct vmxnet3_rxring *, int);
 134 static void     vmxnet3_rxq_eof(struct vmxnet3_rxqueue *);
 135 static void     vmxnet3_legacy_intr(void *);
 136 static void     vmxnet3_txq_intr(void *);
 137 static void     vmxnet3_rxq_intr(void *);
 138 static void     vmxnet3_event_intr(void *);
 139
 140 static void     vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
 141 static void     vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
 142 static void     vmxnet3_stop(struct vmxnet3_softc *);
 143
 144 static void     vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
 145 static int      vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
 146 static int      vmxnet3_reinit_queues(struct vmxnet3_softc *);
 147 static int      vmxnet3_enable_device(struct vmxnet3_softc *);
 148 static void     vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
 149 static int      vmxnet3_reinit(struct vmxnet3_softc *);
 150 static void     vmxnet3_init_locked(struct vmxnet3_softc *);
 151 static void     vmxnet3_init(void *);
 152
 153 static int      vmxnet3_txq_offload_ctx(struct mbuf *, int *, int *, int *);
 154 static int      vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **,
 155                     bus_dmamap_t, bus_dma_segment_t [], int *);
 156 static void     vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
 157 static int      vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
 158 static void     vmxnet3_start_locked(struct ifnet *);
 159 static void     vmxnet3_start(struct ifnet *);
 160
 161 static void     vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
 162                     uint16_t);
 163 static void     vmxnet3_register_vlan(void *, struct ifnet *, uint16_t);
 164 static void     vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t);
 165 static void     vmxnet3_set_rxfilter(struct vmxnet3_softc *);
 166 static int      vmxnet3_change_mtu(struct vmxnet3_softc *, int);
 167 static int      vmxnet3_ioctl(struct ifnet *, u_long, caddr_t);
 168
 169 static int      vmxnet3_watchdog(struct vmxnet3_txqueue *);
 170 static void     vmxnet3_tick(void *);
 171 static void     vmxnet3_link_status(struct vmxnet3_softc *);
 172 static void     vmxnet3_media_status(struct ifnet *, struct ifmediareq *);
 173 static int      vmxnet3_media_change(struct ifnet *);
 174 static void     vmxnet3_set_lladdr(struct vmxnet3_softc *);
 175 static void     vmxnet3_get_lladdr(struct vmxnet3_softc *);
 176
 177 static void     vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *,
 178                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 179 static void     vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *,
 180                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 181 static void     vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *,
 182                     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 183 static void     vmxnet3_setup_sysctl(struct vmxnet3_softc *);
 184
 185 static void     vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t,
 186                     uint32_t);
 187 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t);
 188 static void     vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t,
 189                     uint32_t);
 190 static void     vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t);
 191 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t);
 192
 193 static void     vmxnet3_enable_intr(struct vmxnet3_softc *, int);
 194 static void     vmxnet3_disable_intr(struct vmxnet3_softc *, int);
 195 static void     vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
 196 static void     vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
 197
 198 static int      vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t,
 199                     bus_size_t, struct vmxnet3_dma_alloc *);
 200 static void     vmxnet3_dma_free(struct vmxnet3_softc *,
 201                     struct vmxnet3_dma_alloc *);
 202 static int      vmxnet3_tunable_int(struct vmxnet3_softc *,
 203                     const char *, int);
 204
 205 typedef enum {
 206         VMXNET3_BARRIER_RD,
 207         VMXNET3_BARRIER_WR,
 208         VMXNET3_BARRIER_RDWR,
 209 } vmxnet3_barrier_t;
 210
 211 static void     vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
 212
 213 /* Tunables. */
 214 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC;
 215 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc);
 216 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC;
 217 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc);
 218
 219 static device_method_t vmxnet3_methods[] = {
 220         /* Device interface. */
 221         DEVMETHOD(device_probe,         vmxnet3_probe),
 222         DEVMETHOD(device_attach,        vmxnet3_attach),
 223         DEVMETHOD(device_detach,        vmxnet3_detach),
 224         DEVMETHOD(device_shutdown,      vmxnet3_shutdown),
 225
 226         DEVMETHOD_END
 227 };
 228
 229 static driver_t vmxnet3_driver = {
 230         "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc)
 231 };
 232
 233 static devclass_t vmxnet3_devclass;
 234 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0);
 235
 236 MODULE_DEPEND(vmx, pci, 1, 1, 1);
 237 MODULE_DEPEND(vmx, ether, 1, 1, 1);
 238
 239 #define VMXNET3_VMWARE_VENDOR_ID        0x15AD
 240 #define VMXNET3_VMWARE_DEVICE_ID        0x07B0
 241
 242 static int
 243 vmxnet3_probe(device_t dev)
 244 {
 245
 246         if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID &&
 247             pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) {
 248                 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter");
 249                 return (BUS_PROBE_DEFAULT);
 250         }
 251
 252         return (ENXIO);
 253 }
 254
 255 static int
 256 vmxnet3_attach(device_t dev)
 257 {
 258         struct vmxnet3_softc *sc;
 259         int error;
 260
 261         sc = device_get_softc(dev);
 262         sc->vmx_dev = dev;
 263
 264         pci_enable_busmaster(dev);
 265
 266         VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev));
 267         callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0);
 268
 269         vmxnet3_initial_config(sc);
 270
 271         error = vmxnet3_alloc_resources(sc);
 272         if (error)
 273                 goto fail;
 274
 275         error = vmxnet3_check_version(sc);
 276         if (error)
 277                 goto fail;
 278
 279         error = vmxnet3_alloc_rxtx_queues(sc);
 280         if (error)
 281                 goto fail;
 282
 283         error = vmxnet3_alloc_interrupts(sc);
 284         if (error)
 285                 goto fail;
 286
 287         error = vmxnet3_alloc_data(sc);
 288         if (error)
 289                 goto fail;
 290
 291         error = vmxnet3_setup_interface(sc);
 292         if (error)
 293                 goto fail;
 294
 295         error = vmxnet3_setup_interrupts(sc);
 296         if (error) {
 297                 ether_ifdetach(sc->vmx_ifp);
 298                 device_printf(dev, "could not set up interrupt\n");
 299                 goto fail;
 300         }
 301
 302         vmxnet3_setup_sysctl(sc);
 303         vmxnet3_link_status(sc);
 304
 305 fail:
 306         if (error)
 307                 vmxnet3_detach(dev);
 308
 309         return (error);
 310 }
 311
 312 static int
 313 vmxnet3_detach(device_t dev)
 314 {
 315         struct vmxnet3_softc *sc;
 316         struct ifnet *ifp;
 317
 318         sc = device_get_softc(dev);
 319         ifp = sc->vmx_ifp;
 320
 321         if (device_is_attached(dev)) {
 322                 ether_ifdetach(ifp);
 323                 VMXNET3_CORE_LOCK(sc);
 324                 vmxnet3_stop(sc);
 325                 VMXNET3_CORE_UNLOCK(sc);
 326                 callout_drain(&sc->vmx_tick);
 327         }
 328
 329         if (sc->vmx_vlan_attach != NULL) {
 330                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach);
 331                 sc->vmx_vlan_attach = NULL;
 332         }
 333         if (sc->vmx_vlan_detach != NULL) {
 334                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach);
 335                 sc->vmx_vlan_detach = NULL;
 336         }
 337
 338         vmxnet3_free_interrupts(sc);
 339
 340         if (ifp != NULL) {
 341                 if_free(ifp);
 342                 sc->vmx_ifp = NULL;
 343         }
 344
 345         ifmedia_removeall(&sc->vmx_media);
 346
 347         vmxnet3_free_data(sc);
 348         vmxnet3_free_resources(sc);
 349         vmxnet3_free_rxtx_queues(sc);
 350
 351         VMXNET3_CORE_LOCK_DESTROY(sc);
 352
 353         return (0);
 354 }
 355
 356 static int
 357 vmxnet3_shutdown(device_t dev)
 358 {
 359
 360         return (0);
 361 }
 362
 363 static int
 364 vmxnet3_alloc_resources(struct vmxnet3_softc *sc)
 365 {
 366         device_t dev;
 367         int rid;
 368
 369         dev = sc->vmx_dev;
 370
 371         rid = PCIR_BAR(0);
 372         sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 373             RF_ACTIVE);
 374         if (sc->vmx_res0 == NULL) {
 375                 device_printf(dev,
 376                     "could not map BAR0 memory\n");
 377                 return (ENXIO);
 378         }
 379
 380         sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0);
 381         sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0);
 382
 383         rid = PCIR_BAR(1);
 384         sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 385             RF_ACTIVE);
 386         if (sc->vmx_res1 == NULL) {
 387                 device_printf(dev,
 388                     "could not map BAR1 memory\n");
 389                 return (ENXIO);
 390         }
 391
 392         sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1);
 393         sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1);
 394
 395         if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
 396                 rid = PCIR_BAR(2);
 397                 sc->vmx_msix_res = bus_alloc_resource_any(dev,
 398                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 399         }
 400
 401         if (sc->vmx_msix_res == NULL)
 402                 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
 403
 404         return (0);
 405 }
 406
 407 static void
 408 vmxnet3_free_resources(struct vmxnet3_softc *sc)
 409 {
 410         device_t dev;
 411         int rid;
 412
 413         dev = sc->vmx_dev;
 414
 415         if (sc->vmx_res0 != NULL) {
 416                 rid = PCIR_BAR(0);
 417                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0);
 418                 sc->vmx_res0 = NULL;
 419         }
 420
 421         if (sc->vmx_res1 != NULL) {
 422                 rid = PCIR_BAR(1);
 423                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1);
 424                 sc->vmx_res1 = NULL;
 425         }
 426
 427         if (sc->vmx_msix_res != NULL) {
 428                 rid = PCIR_BAR(2);
 429                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
 430                     sc->vmx_msix_res);
 431                 sc->vmx_msix_res = NULL;
 432         }
 433 }
 434
 435 static int
 436 vmxnet3_check_version(struct vmxnet3_softc *sc)
 437 {
 438         device_t dev;
 439         uint32_t version;
 440
 441         dev = sc->vmx_dev;
 442
 443         version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
 444         if ((version & 0x01) == 0) {
 445                 device_printf(dev, "unsupported hardware version %#x\n",
 446                     version);
 447                 return (ENOTSUP);
 448         }
 449         vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
 450
 451         version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
 452         if ((version & 0x01) == 0) {
 453                 device_printf(dev, "unsupported UPT version %#x\n", version);
 454                 return (ENOTSUP);
 455         }
 456         vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
 457
 458         return (0);
 459 }
 460
 461 static void
 462 vmxnet3_initial_config(struct vmxnet3_softc *sc)
 463 {
 464         int ndesc;
 465
 466         /*
 467          * BMV Much of the work is already done, but this driver does
 468          * not support multiqueue yet.
 469          */
 470         sc->vmx_ntxqueues = VMXNET3_TX_QUEUES;
 471         sc->vmx_nrxqueues = VMXNET3_RX_QUEUES;
 472
 473         ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc);
 474         if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC)
 475                 ndesc = VMXNET3_DEF_TX_NDESC;
 476         if (ndesc & VMXNET3_MASK_TX_NDESC)
 477                 ndesc &= ~VMXNET3_MASK_TX_NDESC;
 478         sc->vmx_ntxdescs = ndesc;
 479
 480         ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc);
 481         if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC)
 482                 ndesc = VMXNET3_DEF_RX_NDESC;
 483         if (ndesc & VMXNET3_MASK_RX_NDESC)
 484                 ndesc &= ~VMXNET3_MASK_RX_NDESC;
 485         sc->vmx_nrxdescs = ndesc;
 486         sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
 487 }
 488
 489 static int
 490 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
 491 {
 492         device_t dev;
 493         int nmsix, cnt, required;
 494
 495         dev = sc->vmx_dev;
 496
 497         if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
 498                 return (1);
 499
 500         /* Allocate an additional vector for the events interrupt. */
 501         required = sc->vmx_nrxqueues + sc->vmx_ntxqueues + 1;
 502
 503         nmsix = pci_msix_count(dev);
 504         if (nmsix < required)
 505                 return (1);
 506
 507         cnt = required;
 508         if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
 509                 sc->vmx_nintrs = required;
 510                 return (0);
 511         } else
 512                 pci_release_msi(dev);
 513
 514         return (1);
 515 }
 516
 517 static int
 518 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
 519 {
 520         device_t dev;
 521         int nmsi, cnt, required;
 522
 523         dev = sc->vmx_dev;
 524         required = 1;
 525
 526         nmsi = pci_msi_count(dev);
 527         if (nmsi < required)
 528                 return (1);
 529
 530         cnt = required;
 531         if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) {
 532                 sc->vmx_nintrs = 1;
 533                 return (0);
 534         } else
 535                 pci_release_msi(dev);
 536
 537         return (1);
 538 }
 539
 540 static int
 541 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
 542 {
 543
 544         sc->vmx_nintrs = 1;
 545         return (0);
 546 }
 547
 548 static int
 549 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags,
 550     struct vmxnet3_interrupt *intr)
 551 {
 552         struct resource *irq;
 553
 554         irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags);
 555         if (irq == NULL)
 556                 return (ENXIO);
 557
 558         intr->vmxi_irq = irq;
 559         intr->vmxi_rid = rid;
 560
 561         return (0);
 562 }
 563
 564 static int
 565 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc)
 566 {
 567         int i, rid, flags, error;
 568
 569         rid = 0;
 570         flags = RF_ACTIVE;
 571
 572         if (sc->vmx_intr_type == VMXNET3_IT_LEGACY)
 573                 flags |= RF_SHAREABLE;
 574         else
 575                 rid = 1;
 576
 577         for (i = 0; i < sc->vmx_nintrs; i++, rid++) {
 578                 error = vmxnet3_alloc_interrupt(sc, rid, flags,
 579                     &sc->vmx_intrs[i]);
 580                 if (error)
 581                         return (error);
 582         }
 583
 584         return (0);
 585 }
 586
 587 /*
 588  * NOTE: We only support the simple case of each Rx and Tx queue on its
 589  * own MSIX vector. This is good enough until we support mulitqueue.
 590  */
 591 static int
 592 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
 593 {
 594         device_t dev;
 595         struct vmxnet3_txqueue *txq;
 596         struct vmxnet3_rxqueue *rxq;
 597         struct vmxnet3_interrupt *intr;
 598         enum intr_type type;
 599         int i, error;
 600
 601         dev = sc->vmx_dev;
 602         intr = &sc->vmx_intrs[0];
 603         type = INTR_TYPE_NET | INTR_MPSAFE;
 604
 605         for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) {
 606                 txq = &sc->vmx_txq[i];
 607                 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
 608                      vmxnet3_txq_intr, txq, &intr->vmxi_handler);
 609                 if (error)
 610                         return (error);
 611                 txq->vxtxq_intr_idx = intr->vmxi_rid - 1;
 612         }
 613
 614         for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) {
 615                 rxq = &sc->vmx_rxq[i];
 616                 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
 617                     vmxnet3_rxq_intr, rxq, &intr->vmxi_handler);
 618                 if (error)
 619                         return (error);
 620                 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1;
 621         }
 622
 623         error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL,
 624             vmxnet3_event_intr, sc, &intr->vmxi_handler);
 625         if (error)
 626                 return (error);
 627         sc->vmx_event_intr_idx = intr->vmxi_rid - 1;
 628
 629         return (0);
 630 }
 631
 632 static int
 633 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
 634 {
 635         struct vmxnet3_interrupt *intr;
 636         int i, error;
 637
 638         intr = &sc->vmx_intrs[0];
 639         error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq,
 640             INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc,
 641             &intr->vmxi_handler);
 642
 643         for (i = 0; i < sc->vmx_ntxqueues; i++)
 644                 sc->vmx_txq[i].vxtxq_intr_idx = 0;
 645         for (i = 0; i < sc->vmx_nrxqueues; i++)
 646                 sc->vmx_rxq[i].vxrxq_intr_idx = 0;
 647         sc->vmx_event_intr_idx = 0;
 648
 649         return (error);
 650 }
 651
 652 /*
 653  * XXX BMV Should probably reorganize the attach and just do
 654  * this in vmxnet3_init_shared_data().
 655  */
 656 static void
 657 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
 658 {
 659         struct vmxnet3_txqueue *txq;
 660         struct vmxnet3_txq_shared *txs;
 661         struct vmxnet3_rxqueue *rxq;
 662         struct vmxnet3_rxq_shared *rxs;
 663         int i;
 664
 665         sc->vmx_ds->evintr = sc->vmx_event_intr_idx;
 666
 667         for (i = 0; i < sc->vmx_ntxqueues; i++) {
 668                 txq = &sc->vmx_txq[i];
 669                 txs = txq->vxtxq_ts;
 670                 txs->intr_idx = txq->vxtxq_intr_idx;
 671         }
 672
 673         for (i = 0; i < sc->vmx_nrxqueues; i++) {
 674                 rxq = &sc->vmx_rxq[i];
 675                 rxs = rxq->vxrxq_rs;
 676                 rxs->intr_idx = rxq->vxrxq_intr_idx;
 677         }
 678 }
 679
 680 static int
 681 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
 682 {
 683         int error;
 684
 685         error = vmxnet3_alloc_intr_resources(sc);
 686         if (error)
 687                 return (error);
 688
 689         switch (sc->vmx_intr_type) {
 690         case VMXNET3_IT_MSIX:
 691                 error = vmxnet3_setup_msix_interrupts(sc);
 692                 break;
 693         case VMXNET3_IT_MSI:
 694         case VMXNET3_IT_LEGACY:
 695                 error = vmxnet3_setup_legacy_interrupt(sc);
 696                 break;
 697         default:
 698                 panic("%s: invalid interrupt type %d", __func__,
 699                     sc->vmx_intr_type);
 700         }
 701
 702         if (error == 0)
 703                 vmxnet3_set_interrupt_idx(sc);
 704
 705         return (error);
 706 }
 707
 708 static int
 709 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
 710 {
 711         device_t dev;
 712         uint32_t config;
 713         int error;
 714
 715         dev = sc->vmx_dev;
 716         config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
 717
 718         sc->vmx_intr_type = config & 0x03;
 719         sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
 720
 721         switch (sc->vmx_intr_type) {
 722         case VMXNET3_IT_AUTO:
 723                 sc->vmx_intr_type = VMXNET3_IT_MSIX;
 724                 /* FALLTHROUGH */
 725         case VMXNET3_IT_MSIX:
 726                 error = vmxnet3_alloc_msix_interrupts(sc);
 727                 if (error == 0)
 728                         break;
 729                 sc->vmx_intr_type = VMXNET3_IT_MSI;
 730                 /* FALLTHROUGH */
 731         case VMXNET3_IT_MSI:
 732                 error = vmxnet3_alloc_msi_interrupts(sc);
 733                 if (error == 0)
 734                         break;
 735                 sc->vmx_intr_type = VMXNET3_IT_LEGACY;
 736                 /* FALLTHROUGH */
 737         case VMXNET3_IT_LEGACY:
 738                 error = vmxnet3_alloc_legacy_interrupts(sc);
 739                 if (error == 0)
 740                         break;
 741                 /* FALLTHROUGH */
 742         default:
 743                 sc->vmx_intr_type = -1;
 744                 device_printf(dev, "cannot allocate any interrupt resources\n");
 745                 return (ENXIO);
 746         }
 747
 748         return (error);
 749 }
 750
 751 static void
 752 vmxnet3_free_interrupt(struct vmxnet3_softc *sc,
 753     struct vmxnet3_interrupt *intr)
 754 {
 755         device_t dev;
 756
 757         dev = sc->vmx_dev;
 758
 759         if (intr->vmxi_handler != NULL) {
 760                 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler);
 761                 intr->vmxi_handler = NULL;
 762         }
 763
 764         if (intr->vmxi_irq != NULL) {
 765                 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid,
 766                     intr->vmxi_irq);
 767                 intr->vmxi_irq = NULL;
 768                 intr->vmxi_rid = -1;
 769         }
 770 }
 771
 772 static void
 773 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
 774 {
 775         int i;
 776
 777         for (i = 0; i < sc->vmx_nintrs; i++)
 778                 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]);
 779
 780         if (sc->vmx_intr_type == VMXNET3_IT_MSI ||
 781             sc->vmx_intr_type == VMXNET3_IT_MSIX)
 782                 pci_release_msi(sc->vmx_dev);
 783 }
 784
 785 static int
 786 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
 787 {
 788         struct vmxnet3_rxqueue *rxq;
 789         struct vmxnet3_rxring *rxr;
 790         int i;
 791
 792         rxq = &sc->vmx_rxq[q];
 793
 794         snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
 795             device_get_nameunit(sc->vmx_dev), q);
 796         mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF);
 797
 798         rxq->vxrxq_sc = sc;
 799         rxq->vxrxq_id = q;
 800
 801         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
 802                 rxr = &rxq->vxrxq_cmd_ring[i];
 803                 rxr->vxrxr_rid = i;
 804                 rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
 805                 rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc *
 806                     sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO);
 807                 if (rxr->vxrxr_rxbuf == NULL)
 808                         return (ENOMEM);
 809
 810                 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
 811         }
 812
 813         return (0);
 814 }
 815
 816 static int
 817 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
 818 {
 819         struct vmxnet3_txqueue *txq;
 820         struct vmxnet3_txring *txr;
 821
 822         txq = &sc->vmx_txq[q];
 823         txr = &txq->vxtxq_cmd_ring;
 824
 825         snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
 826             device_get_nameunit(sc->vmx_dev), q);
 827         mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF);
 828
 829         txq->vxtxq_sc = sc;
 830         txq->vxtxq_id = q;
 831
 832         txr->vxtxr_ndesc = sc->vmx_ntxdescs;
 833         txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc *
 834             sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO);
 835         if (txr->vxtxr_txbuf == NULL)
 836                 return (ENOMEM);
 837
 838         txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
 839
 840         return (0);
 841 }
 842
 843 static int
 844 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
 845 {
 846         int i, error;
 847
 848         sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) *
 849             sc->vmx_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
 850         sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) *
 851             sc->vmx_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO);
 852         if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL)
 853                 return (ENOMEM);
 854
 855         for (i = 0; i < sc->vmx_nrxqueues; i++) {
 856                 error = vmxnet3_init_rxq(sc, i);
 857                 if (error)
 858                         return (error);
 859         }
 860
 861         for (i = 0; i < sc->vmx_ntxqueues; i++) {
 862                 error = vmxnet3_init_txq(sc, i);
 863                 if (error)
 864                         return (error);
 865         }
 866
 867         return (0);
 868 }
 869
 870 static void
 871 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
 872 {
 873         struct vmxnet3_rxring *rxr;
 874         int i;
 875
 876         rxq->vxrxq_sc = NULL;
 877         rxq->vxrxq_id = -1;
 878
 879         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
 880                 rxr = &rxq->vxrxq_cmd_ring[i];
 881
 882                 if (rxr->vxrxr_rxbuf != NULL) {
 883                         free(rxr->vxrxr_rxbuf, M_DEVBUF);
 884                         rxr->vxrxr_rxbuf = NULL;
 885                 }
 886         }
 887
 888         if (mtx_initialized(&rxq->vxrxq_mtx) != 0)
 889                 mtx_destroy(&rxq->vxrxq_mtx);
 890 }
 891
 892 static void
 893 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
 894 {
 895         struct vmxnet3_txring *txr;
 896
 897         txr = &txq->vxtxq_cmd_ring;
 898
 899         txq->vxtxq_sc = NULL;
 900         txq->vxtxq_id = -1;
 901
 902         if (txr->vxtxr_txbuf != NULL) {
 903                 free(txr->vxtxr_txbuf, M_DEVBUF);
 904                 txr->vxtxr_txbuf = NULL;
 905         }
 906
 907         if (mtx_initialized(&txq->vxtxq_mtx) != 0)
 908                 mtx_destroy(&txq->vxtxq_mtx);
 909 }
 910
 911 static void
 912 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
 913 {
 914         int i;
 915
 916         if (sc->vmx_rxq != NULL) {
 917                 for (i = 0; i < sc->vmx_nrxqueues; i++)
 918                         vmxnet3_destroy_rxq(&sc->vmx_rxq[i]);
 919                 free(sc->vmx_rxq, M_DEVBUF);
 920                 sc->vmx_rxq = NULL;
 921         }
 922
 923         if (sc->vmx_txq != NULL) {
 924                 for (i = 0; i < sc->vmx_ntxqueues; i++)
 925                         vmxnet3_destroy_txq(&sc->vmx_txq[i]);
 926                 free(sc->vmx_txq, M_DEVBUF);
 927                 sc->vmx_txq = NULL;
 928         }
 929 }
 930
 931 static int
 932 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
 933 {
 934         device_t dev;
 935         uint8_t *kva;
 936         size_t size;
 937         int i, error;
 938
 939         dev = sc->vmx_dev;
 940
 941         size = sizeof(struct vmxnet3_driver_shared);
 942         error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
 943         if (error) {
 944                 device_printf(dev, "cannot alloc shared memory\n");
 945                 return (error);
 946         }
 947         sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;
 948
 949         size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
 950             sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
 951         error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
 952         if (error) {
 953                 device_printf(dev, "cannot alloc queue shared memory\n");
 954                 return (error);
 955         }
 956         sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
 957         kva = sc->vmx_qs;
 958
 959         for (i = 0; i < sc->vmx_ntxqueues; i++) {
 960                 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
 961                 kva += sizeof(struct vmxnet3_txq_shared);
 962         }
 963         for (i = 0; i < sc->vmx_nrxqueues; i++) {
 964                 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
 965                 kva += sizeof(struct vmxnet3_rxq_shared);
 966         }
 967
 968         return (0);
 969 }
 970
 971 static void
 972 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
 973 {
 974
 975         if (sc->vmx_qs != NULL) {
 976                 vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
 977                 sc->vmx_qs = NULL;
 978         }
 979
 980         if (sc->vmx_ds != NULL) {
 981                 vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
 982                 sc->vmx_ds = NULL;
 983         }
 984 }
 985
 986 static int
 987 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
 988 {
 989         device_t dev;
 990         struct vmxnet3_txqueue *txq;
 991         struct vmxnet3_txring *txr;
 992         struct vmxnet3_comp_ring *txc;
 993         size_t descsz, compsz;
 994         int i, q, error;
 995
 996         dev = sc->vmx_dev;
 997
 998         for (q = 0; q < sc->vmx_ntxqueues; q++) {
 999                 txq = &sc->vmx_txq[q];
1000                 txr = &txq->vxtxq_cmd_ring;
1001                 txc = &txq->vxtxq_comp_ring;
1002
1003                 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
1004                 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);
1005
1006                 error = bus_dma_tag_create(bus_get_dma_tag(dev),
1007                     1, 0,                       /* alignment, boundary */
1008                     BUS_SPACE_MAXADDR,          /* lowaddr */
1009                     BUS_SPACE_MAXADDR,          /* highaddr */
1010                     NULL, NULL,                 /* filter, filterarg */
1011                     VMXNET3_TSO_MAXSIZE,        /* maxsize */
1012                     VMXNET3_TX_MAXSEGS,         /* nsegments */
1013                     VMXNET3_TX_MAXSEGSIZE,      /* maxsegsize */
1014                     0,                          /* flags */
1015                     NULL, NULL,                 /* lockfunc, lockarg */
1016                     &txr->vxtxr_txtag);
1017                 if (error) {
1018                         device_printf(dev,
1019                             "unable to create Tx buffer tag for queue %d\n", q);
1020                         return (error);
1021                 }
1022
1023                 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
1024                 if (error) {
1025                         device_printf(dev, "cannot alloc Tx descriptors for "
1026                             "queue %d error %d\n", q, error);
1027                         return (error);
1028                 }
1029                 txr->vxtxr_txd =
1030                     (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;
1031
1032                 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
1033                 if (error) {
1034                         device_printf(dev, "cannot alloc Tx comp descriptors "
1035                            "for queue %d error %d\n", q, error);
1036                         return (error);
1037                 }
1038                 txc->vxcr_u.txcd =
1039                     (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;
1040
1041                 for (i = 0; i < txr->vxtxr_ndesc; i++) {
1042                         error = bus_dmamap_create(txr->vxtxr_txtag, 0,
1043                             &txr->vxtxr_txbuf[i].vtxb_dmamap);
1044                         if (error) {
1045                                 device_printf(dev, "unable to create Tx buf "
1046                                     "dmamap for queue %d idx %d\n", q, i);
1047                                 return (error);
1048                         }
1049                 }
1050         }
1051
1052         return (0);
1053 }
1054
1055 static void
1056 vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
1057 {
1058         device_t dev;
1059         struct vmxnet3_txqueue *txq;
1060         struct vmxnet3_txring *txr;
1061         struct vmxnet3_comp_ring *txc;
1062         struct vmxnet3_txbuf *txb;
1063         int i, q;
1064
1065         dev = sc->vmx_dev;
1066
1067         for (q = 0; q < sc->vmx_ntxqueues; q++) {
1068                 txq = &sc->vmx_txq[q];
1069                 txr = &txq->vxtxq_cmd_ring;
1070                 txc = &txq->vxtxq_comp_ring;
1071
1072                 for (i = 0; i < txr->vxtxr_ndesc; i++) {
1073                         txb = &txr->vxtxr_txbuf[i];
1074                         if (txb->vtxb_dmamap != NULL) {
1075                                 bus_dmamap_destroy(txr->vxtxr_txtag,
1076                                     txb->vtxb_dmamap);
1077                                 txb->vtxb_dmamap = NULL;
1078                         }
1079                 }
1080
1081                 if (txc->vxcr_u.txcd != NULL) {
1082                         vmxnet3_dma_free(sc, &txc->vxcr_dma);
1083                         txc->vxcr_u.txcd = NULL;
1084                 }
1085
1086                 if (txr->vxtxr_txd != NULL) {
1087                         vmxnet3_dma_free(sc, &txr->vxtxr_dma);
1088                         txr->vxtxr_txd = NULL;
1089                 }
1090
1091                 if (txr->vxtxr_txtag != NULL) {
1092                         bus_dma_tag_destroy(txr->vxtxr_txtag);
1093                         txr->vxtxr_txtag = NULL;
1094                 }
1095         }
1096 }
1097
1098 static int
1099 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
1100 {
1101         device_t dev;
1102         struct vmxnet3_rxqueue *rxq;
1103         struct vmxnet3_rxring *rxr;
1104         struct vmxnet3_comp_ring *rxc;
1105         int descsz, compsz;
1106         int i, j, q, error;
1107
1108         dev = sc->vmx_dev;
1109
1110         for (q = 0; q < sc->vmx_nrxqueues; q++) {
1111                 rxq = &sc->vmx_rxq[q];
1112                 rxc = &rxq->vxrxq_comp_ring;
1113                 compsz = 0;
1114
1115                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1116                         rxr = &rxq->vxrxq_cmd_ring[i];
1117
1118                         descsz = rxr->vxrxr_ndesc *
1119                             sizeof(struct vmxnet3_rxdesc);
1120                         compsz += rxr->vxrxr_ndesc *
1121                             sizeof(struct vmxnet3_rxcompdesc);
1122
1123                         error = bus_dma_tag_create(bus_get_dma_tag(dev),
1124                             1, 0,               /* alignment, boundary */
1125                             BUS_SPACE_MAXADDR,  /* lowaddr */
1126                             BUS_SPACE_MAXADDR,  /* highaddr */
1127                             NULL, NULL,         /* filter, filterarg */
1128                             MJUMPAGESIZE,       /* maxsize */
1129                             1,                  /* nsegments */
1130                             MJUMPAGESIZE,       /* maxsegsize */
1131                             0,                  /* flags */
1132                             NULL, NULL,         /* lockfunc, lockarg */
1133                             &rxr->vxrxr_rxtag);
1134                         if (error) {
1135                                 device_printf(dev,
1136                                     "unable to create Rx buffer tag for "
1137                                     "queue %d\n", q);
1138                                 return (error);
1139                         }
1140
1141                         error = vmxnet3_dma_malloc(sc, descsz, 512,
1142                             &rxr->vxrxr_dma);
1143                         if (error) {
1144                                 device_printf(dev, "cannot allocate Rx "
1145                                     "descriptors for queue %d/%d error %d\n",
1146                                     i, q, error);
1147                                 return (error);
1148                         }
1149                         rxr->vxrxr_rxd =
1150                             (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
1151                 }
1152
1153                 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
1154                 if (error) {
1155                         device_printf(dev, "cannot alloc Rx comp descriptors "
1156                             "for queue %d error %d\n", q, error);
1157                         return (error);
1158                 }
1159                 rxc->vxcr_u.rxcd =
1160                     (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;
1161
1162                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1163                         rxr = &rxq->vxrxq_cmd_ring[i];
1164
1165                         error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1166                             &rxr->vxrxr_spare_dmap);
1167                         if (error) {
1168                                 device_printf(dev, "unable to create spare "
1169                                     "dmamap for queue %d/%d error %d\n",
1170                                     q, i, error);
1171                                 return (error);
1172                         }
1173
1174                         for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1175                                 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1176                                     &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
1177                                 if (error) {
1178                                         device_printf(dev, "unable to create "
1179                                             "dmamap for queue %d/%d slot %d "
1180                                             "error %d\n",
1181                                             q, i, j, error);
1182                                         return (error);
1183                                 }
1184                         }
1185                 }
1186         }
1187
1188         return (0);
1189 }
1190
1191 static void
1192 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
1193 {
1194         device_t dev;
1195         struct vmxnet3_rxqueue *rxq;
1196         struct vmxnet3_rxring *rxr;
1197         struct vmxnet3_comp_ring *rxc;
1198         struct vmxnet3_rxbuf *rxb;
1199         int i, j, q;
1200
1201         dev = sc->vmx_dev;
1202
1203         for (q = 0; q < sc->vmx_nrxqueues; q++) {
1204                 rxq = &sc->vmx_rxq[q];
1205                 rxc = &rxq->vxrxq_comp_ring;
1206
1207                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1208                         rxr = &rxq->vxrxq_cmd_ring[i];
1209
1210                         if (rxr->vxrxr_spare_dmap != NULL) {
1211                                 bus_dmamap_destroy(rxr->vxrxr_rxtag,
1212                                     rxr->vxrxr_spare_dmap);
1213                                 rxr->vxrxr_spare_dmap = NULL;
1214                         }
1215
1216                         for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1217                                 rxb = &rxr->vxrxr_rxbuf[j];
1218                                 if (rxb->vrxb_dmamap != NULL) {
1219                                         bus_dmamap_destroy(rxr->vxrxr_rxtag,
1220                                             rxb->vrxb_dmamap);
1221                                         rxb->vrxb_dmamap = NULL;
1222                                 }
1223                         }
1224                 }
1225
1226                 if (rxc->vxcr_u.rxcd != NULL) {
1227                         vmxnet3_dma_free(sc, &rxc->vxcr_dma);
1228                         rxc->vxcr_u.rxcd = NULL;
1229                 }
1230
1231                 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1232                         rxr = &rxq->vxrxq_cmd_ring[i];
1233
1234                         if (rxr->vxrxr_rxd != NULL) {
1235                                 vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
1236                                 rxr->vxrxr_rxd = NULL;
1237                         }
1238
1239                         if (rxr->vxrxr_rxtag != NULL) {
1240                                 bus_dma_tag_destroy(rxr->vxrxr_rxtag);
1241                                 rxr->vxrxr_rxtag = NULL;
1242                         }
1243                 }
1244         }
1245 }
1246
1247 static int
1248 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
1249 {
1250         int error;
1251
1252         error = vmxnet3_alloc_txq_data(sc);
1253         if (error)
1254                 return (error);
1255
1256         error = vmxnet3_alloc_rxq_data(sc);
1257         if (error)
1258                 return (error);
1259
1260         return (0);
1261 }
1262
1263 static void
1264 vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
1265 {
1266
1267         if (sc->vmx_rxq != NULL)
1268                 vmxnet3_free_rxq_data(sc);
1269
1270         if (sc->vmx_txq != NULL)
1271                 vmxnet3_free_txq_data(sc);
1272 }
1273
1274 static int
1275 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1276 {
1277         int error;
1278
1279         error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
1280             32, &sc->vmx_mcast_dma);
1281         if (error)
1282                 device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1283         else
1284                 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;
1285
1286         return (error);
1287 }
1288
1289 static void
1290 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1291 {
1292
1293         if (sc->vmx_mcast != NULL) {
1294                 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
1295                 sc->vmx_mcast = NULL;
1296         }
1297 }
1298
1299 static void
1300 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1301 {
1302         struct vmxnet3_driver_shared *ds;
1303         struct vmxnet3_txqueue *txq;
1304         struct vmxnet3_txq_shared *txs;
1305         struct vmxnet3_rxqueue *rxq;
1306         struct vmxnet3_rxq_shared *rxs;
1307         int i;
1308
1309         ds = sc->vmx_ds;
1310
1311         /*
1312          * Initialize fields of the shared data that remains the same across
1313          * reinits. Note the shared data is zero'd when allocated.
1314          */
1315
1316         ds->magic = VMXNET3_REV1_MAGIC;
1317
1318         /* DriverInfo */
1319         ds->version = VMXNET3_DRIVER_VERSION;
1320         ds->guest = VMXNET3_GOS_FREEBSD |
1321 #ifdef __LP64__
1322             VMXNET3_GOS_64BIT;
1323 #else
1324             VMXNET3_GOS_32BIT;
1325 #endif
1326         ds->vmxnet3_revision = 1;
1327         ds->upt_version = 1;
1328
1329         /* Misc. conf */
1330         ds->driver_data = vtophys(sc);
1331         ds->driver_data_len = sizeof(struct vmxnet3_softc);
1332         ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
1333         ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
1334         ds->nrxsg_max = sc->vmx_max_rxsegs;
1335
1336         /* Interrupt control. */
1337         ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1338         ds->nintr = sc->vmx_nintrs;
1339         ds->evintr = sc->vmx_event_intr_idx;
1340         ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1341
1342         for (i = 0; i < sc->vmx_nintrs; i++)
1343                 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1344
1345         /* Receive filter. */
1346         ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
1347         ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;
1348
1349         /* Tx queues */
1350         for (i = 0; i < sc->vmx_ntxqueues; i++) {
1351                 txq = &sc->vmx_txq[i];
1352                 txs = txq->vxtxq_ts;
1353
1354                 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
1355                 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1356                 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
1357                 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1358                 txs->driver_data = vtophys(txq);
1359                 txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1360         }
1361
1362         /* Rx queues */
1363         for (i = 0; i < sc->vmx_nrxqueues; i++) {
1364                 rxq = &sc->vmx_rxq[i];
1365                 rxs = rxq->vxrxq_rs;
1366
1367                 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
1368                 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1369                 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
1370                 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1371                 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
1372                 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1373                 rxs->driver_data = vtophys(rxq);
1374                 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1375         }
1376 }
1377
1378 static void
1379 vmxnet3_reinit_interface(struct vmxnet3_softc *sc)
1380 {
1381         struct ifnet *ifp;
1382
1383         ifp = sc->vmx_ifp;
1384
1385         /* Use the current MAC address. */
1386         bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN);
1387         vmxnet3_set_lladdr(sc);
1388
1389         ifp->if_hwassist = 0;
1390         if (ifp->if_capenable & IFCAP_TXCSUM)
1391                 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD;
1392         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1393                 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6;
1394         if (ifp->if_capenable & IFCAP_TSO4)
1395                 ifp->if_hwassist |= CSUM_TSO;
1396         if (ifp->if_capenable & IFCAP_TSO6)
1397                 ifp->if_hwassist |= CSUM_TSO; /* No CSUM_TSO_IPV6. */
1398 }
1399
1400 static void
1401 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1402 {
1403         struct ifnet *ifp;
1404         struct vmxnet3_driver_shared *ds;
1405
1406         ifp = sc->vmx_ifp;
1407         ds = sc->vmx_ds;
1408
1409         ds->upt_features = 0;
1410         if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
1411                 ds->upt_features |= UPT1_F_CSUM;
1412         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1413                 ds->upt_features |= UPT1_F_VLAN;
1414         if (ifp->if_capenable & IFCAP_LRO)
1415                 ds->upt_features |= UPT1_F_LRO;
1416
1417         ds->mtu = ifp->if_mtu;
1418         ds->ntxqueue = sc->vmx_ntxqueues;
1419         ds->nrxqueue = sc->vmx_nrxqueues;
1420
1421         vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
1422         vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1423             (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
1424 }
1425
1426 static int
1427 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1428 {
1429         int error;
1430
1431         error = vmxnet3_alloc_shared_data(sc);
1432         if (error)
1433                 return (error);
1434
1435         error = vmxnet3_alloc_queue_data(sc);
1436         if (error)
1437                 return (error);
1438
1439         error = vmxnet3_alloc_mcast_table(sc);
1440         if (error)
1441                 return (error);
1442
1443         vmxnet3_init_shared_data(sc);
1444
1445         return (0);
1446 }
1447
1448 static void
1449 vmxnet3_free_data(struct vmxnet3_softc *sc)
1450 {
1451
1452         vmxnet3_free_mcast_table(sc);
1453         vmxnet3_free_queue_data(sc);
1454         vmxnet3_free_shared_data(sc);
1455 }
1456
1457 static int
1458 vmxnet3_setup_interface(struct vmxnet3_softc *sc)
1459 {
1460         device_t dev;
1461         struct ifnet *ifp;
1462
1463         dev = sc->vmx_dev;
1464
1465         ifp = sc->vmx_ifp = if_alloc(IFT_ETHER);
1466         if (ifp == NULL) {
1467                 device_printf(dev, "cannot allocate ifnet structure\n");
1468                 return (ENOSPC);
1469         }
1470
1471         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1472 #if __FreeBSD_version < 1000025
1473         ifp->if_baudrate = 1000000000;
1474 #else
1475         if_initbaudrate(ifp, IF_Gbps(10));
1476 #endif
1477         ifp->if_softc = sc;
1478         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1479         ifp->if_init = vmxnet3_init;
1480         ifp->if_ioctl = vmxnet3_ioctl;
1481         ifp->if_start = vmxnet3_start;
1482         ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1;
1483         IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1);
1484         IFQ_SET_READY(&ifp->if_snd);
1485
1486         vmxnet3_get_lladdr(sc);
1487         ether_ifattach(ifp, sc->vmx_lladdr);
1488
1489         ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM;
1490         ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6;
1491         ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
1492         ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
1493             IFCAP_VLAN_HWCSUM;
1494         ifp->if_capenable = ifp->if_capabilities;
1495
1496         /* These capabilities are not enabled by default. */
1497         ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER;
1498
1499         sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
1500             vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
1501         sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config,
1502             vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
1503
1504         ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change,
1505             vmxnet3_media_status);
1506         ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1507         ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO);
1508
1509         return (0);
1510 }
1511
1512 static void
1513 vmxnet3_evintr(struct vmxnet3_softc *sc)
1514 {
1515         device_t dev;
1516         struct ifnet *ifp;
1517         struct vmxnet3_txq_shared *ts;
1518         struct vmxnet3_rxq_shared *rs;
1519         uint32_t event;
1520         int reset;
1521
1522         dev = sc->vmx_dev;
1523         ifp = sc->vmx_ifp;
1524         reset = 0;
1525
1526         VMXNET3_CORE_LOCK(sc);
1527
1528         /* Clear events. */
1529         event = sc->vmx_ds->event;
1530         vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
1531
1532         if (event & VMXNET3_EVENT_LINK)
1533                 vmxnet3_link_status(sc);
1534
1535         if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
1536                 reset = 1;
1537                 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
1538                 ts = sc->vmx_txq[0].vxtxq_ts;
1539                 if (ts->stopped != 0)
1540                         device_printf(dev, "Tx queue error %#x\n", ts->error);
1541                 rs = sc->vmx_rxq[0].vxrxq_rs;
1542                 if (rs->stopped != 0)
1543                         device_printf(dev, "Rx queue error %#x\n", rs->error);
1544                 device_printf(dev, "Rx/Tx queue error event ... resetting\n");
1545         }
1546
1547         if (event & VMXNET3_EVENT_DIC)
1548                 device_printf(dev, "device implementation change event\n");
1549         if (event & VMXNET3_EVENT_DEBUG)
1550                 device_printf(dev, "debug event\n");
1551
1552         if (reset != 0) {
1553                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1554                 vmxnet3_init_locked(sc);
1555         }
1556
1557         VMXNET3_CORE_UNLOCK(sc);
1558 }
1559
1560 static void
1561 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq)
1562 {
1563         struct vmxnet3_softc *sc;
1564         struct ifnet *ifp;
1565         struct vmxnet3_txring *txr;
1566         struct vmxnet3_comp_ring *txc;
1567         struct vmxnet3_txcompdesc *txcd;
1568         struct vmxnet3_txbuf *txb;
1569         u_int sop;
1570
1571         sc = txq->vxtxq_sc;
1572         ifp = sc->vmx_ifp;
1573         txr = &txq->vxtxq_cmd_ring;
1574         txc = &txq->vxtxq_comp_ring;
1575
1576         VMXNET3_TXQ_LOCK_ASSERT(txq);
1577
1578         for (;;) {
1579                 txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
1580                 if (txcd->gen != txc->vxcr_gen)
1581                         break;
1582                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1583
1584                 if (++txc->vxcr_next == txc->vxcr_ndesc) {
1585                         txc->vxcr_next = 0;
1586                         txc->vxcr_gen ^= 1;
1587                 }
1588
1589                 sop = txr->vxtxr_next;
1590                 txb = &txr->vxtxr_txbuf[sop];
1591
1592                 if (txb->vtxb_m != NULL) {
1593                         bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
1594                             BUS_DMASYNC_POSTWRITE);
1595                         bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
1596
1597                         m_freem(txb->vtxb_m);
1598                         txb->vtxb_m = NULL;
1599
1600                         ifp->if_opackets++;
1601                 }
1602
1603                 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
1604         }
1605
1606         if (txr->vxtxr_head == txr->vxtxr_next)
1607                 txq->vxtxq_watchdog = 0;
1608 }
1609
1610 static int
1611 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr)
1612 {
1613         struct ifnet *ifp;
1614         struct mbuf *m;
1615         struct vmxnet3_rxdesc *rxd;
1616         struct vmxnet3_rxbuf *rxb;
1617         bus_dma_tag_t tag;
1618         bus_dmamap_t dmap;
1619         bus_dma_segment_t segs[1];
1620         int idx, clsize, btype, flags, nsegs, error;
1621
1622         ifp = sc->vmx_ifp;
1623         tag = rxr->vxrxr_rxtag;
1624         dmap = rxr->vxrxr_spare_dmap;
1625         idx = rxr->vxrxr_fill;
1626         rxd = &rxr->vxrxr_rxd[idx];
1627         rxb = &rxr->vxrxr_rxbuf[idx];
1628
1629 #ifdef VMXNET3_FAILPOINTS
1630         KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS);
1631         if (rxr->vxrxr_rid != 0)
1632                 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS);
1633 #endif
1634
1635         if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) {
1636                 flags = M_PKTHDR;
1637                 clsize = MCLBYTES;
1638                 btype = VMXNET3_BTYPE_HEAD;
1639         } else {
1640 #if __FreeBSD_version < 902001
1641                 /*
1642                  * These mbufs will never be used for the start of a frame.
1643                  * Roughly prior to branching releng/9.2, the load_mbuf_sg()
1644                  * required the mbuf to always be a packet header. Avoid
1645                  * unnecessary mbuf initialization in newer versions where
1646                  * that is not the case.
1647                  */
1648                 flags = M_PKTHDR;
1649 #else
1650                 flags = 0;
1651 #endif
1652                 clsize = MJUMPAGESIZE;
1653                 btype = VMXNET3_BTYPE_BODY;
1654         }
1655
1656         m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize);
1657         if (m == NULL) {
1658                 sc->vmx_stats.vmst_mgetcl_failed++;
1659                 return (ENOBUFS);
1660         }
1661
1662         if (btype == VMXNET3_BTYPE_HEAD) {
1663                 m->m_len = m->m_pkthdr.len = clsize;
1664                 m_adj(m, ETHER_ALIGN);
1665         } else
1666                 m->m_len = clsize;
1667
1668         error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs,
1669             BUS_DMA_NOWAIT);
1670         if (error) {
1671                 m_freem(m);
1672                 sc->vmx_stats.vmst_mbuf_load_failed++;
1673                 return (error);
1674         }
1675         KASSERT(nsegs == 1,
1676             ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
1677 #if __FreeBSD_version < 902001
1678         if (btype == VMXNET3_BTYPE_BODY)
1679                 m->m_flags &= ~M_PKTHDR;
1680 #endif
1681
1682         if (rxb->vrxb_m != NULL) {
1683                 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD);
1684                 bus_dmamap_unload(tag, rxb->vrxb_dmamap);
1685         }
1686
1687         rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
1688         rxb->vrxb_dmamap = dmap;
1689         rxb->vrxb_m = m;
1690
1691         rxd->addr = segs[0].ds_addr;
1692         rxd->len = segs[0].ds_len;
1693         rxd->btype = btype;
1694         rxd->gen = rxr->vxrxr_gen;
1695
1696         vmxnet3_rxr_increment_fill(rxr);
1697         return (0);
1698 }
1699
1700 static void
1701 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
1702     struct vmxnet3_rxring *rxr, int idx)
1703 {
1704         struct vmxnet3_rxdesc *rxd;
1705
1706         rxd = &rxr->vxrxr_rxd[idx];
1707         rxd->gen = rxr->vxrxr_gen;
1708         vmxnet3_rxr_increment_fill(rxr);
1709 }
1710
1711 static void
1712 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
1713 {
1714         struct vmxnet3_softc *sc;
1715         struct vmxnet3_rxring *rxr;
1716         struct vmxnet3_comp_ring *rxc;
1717         struct vmxnet3_rxcompdesc *rxcd;
1718         int idx, eof;
1719
1720         sc = rxq->vxrxq_sc;
1721         rxc = &rxq->vxrxq_comp_ring;
1722
1723         do {
1724                 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
1725                 if (rxcd->gen != rxc->vxcr_gen)
1726                         break;          /* Not expected. */
1727                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1728
1729                 if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
1730                         rxc->vxcr_next = 0;
1731                         rxc->vxcr_gen ^= 1;
1732                 }
1733
1734                 idx = rxcd->rxd_idx;
1735                 eof = rxcd->eop;
1736                 if (rxcd->qid < sc->vmx_nrxqueues)
1737                         rxr = &rxq->vxrxq_cmd_ring[0];
1738                 else
1739                         rxr = &rxq->vxrxq_cmd_ring[1];
1740                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
1741         } while (!eof);
1742 }
1743
1744 static void
1745 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
1746 {
1747
1748         if (rxcd->ipv4) {
1749                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
1750                 if (rxcd->ipcsum_ok)
1751                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1752         }
1753
1754         if (!rxcd->fragment) {
1755                 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) {
1756                         m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
1757                             CSUM_PSEUDO_HDR;
1758                         m->m_pkthdr.csum_data = 0xFFFF;
1759                 }
1760         }
1761 }
1762
1763 static void
1764 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
1765     struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
1766 {
1767         struct vmxnet3_softc *sc;
1768         struct ifnet *ifp;
1769
1770         sc = rxq->vxrxq_sc;
1771         ifp = sc->vmx_ifp;
1772
1773         if (rxcd->error) {
1774                 ifp->if_ierrors++;
1775                 m_freem(m);
1776                 return;
1777         }
1778
1779         if (!rxcd->no_csum)
1780                 vmxnet3_rx_csum(rxcd, m);
1781         if (rxcd->vlan) {
1782                 m->m_flags |= M_VLANTAG;
1783                 m->m_pkthdr.ether_vtag = rxcd->vtag;
1784         }
1785
1786         ifp->if_ipackets++;
1787         VMXNET3_RXQ_UNLOCK(rxq);
1788         (*ifp->if_input)(ifp, m);
1789         VMXNET3_RXQ_LOCK(rxq);
1790 }
1791
1792 static void
1793 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq)
1794 {
1795         struct vmxnet3_softc *sc;
1796         struct ifnet *ifp;
1797         struct vmxnet3_rxring *rxr;
1798         struct vmxnet3_comp_ring *rxc;
1799         struct vmxnet3_rxdesc *rxd;
1800         struct vmxnet3_rxcompdesc *rxcd;
1801         struct mbuf *m, *m_head, *m_tail;
1802         int idx, length;
1803
1804         sc = rxq->vxrxq_sc;
1805         ifp = sc->vmx_ifp;
1806         rxc = &rxq->vxrxq_comp_ring;
1807         m_head = m_tail = NULL;
1808
1809         VMXNET3_RXQ_LOCK_ASSERT(rxq);
1810
1811         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1812                 return;
1813
1814         for (;;) {
1815                 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
1816                 if (rxcd->gen != rxc->vxcr_gen)
1817                         break;
1818                 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1819
1820                 if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
1821                         rxc->vxcr_next = 0;
1822                         rxc->vxcr_gen ^= 1;
1823                 }
1824
1825                 idx = rxcd->rxd_idx;
1826                 length = rxcd->len;
1827                 if (rxcd->qid < sc->vmx_nrxqueues)
1828                         rxr = &rxq->vxrxq_cmd_ring[0];
1829                 else
1830                         rxr = &rxq->vxrxq_cmd_ring[1];
1831                 rxd = &rxr->vxrxr_rxd[idx];
1832
1833                 m = rxr->vxrxr_rxbuf[idx].vrxb_m;
1834                 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf",
1835                     __func__, rxcd->qid, idx));
1836
1837                 /*
1838                  * The host may skip descriptors. We detect this when this
1839                  * descriptor does not match the previous fill index. Catch
1840                  * up with the host now.
1841                  */
1842                 if (__predict_false(rxr->vxrxr_fill != idx)) {
1843                         while (rxr->vxrxr_fill != idx) {
1844                                 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
1845                                     rxr->vxrxr_gen;
1846                                 vmxnet3_rxr_increment_fill(rxr);
1847                         }
1848                 }
1849
1850                 if (rxcd->sop) {
1851                         KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD,
1852                             ("%s: start of frame w/o head buffer", __func__));
1853                         KASSERT(rxr == &rxq->vxrxq_cmd_ring[0],
1854                             ("%s: start of frame not in ring 0", __func__));
1855                         KASSERT((idx % sc->vmx_rx_max_chain) == 0,
1856                             ("%s: start of frame at unexcepted index %d (%d)",
1857                              __func__, idx, sc->vmx_rx_max_chain));
1858                         KASSERT(m_head == NULL,
1859                             ("%s: duplicate start of frame?", __func__));
1860
1861                         if (length == 0) {
1862                                 /* Just ignore this descriptor. */
1863                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
1864                                 goto nextp;
1865                         }
1866
1867                         if (vmxnet3_newbuf(sc, rxr) != 0) {
1868                                 ifp->if_iqdrops++;
1869                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
1870                                 if (!rxcd->eop)
1871                                         vmxnet3_rxq_discard_chain(rxq);
1872                                 goto nextp;
1873                         }
1874
1875                         m->m_pkthdr.rcvif = ifp;
1876                         m->m_pkthdr.len = m->m_len = length;
1877                         m->m_pkthdr.csum_flags = 0;
1878                         m_head = m_tail = m;
1879
1880                 } else {
1881                         KASSERT(rxd->btype == VMXNET3_BTYPE_BODY,
1882                             ("%s: non start of frame w/o body buffer", __func__));
1883                         KASSERT(m_head != NULL,
1884                             ("%s: frame not started?", __func__));
1885
1886                         if (vmxnet3_newbuf(sc, rxr) != 0) {
1887                                 ifp->if_iqdrops++;
1888                                 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
1889                                 if (!rxcd->eop)
1890                                         vmxnet3_rxq_discard_chain(rxq);
1891                                 m_freem(m_head);
1892                                 m_head = m_tail = NULL;
1893                                 goto nextp;
1894                         }
1895
1896                         m->m_len = length;
1897                         m_head->m_pkthdr.len += length;
1898                         m_tail->m_next = m;
1899                         m_tail = m;
1900                 }
1901
1902                 if (rxcd->eop) {
1903                         vmxnet3_rxq_input(rxq, rxcd, m_head);
1904                         m_head = m_tail = NULL;
1905
1906                         /* Must recheck after dropping the Rx lock. */
1907                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1908                                 break;
1909                 }
1910
1911 nextp:
1912                 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
1913                         int qid = rxcd->qid;
1914                         bus_size_t r;
1915
1916                         idx = (idx + 1) % rxr->vxrxr_ndesc;
1917                         if (qid >= sc->vmx_nrxqueues) {
1918                                 qid -= sc->vmx_nrxqueues;
1919                                 r = VMXNET3_BAR0_RXH2(qid);
1920                         } else
1921                                 r = VMXNET3_BAR0_RXH1(qid);
1922                         vmxnet3_write_bar0(sc, r, idx);
1923                 }
1924         }
1925 }
1926
1927 static void
1928 vmxnet3_legacy_intr(void *xsc)
1929 {
1930         struct vmxnet3_softc *sc;
1931         struct vmxnet3_rxqueue *rxq;
1932         struct vmxnet3_txqueue *txq;
1933         struct ifnet *ifp;
1934
1935         sc = xsc;
1936         rxq = &sc->vmx_rxq[0];
1937         txq = &sc->vmx_txq[0];
1938         ifp = sc->vmx_ifp;
1939
1940         if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
1941                 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
1942                         return;
1943         }
1944         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1945                 vmxnet3_disable_all_intrs(sc);
1946
1947         if (sc->vmx_ds->event != 0)
1948                 vmxnet3_evintr(sc);
1949
1950         VMXNET3_RXQ_LOCK(rxq);
1951         vmxnet3_rxq_eof(rxq);
1952         VMXNET3_RXQ_UNLOCK(rxq);
1953
1954         VMXNET3_TXQ_LOCK(txq);
1955         vmxnet3_txq_eof(txq);
1956         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1957                 vmxnet3_start_locked(ifp);
1958         VMXNET3_TXQ_UNLOCK(txq);
1959
1960         vmxnet3_enable_all_intrs(sc);
1961 }
1962
1963 static void
1964 vmxnet3_txq_intr(void *xtxq)
1965 {
1966         struct vmxnet3_softc *sc;
1967         struct vmxnet3_txqueue *txq;
1968         struct ifnet *ifp;
1969
1970         txq = xtxq;
1971         sc = txq->vxtxq_sc;
1972         ifp = sc->vmx_ifp;
1973
1974         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1975                 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx);
1976
1977         VMXNET3_TXQ_LOCK(txq);
1978         vmxnet3_txq_eof(txq);
1979         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1980                 vmxnet3_start_locked(ifp);
1981         VMXNET3_TXQ_UNLOCK(txq);
1982
1983         vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx);
1984 }
1985
1986 static void
1987 vmxnet3_rxq_intr(void *xrxq)
1988 {
1989         struct vmxnet3_softc *sc;
1990         struct vmxnet3_rxqueue *rxq;
1991
1992         rxq = xrxq;
1993         sc = rxq->vxrxq_sc;
1994
1995         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1996                 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
1997
1998         VMXNET3_RXQ_LOCK(rxq);
1999         vmxnet3_rxq_eof(rxq);
2000         VMXNET3_RXQ_UNLOCK(rxq);
2001
2002         vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx);
2003 }
2004
2005 static void
2006 vmxnet3_event_intr(void *xsc)
2007 {
2008         struct vmxnet3_softc *sc;
2009
2010         sc = xsc;
2011
2012         if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2013                 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
2014
2015         if (sc->vmx_ds->event != 0)
2016                 vmxnet3_evintr(sc);
2017
2018         vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2019 }
2020
2021 static void
2022 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2023 {
2024         struct vmxnet3_txring *txr;
2025         struct vmxnet3_txbuf *txb;
2026         int i;
2027
2028         txr = &txq->vxtxq_cmd_ring;
2029
2030         for (i = 0; i < txr->vxtxr_ndesc; i++) {
2031                 txb = &txr->vxtxr_txbuf[i];
2032
2033                 if (txb->vtxb_m == NULL)
2034                         continue;
2035
2036                 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
2037                     BUS_DMASYNC_POSTWRITE);
2038                 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
2039                 m_freem(txb->vtxb_m);
2040                 txb->vtxb_m = NULL;
2041         }
2042 }
2043
2044 static void
2045 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2046 {
2047         struct vmxnet3_rxring *rxr;
2048         struct vmxnet3_rxbuf *rxb;
2049         int i, j;
2050
2051         for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
2052                 rxr = &rxq->vxrxq_cmd_ring[i];
2053
2054                 for (j = 0; j < rxr->vxrxr_ndesc; j++) {
2055                         rxb = &rxr->vxrxr_rxbuf[j];
2056
2057                         if (rxb->vrxb_m == NULL)
2058                                 continue;
2059                         bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap,
2060                             BUS_DMASYNC_POSTREAD);
2061                         bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap);
2062                         m_freem(rxb->vrxb_m);
2063                         rxb->vrxb_m = NULL;
2064                 }
2065         }
2066 }
2067
2068 static void
2069 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
2070 {
2071         struct vmxnet3_rxqueue *rxq;
2072         struct vmxnet3_txqueue *txq;
2073         int i;
2074
2075         for (i = 0; i < sc->vmx_nrxqueues; i++) {
2076                 rxq = &sc->vmx_rxq[i];
2077                 VMXNET3_RXQ_LOCK(rxq);
2078                 VMXNET3_RXQ_UNLOCK(rxq);
2079         }
2080
2081         for (i = 0; i < sc->vmx_ntxqueues; i++) {
2082                 txq = &sc->vmx_txq[i];
2083                 VMXNET3_TXQ_LOCK(txq);
2084                 VMXNET3_TXQ_UNLOCK(txq);
2085         }
2086 }
2087
2088 static void
2089 vmxnet3_stop(struct vmxnet3_softc *sc)
2090 {
2091         struct ifnet *ifp;
2092         int q;
2093
2094         ifp = sc->vmx_ifp;
2095         VMXNET3_CORE_LOCK_ASSERT(sc);
2096
2097         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2098         sc->vmx_link_active = 0;
2099         callout_stop(&sc->vmx_tick);
2100
2101         /* Disable interrupts. */
2102         vmxnet3_disable_all_intrs(sc);
2103         vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
2104
2105         vmxnet3_stop_rendezvous(sc);
2106
2107         for (q = 0; q < sc->vmx_ntxqueues; q++)
2108                 vmxnet3_txstop(sc, &sc->vmx_txq[q]);
2109         for (q = 0; q < sc->vmx_nrxqueues; q++)
2110                 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]);
2111
2112         vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
2113 }
2114
2115 static void
2116 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2117 {
2118         struct vmxnet3_txring *txr;
2119         struct vmxnet3_comp_ring *txc;
2120
2121         txr = &txq->vxtxq_cmd_ring;
2122         txr->vxtxr_head = 0;
2123         txr->vxtxr_next = 0;
2124         txr->vxtxr_gen = VMXNET3_INIT_GEN;
2125         bzero(txr->vxtxr_txd,
2126             txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));
2127
2128         txc = &txq->vxtxq_comp_ring;
2129         txc->vxcr_next = 0;
2130         txc->vxcr_gen = VMXNET3_INIT_GEN;
2131         bzero(txc->vxcr_u.txcd,
2132             txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
2133 }
2134
2135 static int
2136 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2137 {
2138         struct ifnet *ifp;
2139         struct vmxnet3_rxring *rxr;
2140         struct vmxnet3_comp_ring *rxc;
2141         int i, populate, idx, frame_size, error;
2142
2143         ifp = sc->vmx_ifp;
2144         frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) +
2145             ifp->if_mtu;
2146
2147         /*
2148          * If the MTU causes us to exceed what a regular sized cluster can
2149          * handle, we allocate a second MJUMPAGESIZE cluster after it in
2150          * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters.
2151          *
2152          * Keep rx_max_chain a divisor of the maximum Rx ring size to make
2153          * our life easier. We do not support changing the ring size after
2154          * the attach.
2155          */
2156         if (frame_size <= MCLBYTES)
2157                 sc->vmx_rx_max_chain = 1;
2158         else
2159                 sc->vmx_rx_max_chain = 2;
2160
2161         /*
2162          * Only populate ring 1 if the configuration will take advantage
2163          * of it. That is either when LRO is enabled or the frame size
2164          * exceeds what ring 0 can contain.
2165          */
2166         if ((ifp->if_capenable & IFCAP_LRO) == 0 &&
2167             frame_size <= MCLBYTES + MJUMPAGESIZE)
2168                 populate = 1;
2169         else
2170                 populate = VMXNET3_RXRINGS_PERQ;
2171
2172         for (i = 0; i < populate; i++) {
2173                 rxr = &rxq->vxrxq_cmd_ring[i];
2174                 rxr->vxrxr_fill = 0;
2175                 rxr->vxrxr_gen = VMXNET3_INIT_GEN;
2176                 bzero(rxr->vxrxr_rxd,
2177                     rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2178
2179                 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
2180                         error = vmxnet3_newbuf(sc, rxr);
2181                         if (error)
2182                                 return (error);
2183                 }
2184         }
2185
2186         for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
2187                 rxr = &rxq->vxrxq_cmd_ring[i];
2188                 rxr->vxrxr_fill = 0;
2189                 rxr->vxrxr_gen = 0;
2190                 bzero(rxr->vxrxr_rxd,
2191                     rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2192         }
2193
2194         rxc = &rxq->vxrxq_comp_ring;
2195         rxc->vxcr_next = 0;
2196         rxc->vxcr_gen = VMXNET3_INIT_GEN;
2197         bzero(rxc->vxcr_u.rxcd,
2198             rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));
2199
2200         return (0);
2201 }
2202
2203 static int
2204 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
2205 {
2206         device_t dev;
2207         int q, error;
2208
2209         dev = sc->vmx_dev;
2210
2211         for (q = 0; q < sc->vmx_ntxqueues; q++)
2212                 vmxnet3_txinit(sc, &sc->vmx_txq[q]);
2213
2214         for (q = 0; q < sc->vmx_nrxqueues; q++) {
2215                 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
2216                 if (error) {
2217                         device_printf(dev, "cannot populate Rx queue %d\n", q);
2218                         return (error);
2219                 }
2220         }
2221
2222         return (0);
2223 }
2224
2225 static int
2226 vmxnet3_enable_device(struct vmxnet3_softc *sc)
2227 {
2228         int q;
2229
2230         if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
2231                 device_printf(sc->vmx_dev, "device enable command failed!\n");
2232                 return (1);
2233         }
2234
2235         /* Reset the Rx queue heads. */
2236         for (q = 0; q < sc->vmx_nrxqueues; q++) {
2237                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
2238                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
2239         }
2240
2241         return (0);
2242 }
2243
2244 static void
2245 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
2246 {
2247         struct ifnet *ifp;
2248
2249         ifp = sc->vmx_ifp;
2250
2251         vmxnet3_set_rxfilter(sc);
2252
2253         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2254                 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter,
2255                     sizeof(sc->vmx_ds->vlan_filter));
2256         else
2257                 bzero(sc->vmx_ds->vlan_filter,
2258                     sizeof(sc->vmx_ds->vlan_filter));
2259         vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
2260 }
2261
2262 static int
2263 vmxnet3_reinit(struct vmxnet3_softc *sc)
2264 {
2265
2266         vmxnet3_reinit_interface(sc);
2267         vmxnet3_reinit_shared_data(sc);
2268
2269         if (vmxnet3_reinit_queues(sc) != 0)
2270                 return (ENXIO);
2271
2272         if (vmxnet3_enable_device(sc) != 0)
2273                 return (ENXIO);
2274
2275         vmxnet3_reinit_rxfilters(sc);
2276
2277         return (0);
2278 }
2279
2280 static void
2281 vmxnet3_init_locked(struct vmxnet3_softc *sc)
2282 {
2283         struct ifnet *ifp;
2284
2285         ifp = sc->vmx_ifp;
2286
2287         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2288                 return;
2289
2290         vmxnet3_stop(sc);
2291
2292         if (vmxnet3_reinit(sc) != 0) {
2293                 vmxnet3_stop(sc);
2294                 return;
2295         }
2296
2297         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2298         vmxnet3_link_status(sc);
2299
2300         vmxnet3_enable_all_intrs(sc);
2301         callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
2302 }
2303
2304 static void
2305 vmxnet3_init(void *xsc)
2306 {
2307         struct vmxnet3_softc *sc;
2308
2309         sc = xsc;
2310
2311         VMXNET3_CORE_LOCK(sc);
2312         vmxnet3_init_locked(sc);
2313         VMXNET3_CORE_UNLOCK(sc);
2314 }
2315
2316 /*
2317  * BMV: Much of this can go away once we finally have offsets in
2318  * the mbuf packet header. Bug andre@.
2319  */
2320 static int
2321 vmxnet3_txq_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start)
2322 {
2323         struct ether_vlan_header *evh;
2324         int offset;
2325
2326         evh = mtod(m, struct ether_vlan_header *);
2327         if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2328                 /* BMV: We should handle nested VLAN tags too. */
2329                 *etype = ntohs(evh->evl_proto);
2330                 offset = sizeof(struct ether_vlan_header);
2331         } else {
2332                 *etype = ntohs(evh->evl_encap_proto);
2333                 offset = sizeof(struct ether_header);
2334         }
2335
2336         switch (*etype) {
2337 #if defined(INET)
2338         case ETHERTYPE_IP: {
2339                 struct ip *ip, iphdr;
2340                 if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
2341                         m_copydata(m, offset, sizeof(struct ip),
2342                             (caddr_t) &iphdr);
2343                         ip = &iphdr;
2344                 } else
2345                         ip = (struct ip *)(m->m_data + offset);
2346                 *proto = ip->ip_p;
2347                 *start = offset + (ip->ip_hl << 2);
2348                 break;
2349         }
2350 #endif
2351 #if defined(INET6)
2352         case ETHERTYPE_IPV6:
2353                 *proto = -1;
2354                 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
2355                 /* Assert the network stack sent us a valid packet. */
2356                 KASSERT(*start > offset,
2357                     ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
2358                     *start, offset, *proto));
2359                 break;
2360 #endif
2361         default:
2362                 return (EINVAL);
2363         }
2364
2365         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2366                 struct tcphdr *tcp, tcphdr;
2367
2368                 if (__predict_false(*proto != IPPROTO_TCP)) {
2369                         /* Likely failed to correctly parse the mbuf. */
2370                         return (EINVAL);
2371                 }
2372
2373                 if (m->m_len < *start + sizeof(struct tcphdr)) {
2374                         m_copydata(m, offset, sizeof(struct tcphdr),
2375                             (caddr_t) &tcphdr);
2376                         tcp = &tcphdr;
2377                 } else
2378                         tcp = (struct tcphdr *)(m->m_data + *start);
2379
2380                 /*
2381                  * For TSO, the size of the protocol header is also
2382                  * included in the descriptor header size.
2383                  */
2384                 *start += (tcp->th_off << 2);
2385         }
2386
2387         return (0);
2388 }
2389
2390 static int
2391 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
2392     bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs)
2393 {
2394         struct vmxnet3_txring *txr;
2395         struct mbuf *m;
2396         bus_dma_tag_t tag;
2397         int maxsegs, error;
2398
2399         txr = &txq->vxtxq_cmd_ring;
2400         m = *m0;
2401         tag = txr->vxtxr_txtag;
2402         maxsegs = VMXNET3_TX_MAXSEGS;
2403
2404         error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
2405         if (error == 0 || error != EFBIG)
2406                 return (error);
2407
2408         m = m_collapse(m, M_NOWAIT, maxsegs);
2409         if (m != NULL) {
2410                 *m0 = m;
2411                 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0);
2412         } else
2413                 error = ENOBUFS;
2414
2415         if (error) {
2416                 m_freem(*m0);
2417                 *m0 = NULL;
2418         } else
2419                 txq->vxtxq_sc->vmx_stats.vmst_collapsed++;
2420
2421         return (error);
2422 }
2423
2424 static void
2425 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
2426 {
2427         struct vmxnet3_txring *txr;
2428
2429         txr = &txq->vxtxq_cmd_ring;
2430         bus_dmamap_unload(txr->vxtxr_txtag, dmap);
2431 }
2432
2433 static int
2434 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
2435 {
2436         struct vmxnet3_softc *sc;
2437         struct ifnet *ifp;
2438         struct vmxnet3_txring *txr;
2439         struct vmxnet3_txdesc *txd, *sop;
2440         struct mbuf *m;
2441         bus_dmamap_t dmap;
2442         bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS];
2443         int i, gen, nsegs, etype, proto, start, error;
2444
2445         sc = txq->vxtxq_sc;
2446         ifp = sc->vmx_ifp;
2447         start = 0;
2448         txd = NULL;
2449         txr = &txq->vxtxq_cmd_ring;
2450         dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
2451
2452         error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs);
2453         if (error)
2454                 return (error);
2455
2456         m = *m0;
2457         M_ASSERTPKTHDR(m);
2458         KASSERT(nsegs <= VMXNET3_TX_MAXSEGS,
2459             ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
2460
2461         if (VMXNET3_TXRING_AVAIL(txr) < nsegs) {
2462                 txq->vxtxq_stats.vtxrs_full++;
2463                 vmxnet3_txq_unload_mbuf(txq, dmap);
2464                 return (ENOSPC);
2465         } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
2466                 error = vmxnet3_txq_offload_ctx(m, &etype, &proto, &start);
2467                 if (error) {
2468                         txq->vxtxq_stats.vtxrs_offload_failed++;
2469                         vmxnet3_txq_unload_mbuf(txq, dmap);
2470                         m_freem(m);
2471                         *m0 = NULL;
2472                         return (error);
2473                 }
2474         }
2475
2476         txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m = *m0;
2477         sop = &txr->vxtxr_txd[txr->vxtxr_head];
2478         gen = txr->vxtxr_gen ^ 1;       /* Owned by cpu (yet) */
2479
2480         for (i = 0; i < nsegs; i++) {
2481                 txd = &txr->vxtxr_txd[txr->vxtxr_head];
2482
2483                 txd->addr = segs[i].ds_addr;
2484                 txd->len = segs[i].ds_len;
2485                 txd->gen = gen;
2486                 txd->dtype = 0;
2487                 txd->offload_mode = VMXNET3_OM_NONE;
2488                 txd->offload_pos = 0;
2489                 txd->hlen = 0;
2490                 txd->eop = 0;
2491                 txd->compreq = 0;
2492                 txd->vtag_mode = 0;
2493                 txd->vtag = 0;
2494
2495                 if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
2496                         txr->vxtxr_head = 0;
2497                         txr->vxtxr_gen ^= 1;
2498                 }
2499                 gen = txr->vxtxr_gen;
2500         }
2501         txd->eop = 1;
2502         txd->compreq = 1;
2503
2504         if (m->m_flags & M_VLANTAG) {
2505                 sop->vtag_mode = 1;
2506                 sop->vtag = m->m_pkthdr.ether_vtag;
2507         }
2508
2509         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2510                 sop->offload_mode = VMXNET3_OM_TSO;
2511                 sop->hlen = start;
2512                 sop->offload_pos = m->m_pkthdr.tso_segsz;
2513         } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
2514             VMXNET3_CSUM_OFFLOAD_IPV6)) {
2515                 sop->offload_mode = VMXNET3_OM_CSUM;
2516                 sop->hlen = start;
2517                 sop->offload_pos = start + m->m_pkthdr.csum_data;
2518         }
2519
2520         /* Finally, change the ownership. */
2521         vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
2522         sop->gen ^= 1;
2523
2524         if (++txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
2525                 txq->vxtxq_ts->npending = 0;
2526                 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
2527                     txr->vxtxr_head);
2528         }
2529
2530         return (0);
2531 }
2532
2533 static void
2534 vmxnet3_start_locked(struct ifnet *ifp)
2535 {
2536         struct vmxnet3_softc *sc;
2537         struct vmxnet3_txqueue *txq;
2538         struct vmxnet3_txring *txr;
2539         struct mbuf *m_head;
2540         int tx, avail;
2541
2542         sc = ifp->if_softc;
2543         txq = &sc->vmx_txq[0];
2544         txr = &txq->vxtxq_cmd_ring;
2545         tx = 0;
2546
2547         VMXNET3_TXQ_LOCK_ASSERT(txq);
2548
2549         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
2550             sc->vmx_link_active == 0)
2551                 return;
2552
2553         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
2554                 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2)
2555                         break;
2556
2557                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
2558                 if (m_head == NULL)
2559                         break;
2560
2561                 /* Assume worse case if this mbuf is the head of a chain. */
2562                 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
2563                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2564                         break;
2565                 }
2566
2567                 if (vmxnet3_txq_encap(txq, &m_head) != 0) {
2568                         if (m_head != NULL)
2569                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2570                         break;
2571                 }
2572
2573                 tx++;
2574                 ETHER_BPF_MTAP(ifp, m_head);
2575         }
2576
2577         if (tx > 0) {
2578                 if (txq->vxtxq_ts->npending > 0) {
2579                         txq->vxtxq_ts->npending = 0;
2580                         vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
2581                             txr->vxtxr_head);
2582                 }
2583                 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
2584         }
2585 }
2586
2587 static void
2588 vmxnet3_start(struct ifnet *ifp)
2589 {
2590         struct vmxnet3_softc *sc;
2591         struct vmxnet3_txqueue *txq;
2592
2593         sc = ifp->if_softc;
2594         txq = &sc->vmx_txq[0];
2595
2596         VMXNET3_TXQ_LOCK(txq);
2597         vmxnet3_start_locked(ifp);
2598         VMXNET3_TXQ_UNLOCK(txq);
2599 }
2600
2601 static void
2602 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
2603 {
2604         struct ifnet *ifp;
2605         int idx, bit;
2606
2607         ifp = sc->vmx_ifp;
2608         idx = (tag >> 5) & 0x7F;
2609         bit = tag & 0x1F;
2610
2611         if (tag == 0 || tag > 4095)
2612                 return;
2613
2614         VMXNET3_CORE_LOCK(sc);
2615
2616         /* Update our private VLAN bitvector. */
2617         if (add)
2618                 sc->vmx_vlan_filter[idx] |= (1 << bit);
2619         else
2620                 sc->vmx_vlan_filter[idx] &= ~(1 << bit);
2621
2622         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
2623                 if (add)
2624                         sc->vmx_ds->vlan_filter[idx] |= (1 << bit);
2625                 else
2626                         sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit);
2627                 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
2628         }
2629
2630         VMXNET3_CORE_UNLOCK(sc);
2631 }
2632
2633 static void
2634 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2635 {
2636
2637         if (ifp->if_softc == arg)
2638                 vmxnet3_update_vlan_filter(arg, 1, tag);
2639 }
2640
2641 static void
2642 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
2643 {
2644
2645         if (ifp->if_softc == arg)
2646                 vmxnet3_update_vlan_filter(arg, 0, tag);
2647 }
2648
2649 static void
2650 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
2651 {
2652         struct ifnet *ifp;
2653         struct vmxnet3_driver_shared *ds;
2654         struct ifmultiaddr *ifma;
2655         u_int mode;
2656
2657         ifp = sc->vmx_ifp;
2658         ds = sc->vmx_ds;
2659
2660         mode = VMXNET3_RXMODE_UCAST;
2661         if (ifp->if_flags & IFF_BROADCAST)
2662                 mode |= VMXNET3_RXMODE_BCAST;
2663         if (ifp->if_flags & IFF_PROMISC)
2664                 mode |= VMXNET3_RXMODE_PROMISC;
2665         if (ifp->if_flags & IFF_ALLMULTI)
2666                 mode |= VMXNET3_RXMODE_ALLMULTI;
2667         else {
2668                 int cnt = 0, overflow = 0;
2669
2670                 if_maddr_rlock(ifp);
2671                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2672                         if (ifma->ifma_addr->sa_family != AF_LINK)
2673                                 continue;
2674                         else if (cnt == VMXNET3_MULTICAST_MAX) {
2675                                 overflow = 1;
2676                                 break;
2677                         }
2678
2679                         bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2680                            &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN);
2681                         cnt++;
2682                 }
2683                 if_maddr_runlock(ifp);
2684
2685                 if (overflow != 0) {
2686                         cnt = 0;
2687                         mode |= VMXNET3_RXMODE_ALLMULTI;
2688                 } else if (cnt > 0)
2689                         mode |= VMXNET3_RXMODE_MCAST;
2690                 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN;
2691         }
2692
2693         ds->rxmode = mode;
2694
2695         vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
2696         vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
2697 }
2698
2699 static int
2700 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu)
2701 {
2702         struct ifnet *ifp;
2703
2704         ifp = sc->vmx_ifp;
2705
2706         if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU)
2707                 return (EINVAL);
2708
2709         ifp->if_mtu = mtu;
2710
2711         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2712                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2713                 vmxnet3_init_locked(sc);
2714         }
2715
2716         return (0);
2717 }
2718
2719 static int
2720 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
2721 {
2722         struct vmxnet3_softc *sc;
2723         struct ifreq *ifr;
2724         int reinit, mask, error;
2725
2726         sc = ifp->if_softc;
2727         ifr = (struct ifreq *) data;
2728         error = 0;
2729
2730         switch (cmd) {
2731         case SIOCSIFMTU:
2732                 if (ifp->if_mtu != ifr->ifr_mtu) {
2733                         VMXNET3_CORE_LOCK(sc);
2734                         error = vmxnet3_change_mtu(sc, ifr->ifr_mtu);
2735                         VMXNET3_CORE_UNLOCK(sc);
2736                 }
2737                 break;
2738
2739         case SIOCSIFFLAGS:
2740                 VMXNET3_CORE_LOCK(sc);
2741                 if (ifp->if_flags & IFF_UP) {
2742                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2743                                 if ((ifp->if_flags ^ sc->vmx_if_flags) &
2744                                     (IFF_PROMISC | IFF_ALLMULTI)) {
2745                                         vmxnet3_set_rxfilter(sc);
2746                                 }
2747                         } else
2748                                 vmxnet3_init_locked(sc);
2749                 } else {
2750                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2751                                 vmxnet3_stop(sc);
2752                 }
2753                 sc->vmx_if_flags = ifp->if_flags;
2754                 VMXNET3_CORE_UNLOCK(sc);
2755                 break;
2756
2757         case SIOCADDMULTI:
2758         case SIOCDELMULTI:
2759                 VMXNET3_CORE_LOCK(sc);
2760                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2761                         vmxnet3_set_rxfilter(sc);
2762                 VMXNET3_CORE_UNLOCK(sc);
2763                 break;
2764
2765         case SIOCSIFMEDIA:
2766         case SIOCGIFMEDIA:
2767                 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd);
2768                 break;
2769
2770         case SIOCSIFCAP:
2771                 VMXNET3_CORE_LOCK(sc);
2772                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2773
2774                 if (mask & IFCAP_TXCSUM)
2775                         ifp->if_capenable ^= IFCAP_TXCSUM;
2776                 if (mask & IFCAP_TXCSUM_IPV6)
2777                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
2778                 if (mask & IFCAP_TSO4)
2779                         ifp->if_capenable ^= IFCAP_TSO4;
2780                 if (mask & IFCAP_TSO6)
2781                         ifp->if_capenable ^= IFCAP_TSO6;
2782
2783                 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO |
2784                     IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) {
2785                         /* Changing these features requires us to reinit. */
2786                         reinit = 1;
2787
2788                         if (mask & IFCAP_RXCSUM)
2789                                 ifp->if_capenable ^= IFCAP_RXCSUM;
2790                         if (mask & IFCAP_RXCSUM_IPV6)
2791                                 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
2792                         if (mask & IFCAP_LRO)
2793                                 ifp->if_capenable ^= IFCAP_LRO;
2794                         if (mask & IFCAP_VLAN_HWTAGGING)
2795                                 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2796                         if (mask & IFCAP_VLAN_HWFILTER)
2797                                 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
2798                 } else
2799                         reinit = 0;
2800
2801                 if (mask & IFCAP_VLAN_HWTSO)
2802                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
2803
2804                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2805                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2806                         vmxnet3_init_locked(sc);
2807                 }
2808
2809                 VMXNET3_CORE_UNLOCK(sc);
2810                 VLAN_CAPABILITIES(ifp);
2811                 break;
2812
2813         default:
2814                 error = ether_ioctl(ifp, cmd, data);
2815                 break;
2816         }
2817
2818         VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc);
2819
2820         return (error);
2821 }
2822
2823 static int
2824 vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
2825 {
2826         struct vmxnet3_softc *sc;
2827
2828         sc = txq->vxtxq_sc;
2829
2830         VMXNET3_TXQ_LOCK(txq);
2831         if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
2832                 VMXNET3_TXQ_UNLOCK(txq);
2833                 return (0);
2834         }
2835         VMXNET3_TXQ_UNLOCK(txq);
2836
2837         if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n",
2838             txq->vxtxq_id);
2839         return (1);
2840 }
2841
2842 static void
2843 vmxnet3_refresh_stats(struct vmxnet3_softc *sc)
2844 {
2845
2846         vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
2847 }
2848
2849 static void
2850 vmxnet3_tick(void *xsc)
2851 {
2852         struct vmxnet3_softc *sc;
2853         struct ifnet *ifp;
2854         int i, timedout;
2855
2856         sc = xsc;
2857         ifp = sc->vmx_ifp;
2858         timedout = 0;
2859
2860         VMXNET3_CORE_LOCK_ASSERT(sc);
2861         vmxnet3_refresh_stats(sc);
2862
2863         for (i = 0; i < sc->vmx_ntxqueues; i++)
2864                 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]);
2865
2866         if (timedout != 0) {
2867                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2868                 vmxnet3_init_locked(sc);
2869         } else
2870                 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
2871 }
2872
2873 static int
2874 vmxnet3_link_is_up(struct vmxnet3_softc *sc)
2875 {
2876         uint32_t status;
2877
2878         /* Also update the link speed while here. */
2879         status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
2880         sc->vmx_link_speed = status >> 16;
2881         return !!(status & 0x1);
2882 }
2883
2884 static void
2885 vmxnet3_link_status(struct vmxnet3_softc *sc)
2886 {
2887         struct ifnet *ifp;
2888         int link;
2889
2890         ifp = sc->vmx_ifp;
2891         link = vmxnet3_link_is_up(sc);
2892
2893         if (link != 0 && sc->vmx_link_active == 0) {
2894                 sc->vmx_link_active = 1;
2895                 if_link_state_change(ifp, LINK_STATE_UP);
2896         } else if (link == 0 && sc->vmx_link_active != 0) {
2897                 sc->vmx_link_active = 0;
2898                 if_link_state_change(ifp, LINK_STATE_DOWN);
2899         }
2900 }
2901
2902 static void
2903 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2904 {
2905         struct vmxnet3_softc *sc;
2906
2907         sc = ifp->if_softc;
2908
2909         ifmr->ifm_active = IFM_ETHER | IFM_AUTO;
2910         ifmr->ifm_status = IFM_AVALID;
2911
2912         VMXNET3_CORE_LOCK(sc);
2913         if (vmxnet3_link_is_up(sc) != 0)
2914                 ifmr->ifm_status |= IFM_ACTIVE;
2915         else
2916                 ifmr->ifm_status |= IFM_NONE;
2917         VMXNET3_CORE_UNLOCK(sc);
2918 }
2919
2920 static int
2921 vmxnet3_media_change(struct ifnet *ifp)
2922 {
2923
2924         /* Ignore. */
2925         return (0);
2926 }
2927
2928 static void
2929 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
2930 {
2931         uint32_t ml, mh;
2932
2933         ml  = sc->vmx_lladdr[0];
2934         ml |= sc->vmx_lladdr[1] << 8;
2935         ml |= sc->vmx_lladdr[2] << 16;
2936         ml |= sc->vmx_lladdr[3] << 24;
2937         vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
2938
2939         mh  = sc->vmx_lladdr[4];
2940         mh |= sc->vmx_lladdr[5] << 8;
2941         vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
2942 }
2943
2944 static void
2945 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
2946 {
2947         uint32_t ml, mh;
2948
2949         ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
2950         mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
2951
2952         sc->vmx_lladdr[0] = ml;
2953         sc->vmx_lladdr[1] = ml >> 8;
2954         sc->vmx_lladdr[2] = ml >> 16;
2955         sc->vmx_lladdr[3] = ml >> 24;
2956         sc->vmx_lladdr[4] = mh;
2957         sc->vmx_lladdr[5] = mh >> 8;
2958 }
2959
2960 static void
2961 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq,
2962     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2963 {
2964         struct sysctl_oid *node, *txsnode;
2965         struct sysctl_oid_list *list, *txslist;
2966         struct vmxnet3_txq_stats *stats;
2967         struct UPT1_TxStats *txstats;
2968         char namebuf[16];
2969
2970         stats = &txq->vxtxq_stats;
2971         txstats = &txq->vxtxq_ts->stats;
2972
2973         snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id);
2974         node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
2975             NULL, "Transmit Queue");
2976         txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node);
2977
2978         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD,
2979             &stats->vtxrs_full, "Tx ring full");
2980         SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD,
2981             &stats->vtxrs_offload_failed, "Tx checksum offload failed");
2982
2983         /*
2984          * Add statistics reported by the host. These are updated once
2985          * per second.
2986          */
2987         txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
2988             NULL, "Host Statistics");
2989         txslist = SYSCTL_CHILDREN(txsnode);
2990         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD,
2991             &txstats->TSO_packets, "TSO packets");
2992         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD,
2993             &txstats->TSO_bytes, "TSO bytes");
2994         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
2995             &txstats->ucast_packets, "Unicast packets");
2996         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
2997             &txstats->ucast_bytes, "Unicast bytes");
2998         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
2999             &txstats->mcast_packets, "Multicast packets");
3000         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3001             &txstats->mcast_bytes, "Multicast bytes");
3002         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD,
3003             &txstats->error, "Errors");
3004         SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD,
3005             &txstats->discard, "Discards");
3006 }
3007
3008 static void
3009 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq,
3010     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3011 {
3012         struct sysctl_oid *node, *rxsnode;
3013         struct sysctl_oid_list *list, *rxslist;
3014         struct vmxnet3_rxq_stats *stats;
3015         struct UPT1_RxStats *rxstats;
3016         char namebuf[16];
3017
3018         stats = &rxq->vxrxq_stats;
3019         rxstats = &rxq->vxrxq_rs->stats;
3020
3021         snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id);
3022         node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3023             NULL, "Receive Queue");
3024         rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node);
3025
3026         /*
3027          * Add statistics reported by the host. These are updated once
3028          * per second.
3029          */
3030         rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3031             NULL, "Host Statistics");
3032         rxslist = SYSCTL_CHILDREN(rxsnode);
3033         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD,
3034             &rxstats->LRO_packets, "LRO packets");
3035         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD,
3036             &rxstats->LRO_bytes, "LRO bytes");
3037         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3038             &rxstats->ucast_packets, "Unicast packets");
3039         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3040             &rxstats->ucast_bytes, "Unicast bytes");
3041         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3042             &rxstats->mcast_packets, "Multicast packets");
3043         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3044             &rxstats->mcast_bytes, "Multicast bytes");
3045         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD,
3046             &rxstats->bcast_packets, "Broadcast packets");
3047         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD,
3048             &rxstats->bcast_bytes, "Broadcast bytes");
3049         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD,
3050             &rxstats->nobuffer, "No buffer");
3051         SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD,
3052             &rxstats->error, "Errors");
3053 }
3054
3055 #ifdef VMXNET3_DEBUG_SYSCTL
3056 static void
3057 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc,
3058     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3059 {
3060         struct sysctl_oid *node;
3061         struct sysctl_oid_list *list;
3062         int i;
3063
3064         for (i = 0; i < sc->vmx_ntxqueues; i++) {
3065                 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i];
3066
3067                 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO,
3068                     "debug", CTLFLAG_RD, NULL, "");
3069                 list = SYSCTL_CHILDREN(node);
3070
3071                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD,
3072                     &txq->vxtxq_cmd_ring.vxtxr_head, 0, "");
3073                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD,
3074                     &txq->vxtxq_cmd_ring.vxtxr_next, 0, "");
3075                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD,
3076                     &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, "");
3077                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD,
3078                     &txq->vxtxq_cmd_ring.vxtxr_gen, 0, "");
3079                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3080                     &txq->vxtxq_comp_ring.vxcr_next, 0, "");
3081                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3082                     &txq->vxtxq_comp_ring.vxcr_ndesc, 0,"");
3083                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3084                     &txq->vxtxq_comp_ring.vxcr_gen, 0, "");
3085         }
3086
3087         for (i = 0; i < sc->vmx_nrxqueues; i++) {
3088                 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i];
3089
3090                 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
3091                     "debug", CTLFLAG_RD, NULL, "");
3092                 list = SYSCTL_CHILDREN(node);
3093
3094                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD,
3095                     &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, "");
3096                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
3097                     &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
3098                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
3099                     &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
3100                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD,
3101                     &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, "");
3102                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
3103                     &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
3104                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
3105                     &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
3106                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3107                     &rxq->vxrxq_comp_ring.vxcr_next, 0, "");
3108                 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3109                     &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
3110                 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3111                     &rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
3112         }
3113 }
3114 #endif
3115
3116 static void
3117 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
3118     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3119 {
3120         int i;
3121
3122         for (i = 0; i < sc->vmx_ntxqueues; i++)
3123                 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child);
3124         for (i = 0; i < sc->vmx_nrxqueues; i++)
3125                 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child);
3126
3127 #ifdef VMXNET3_DEBUG_SYSCTL
3128         vmxnet3_setup_debug_sysctl(sc, ctx, child);
3129 #endif
3130 }
3131
3132 static void
3133 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
3134 {
3135         device_t dev;
3136         struct vmxnet3_statistics *stats;
3137         struct sysctl_ctx_list *ctx;
3138         struct sysctl_oid *tree;
3139         struct sysctl_oid_list *child;
3140
3141         dev = sc->vmx_dev;
3142         ctx = device_get_sysctl_ctx(dev);
3143         tree = device_get_sysctl_tree(dev);
3144         child = SYSCTL_CHILDREN(tree);
3145
3146         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD,
3147             &sc->vmx_ntxqueues, 0, "Number of Tx queues");
3148         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD,
3149             &sc->vmx_nrxqueues, 0, "Number of Rx queues");
3150
3151         stats = &sc->vmx_stats;
3152         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "collapsed", CTLFLAG_RD,
3153             &stats->vmst_collapsed, 0, "Tx mbuf chains collapsed");
3154         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD,
3155             &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed");
3156         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD,
3157             &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed");
3158
3159         vmxnet3_setup_queue_sysctl(sc, ctx, child);
3160 }
3161
3162 static void
3163 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3164 {
3165
3166         bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
3167 }
3168
3169 static uint32_t
3170 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
3171 {
3172
3173         return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
3174 }
3175
3176 static void
3177 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3178 {
3179
3180         bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
3181 }
3182
3183 static void
3184 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3185 {
3186
3187         vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
3188 }
3189
3190 static uint32_t
3191 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3192 {
3193
3194         vmxnet3_write_cmd(sc, cmd);
3195         bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0,
3196             BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
3197         return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
3198 }
3199
3200 static void
3201 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
3202 {
3203
3204         vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
3205 }
3206
3207 static void
3208 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
3209 {
3210
3211         vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
3212 }
3213
3214 static void
3215 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
3216 {
3217         int i;
3218
3219         sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
3220         for (i = 0; i < sc->vmx_nintrs; i++)
3221                 vmxnet3_enable_intr(sc, i);
3222 }
3223
3224 static void
3225 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
3226 {
3227         int i;
3228
3229         sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
3230         for (i = 0; i < sc->vmx_nintrs; i++)
3231                 vmxnet3_disable_intr(sc, i);
3232 }
3233
3234 static void
3235 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3236 {
3237         bus_addr_t *baddr = arg;
3238
3239         if (error == 0)
3240                 *baddr = segs->ds_addr;
3241 }
3242
3243 static int
3244 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
3245     struct vmxnet3_dma_alloc *dma)
3246 {
3247         device_t dev;
3248         int error;
3249
3250         dev = sc->vmx_dev;
3251         bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3252
3253         error = bus_dma_tag_create(bus_get_dma_tag(dev),
3254             align, 0,           /* alignment, bounds */
3255             BUS_SPACE_MAXADDR,  /* lowaddr */
3256             BUS_SPACE_MAXADDR,  /* highaddr */
3257             NULL, NULL,         /* filter, filterarg */
3258             size,               /* maxsize */
3259             1,                  /* nsegments */
3260             size,               /* maxsegsize */
3261             BUS_DMA_ALLOCNOW,   /* flags */
3262             NULL,               /* lockfunc */
3263             NULL,               /* lockfuncarg */
3264             &dma->dma_tag);
3265         if (error) {
3266                 device_printf(dev, "bus_dma_tag_create failed: %d\n", error);
3267                 goto fail;
3268         }
3269
3270         error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
3271             BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
3272         if (error) {
3273                 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error);
3274                 goto fail;
3275         }
3276
3277         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3278             size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
3279         if (error) {
3280                 device_printf(dev, "bus_dmamap_load failed: %d\n", error);
3281                 goto fail;
3282         }
3283
3284         dma->dma_size = size;
3285
3286 fail:
3287         if (error)
3288                 vmxnet3_dma_free(sc, dma);
3289
3290         return (error);
3291 }
3292
3293 static void
3294 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
3295 {
3296
3297         if (dma->dma_tag != NULL) {
3298                 if (dma->dma_map != NULL) {
3299                         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3300                             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3301                         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3302                 }
3303
3304                 if (dma->dma_vaddr != NULL) {
3305                         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
3306                             dma->dma_map);
3307                 }
3308
3309                 bus_dma_tag_destroy(dma->dma_tag);
3310         }
3311         bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3312 }
3313
3314 static int
3315 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def)
3316 {
3317         char path[64];
3318
3319         snprintf(path, sizeof(path),
3320             "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob);
3321         TUNABLE_INT_FETCH(path, &def);
3322
3323         return (def);
3324 }
3325
3326 /*
3327  * Since this is a purely paravirtualized device, we do not have
3328  * to worry about DMA coherency. But at times, we must make sure
3329  * both the compiler and CPU do not reorder memory operations.
3330  */
3331 static inline void
3332 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
3333 {
3334
3335         switch (type) {
3336         case VMXNET3_BARRIER_RD:
3337                 rmb();
3338                 break;
3339         case VMXNET3_BARRIER_WR:
3340                 wmb();
3341                 break;
3342         case VMXNET3_BARRIER_RDWR:
3343                 mb();
3344                 break;
3345         default:
3346                 panic("%s: bad barrier type %d", __func__, type);
3347         }
3348 }