sys/dev/vnic/nicvf_main.c

   1 /*
   2  * Copyright (C) 2015 Cavium Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  *
  28  */
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_inet.h"
  33 #include "opt_inet6.h"
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/bitset.h>
  38 #include <sys/bitstring.h>
  39 #include <sys/bus.h>
  40 #include <sys/endian.h>
  41 #include <sys/kernel.h>
  42 #include <sys/malloc.h>
  43 #include <sys/mbuf.h>
  44 #include <sys/module.h>
  45 #include <sys/rman.h>
  46 #include <sys/pciio.h>
  47 #include <sys/pcpu.h>
  48 #include <sys/proc.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/stdatomic.h>
  52 #include <sys/cpuset.h>
  53 #include <sys/lock.h>
  54 #include <sys/mutex.h>
  55 #include <sys/smp.h>
  56 #include <sys/taskqueue.h>
  57
  58 #include <net/bpf.h>
  59 #include <net/ethernet.h>
  60 #include <net/if.h>
  61 #include <net/if_var.h>
  62 #include <net/if_arp.h>
  63 #include <net/if_dl.h>
  64 #include <net/if_media.h>
  65 #include <net/if_types.h>
  66 #include <net/if_vlan_var.h>
  67
  68 #include <netinet/in.h>
  69 #include <netinet/ip.h>
  70 #include <netinet/if_ether.h>
  71 #include <netinet/tcp_lro.h>
  72
  73 #include <dev/pci/pcireg.h>
  74 #include <dev/pci/pcivar.h>
  75
  76 #include <sys/dnv.h>
  77 #include <sys/nv.h>
  78 #include <sys/iov_schema.h>
  79
  80 #include <machine/bus.h>
  81
  82 #include "thunder_bgx.h"
  83 #include "nic_reg.h"
  84 #include "nic.h"
  85 #include "nicvf_queues.h"
  86
  87 #define VNIC_VF_DEVSTR          "Cavium Thunder NIC Virtual Function Driver"
  88
  89 #define VNIC_VF_REG_RID         PCIR_BAR(PCI_CFG_REG_BAR_NUM)
  90
  91 /* Lock for core interface settings */
  92 #define NICVF_CORE_LOCK_INIT(nic)                               \
  93     sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev))
  94
  95 #define NICVF_CORE_LOCK_DESTROY(nic)                            \
  96     sx_destroy(&(nic)->core_sx)
  97
  98 #define NICVF_CORE_LOCK(nic)            sx_xlock(&(nic)->core_sx)
  99 #define NICVF_CORE_UNLOCK(nic)          sx_xunlock(&(nic)->core_sx)
 100
 101 #define NICVF_CORE_LOCK_ASSERT(nic)     sx_assert(&(nic)->core_sx, SA_XLOCKED)
 102
 103 #define SPEED_10        10
 104 #define SPEED_100       100
 105 #define SPEED_1000      1000
 106 #define SPEED_10000     10000
 107 #define SPEED_40000     40000
 108
 109 MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory");
 110
 111 static int nicvf_probe(device_t);
 112 static int nicvf_attach(device_t);
 113 static int nicvf_detach(device_t);
 114
 115 static device_method_t nicvf_methods[] = {
 116         /* Device interface */
 117         DEVMETHOD(device_probe,         nicvf_probe),
 118         DEVMETHOD(device_attach,        nicvf_attach),
 119         DEVMETHOD(device_detach,        nicvf_detach),
 120
 121         DEVMETHOD_END,
 122 };
 123
 124 static driver_t nicvf_driver = {
 125         "vnic",
 126         nicvf_methods,
 127         sizeof(struct nicvf),
 128 };
 129
 130 static devclass_t nicvf_devclass;
 131
 132 DRIVER_MODULE(vnicvf, pci, nicvf_driver, nicvf_devclass, 0, 0);
 133 MODULE_VERSION(vnicvf, 1);
 134 MODULE_DEPEND(vnicvf, pci, 1, 1, 1);
 135 MODULE_DEPEND(vnicvf, ether, 1, 1, 1);
 136 MODULE_DEPEND(vnicvf, vnicpf, 1, 1, 1);
 137
 138 static int nicvf_allocate_misc_interrupt(struct nicvf *);
 139 static int nicvf_enable_misc_interrupt(struct nicvf *);
 140 static int nicvf_allocate_net_interrupts(struct nicvf *);
 141 static void nicvf_release_all_interrupts(struct nicvf *);
 142 static int nicvf_update_hw_max_frs(struct nicvf *, int);
 143 static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *);
 144 static void nicvf_config_cpi(struct nicvf *);
 145 static int nicvf_rss_init(struct nicvf *);
 146 static int nicvf_init_resources(struct nicvf *);
 147
 148 static int nicvf_setup_ifnet(struct nicvf *);
 149 static int nicvf_setup_ifmedia(struct nicvf *);
 150 static void nicvf_hw_addr_random(uint8_t *);
 151
 152 static int nicvf_if_ioctl(struct ifnet *, u_long, caddr_t);
 153 static void nicvf_if_init(void *);
 154 static void nicvf_if_init_locked(struct nicvf *);
 155 static int nicvf_if_transmit(struct ifnet *, struct mbuf *);
 156 static void nicvf_if_qflush(struct ifnet *);
 157 static uint64_t nicvf_if_getcounter(struct ifnet *, ift_counter);
 158
 159 static int nicvf_stop_locked(struct nicvf *);
 160
 161 static void nicvf_media_status(struct ifnet *, struct ifmediareq *);
 162 static int nicvf_media_change(struct ifnet *);
 163
 164 static void nicvf_tick_stats(void *);
 165
 166 static int
 167 nicvf_probe(device_t dev)
 168 {
 169         uint16_t vendor_id;
 170         uint16_t device_id;
 171
 172         vendor_id = pci_get_vendor(dev);
 173         device_id = pci_get_device(dev);
 174
 175         if (vendor_id != PCI_VENDOR_ID_CAVIUM)
 176                 return (ENXIO);
 177
 178         if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF ||
 179             device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) {
 180                 device_set_desc(dev, VNIC_VF_DEVSTR);
 181                 return (BUS_PROBE_DEFAULT);
 182         }
 183
 184         return (ENXIO);
 185 }
 186
 187 static int
 188 nicvf_attach(device_t dev)
 189 {
 190         struct nicvf *nic;
 191         int rid, qcount;
 192         int err = 0;
 193         uint8_t hwaddr[ETHER_ADDR_LEN];
 194         uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0};
 195
 196         nic = device_get_softc(dev);
 197         nic->dev = dev;
 198         nic->pnicvf = nic;
 199
 200         NICVF_CORE_LOCK_INIT(nic);
 201         /* Enable HW TSO on Pass2 */
 202         if (!pass1_silicon(dev))
 203                 nic->hw_tso = TRUE;
 204
 205         rid = VNIC_VF_REG_RID;
 206         nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 207             RF_ACTIVE);
 208         if (nic->reg_base == NULL) {
 209                 device_printf(dev, "Could not allocate registers memory\n");
 210                 return (ENXIO);
 211         }
 212
 213         qcount = MAX_CMP_QUEUES_PER_QS;
 214         nic->max_queues = qcount;
 215
 216         err = nicvf_set_qset_resources(nic);
 217         if (err != 0)
 218                 goto err_free_res;
 219
 220         /* Check if PF is alive and get MAC address for this VF */
 221         err = nicvf_allocate_misc_interrupt(nic);
 222         if (err != 0)
 223                 goto err_free_res;
 224
 225         NICVF_CORE_LOCK(nic);
 226         err = nicvf_enable_misc_interrupt(nic);
 227         NICVF_CORE_UNLOCK(nic);
 228         if (err != 0)
 229                 goto err_release_intr;
 230
 231         err = nicvf_allocate_net_interrupts(nic);
 232         if (err != 0) {
 233                 device_printf(dev,
 234                     "Could not allocate network interface interrupts\n");
 235                 goto err_free_ifnet;
 236         }
 237
 238         /* If no MAC address was obtained we generate random one */
 239         if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) {
 240                 nicvf_hw_addr_random(hwaddr);
 241                 memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN);
 242                 NICVF_CORE_LOCK(nic);
 243                 nicvf_hw_set_mac_addr(nic, hwaddr);
 244                 NICVF_CORE_UNLOCK(nic);
 245         }
 246
 247         /* Configure CPI alorithm */
 248         nic->cpi_alg = CPI_ALG_NONE;
 249         NICVF_CORE_LOCK(nic);
 250         nicvf_config_cpi(nic);
 251         /* Configure receive side scaling */
 252         if (nic->qs->rq_cnt > 1)
 253                 nicvf_rss_init(nic);
 254         NICVF_CORE_UNLOCK(nic);
 255
 256         err = nicvf_setup_ifnet(nic);
 257         if (err != 0) {
 258                 device_printf(dev, "Could not set-up ifnet\n");
 259                 goto err_release_intr;
 260         }
 261
 262         err = nicvf_setup_ifmedia(nic);
 263         if (err != 0) {
 264                 device_printf(dev, "Could not set-up ifmedia\n");
 265                 goto err_free_ifnet;
 266         }
 267
 268         mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF);
 269         callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0);
 270
 271         ether_ifattach(nic->ifp, nic->hwaddr);
 272
 273         return (0);
 274
 275 err_free_ifnet:
 276         if_free(nic->ifp);
 277 err_release_intr:
 278         nicvf_release_all_interrupts(nic);
 279 err_free_res:
 280         bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base),
 281             nic->reg_base);
 282
 283         return (err);
 284 }
 285
 286 static int
 287 nicvf_detach(device_t dev)
 288 {
 289         struct nicvf *nic;
 290
 291         nic = device_get_softc(dev);
 292
 293         NICVF_CORE_LOCK(nic);
 294         /* Shut down the port and release ring resources */
 295         nicvf_stop_locked(nic);
 296         /* Release stats lock */
 297         mtx_destroy(&nic->stats_mtx);
 298         /* Release interrupts */
 299         nicvf_release_all_interrupts(nic);
 300         /* Release memory resource */
 301         if (nic->reg_base != NULL) {
 302                 bus_release_resource(dev, SYS_RES_MEMORY,
 303                     rman_get_rid(nic->reg_base), nic->reg_base);
 304         }
 305
 306         /* Remove all ifmedia configurations */
 307         ifmedia_removeall(&nic->if_media);
 308         /* Free this ifnet */
 309         if_free(nic->ifp);
 310         NICVF_CORE_UNLOCK(nic);
 311         /* Finally destroy the lock */
 312         NICVF_CORE_LOCK_DESTROY(nic);
 313
 314         return (0);
 315 }
 316
 317 static void
 318 nicvf_hw_addr_random(uint8_t *hwaddr)
 319 {
 320         uint32_t rnd;
 321         uint8_t addr[ETHER_ADDR_LEN];
 322
 323         /*
 324          * Create randomized MAC address.
 325          * Set 'bsd' + random 24 low-order bits.
 326          */
 327         rnd = arc4random() & 0x00ffffff;
 328         addr[0] = 'b';
 329         addr[1] = 's';
 330         addr[2] = 'd';
 331         addr[3] = rnd >> 16;
 332         addr[4] = rnd >> 8;
 333         addr[5] = rnd >> 0;
 334
 335         memcpy(hwaddr, addr, ETHER_ADDR_LEN);
 336 }
 337
 338 static int
 339 nicvf_setup_ifnet(struct nicvf *nic)
 340 {
 341         struct ifnet *ifp;
 342
 343         ifp = if_alloc(IFT_ETHER);
 344         if (ifp == NULL) {
 345                 device_printf(nic->dev, "Could not allocate ifnet structure\n");
 346                 return (ENOMEM);
 347         }
 348
 349         nic->ifp = ifp;
 350
 351         if_setsoftc(ifp, nic);
 352         if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev));
 353         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
 354
 355         if_settransmitfn(ifp, nicvf_if_transmit);
 356         if_setqflushfn(ifp, nicvf_if_qflush);
 357         if_setioctlfn(ifp, nicvf_if_ioctl);
 358         if_setinitfn(ifp, nicvf_if_init);
 359         if_setgetcounterfn(ifp, nicvf_if_getcounter);
 360
 361         if_setmtu(ifp, ETHERMTU);
 362
 363         /* Reset caps */
 364         if_setcapabilities(ifp, 0);
 365
 366         /* Set the default values */
 367         if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU, 0);
 368         if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
 369         if (nic->hw_tso) {
 370                 /* TSO */
 371                 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0);
 372                 /* TSO parameters */
 373                 if_sethwtsomax(ifp, NICVF_TSO_MAXSIZE);
 374                 if_sethwtsomaxsegcount(ifp, NICVF_TSO_NSEGS);
 375                 if_sethwtsomaxsegsize(ifp, MCLBYTES);
 376         }
 377         /* IP/TCP/UDP HW checksums */
 378         if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0);
 379         if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0);
 380         /*
 381          * HW offload enable
 382          */
 383         if_clearhwassist(ifp);
 384         if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0);
 385         if (nic->hw_tso)
 386                 if_sethwassistbits(ifp, (CSUM_TSO), 0);
 387         if_setcapenable(ifp, if_getcapabilities(ifp));
 388
 389         return (0);
 390 }
 391
 392 static int
 393 nicvf_setup_ifmedia(struct nicvf *nic)
 394 {
 395
 396         ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change,
 397             nicvf_media_status);
 398
 399         /*
 400          * Advertise availability of all possible connection types,
 401          * even though not all are possible at the same time.
 402          */
 403
 404         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX),
 405             0, NULL);
 406         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX),
 407             0, NULL);
 408         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX),
 409             0, NULL);
 410         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX),
 411             0, NULL);
 412         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX),
 413             0, NULL);
 414         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX),
 415             0, NULL);
 416
 417         ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX));
 418
 419         return (0);
 420 }
 421
 422 static int
 423 nicvf_if_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 424 {
 425         struct nicvf *nic;
 426         struct rcv_queue *rq;
 427         struct ifreq *ifr;
 428         int mask, err;
 429         int rq_idx;
 430 #if defined(INET) || defined(INET6)
 431         struct ifaddr *ifa;
 432         boolean_t avoid_reset = FALSE;
 433 #endif
 434
 435         nic = if_getsoftc(ifp);
 436         ifr = (struct ifreq *)data;
 437 #if defined(INET) || defined(INET6)
 438         ifa = (struct ifaddr *)data;
 439 #endif
 440         err = 0;
 441         switch (cmd) {
 442         case SIOCSIFADDR:
 443 #ifdef INET
 444                 if (ifa->ifa_addr->sa_family == AF_INET)
 445                         avoid_reset = TRUE;
 446 #endif
 447 #ifdef INET6
 448                 if (ifa->ifa_addr->sa_family == AF_INET6)
 449                         avoid_reset = TRUE;
 450 #endif
 451
 452 #if defined(INET) || defined(INET6)
 453                 /* Avoid reinitialization unless it's necessary */
 454                 if (avoid_reset) {
 455                         if_setflagbits(ifp, IFF_UP, 0);
 456                         if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
 457                                 nicvf_if_init(nic);
 458 #ifdef INET
 459                         if (!(if_getflags(ifp) & IFF_NOARP))
 460                                 arp_ifinit(ifp, ifa);
 461 #endif
 462
 463                         return (0);
 464                 }
 465 #endif
 466                 err = ether_ioctl(ifp, cmd, data);
 467                 break;
 468         case SIOCSIFMTU:
 469                 if (ifr->ifr_mtu < NIC_HW_MIN_FRS ||
 470                     ifr->ifr_mtu > NIC_HW_MAX_FRS) {
 471                         err = EINVAL;
 472                 } else {
 473                         NICVF_CORE_LOCK(nic);
 474                         err = nicvf_update_hw_max_frs(nic, ifr->ifr_mtu);
 475                         if (err == 0)
 476                                 if_setmtu(ifp, ifr->ifr_mtu);
 477                         NICVF_CORE_UNLOCK(nic);
 478                 }
 479                 break;
 480         case SIOCSIFFLAGS:
 481                 NICVF_CORE_LOCK(nic);
 482                 if (if_getflags(ifp) & IFF_UP) {
 483                         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
 484                                 if ((nic->if_flags & if_getflags(ifp)) &
 485                                     IFF_PROMISC) {
 486                                         /* Change promiscous mode */
 487 #if 0
 488                                         /* ARM64TODO */
 489                                         nicvf_set_promiscous(nic);
 490 #endif
 491                                 }
 492
 493                                 if ((nic->if_flags ^ if_getflags(ifp)) &
 494                                     IFF_ALLMULTI) {
 495                                         /* Change multicasting settings */
 496 #if 0
 497                                         /* ARM64TODO */
 498                                         nicvf_set_multicast(nic);
 499 #endif
 500                                 }
 501                         } else {
 502                                 nicvf_if_init_locked(nic);
 503                         }
 504                 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
 505                         nicvf_stop_locked(nic);
 506
 507                 nic->if_flags = if_getflags(ifp);
 508                 NICVF_CORE_UNLOCK(nic);
 509                 break;
 510
 511         case SIOCADDMULTI:
 512         case SIOCDELMULTI:
 513                 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
 514 #if 0
 515                         NICVF_CORE_LOCK(nic);
 516                         /* ARM64TODO */
 517                         nicvf_set_multicast(nic);
 518                         NICVF_CORE_UNLOCK(nic);
 519 #endif
 520                 }
 521                 break;
 522
 523         case SIOCSIFMEDIA:
 524         case SIOCGIFMEDIA:
 525                 err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd);
 526                 break;
 527
 528         case SIOCSIFCAP:
 529                 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap;
 530                 if (mask & IFCAP_VLAN_MTU) {
 531                         /* No work to do except acknowledge the change took. */
 532                         if_togglecapenable(ifp, IFCAP_VLAN_MTU);
 533                 }
 534                 if (mask & IFCAP_TXCSUM)
 535                         if_togglecapenable(ifp, IFCAP_TXCSUM);
 536                 if (mask & IFCAP_RXCSUM)
 537                         if_togglecapenable(ifp, IFCAP_RXCSUM);
 538                 if ((mask & IFCAP_TSO4) && nic->hw_tso)
 539                         if_togglecapenable(ifp, IFCAP_TSO4);
 540                 if (mask & IFCAP_LRO) {
 541                         /*
 542                          * Lock the driver for a moment to avoid
 543                          * mismatch in per-queue settings.
 544                          */
 545                         NICVF_CORE_LOCK(nic);
 546                         if_togglecapenable(ifp, IFCAP_LRO);
 547                         if ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0) {
 548                                 /*
 549                                  * Now disable LRO for subsequent packets.
 550                                  * Atomicity of this change is not necessary
 551                                  * as we don't need precise toggle of this
 552                                  * feature for all threads processing the
 553                                  * completion queue.
 554                                  */
 555                                 for (rq_idx = 0;
 556                                     rq_idx < nic->qs->rq_cnt; rq_idx++) {
 557                                         rq = &nic->qs->rq[rq_idx];
 558                                         rq->lro_enabled = !rq->lro_enabled;
 559                                 }
 560                         }
 561                         NICVF_CORE_UNLOCK(nic);
 562                 }
 563
 564                 break;
 565
 566         default:
 567                 err = ether_ioctl(ifp, cmd, data);
 568                 break;
 569         }
 570
 571         return (err);
 572 }
 573
 574 static void
 575 nicvf_if_init_locked(struct nicvf *nic)
 576 {
 577         struct queue_set *qs = nic->qs;
 578         struct ifnet *ifp;
 579         int qidx;
 580         int err;
 581         caddr_t if_addr;
 582
 583         NICVF_CORE_LOCK_ASSERT(nic);
 584         ifp = nic->ifp;
 585
 586         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)
 587                 nicvf_stop_locked(nic);
 588
 589         err = nicvf_enable_misc_interrupt(nic);
 590         if (err != 0) {
 591                 if_printf(ifp, "Could not reenable Mbox interrupt\n");
 592                 return;
 593         }
 594
 595         /* Get the latest MAC address */
 596         if_addr = if_getlladdr(ifp);
 597         /* Update MAC address if changed */
 598         if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) {
 599                 memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN);
 600                 nicvf_hw_set_mac_addr(nic, if_addr);
 601         }
 602
 603         /* Initialize the queues */
 604         err = nicvf_init_resources(nic);
 605         if (err != 0)
 606                 goto error;
 607
 608         /* Make sure queue initialization is written */
 609         wmb();
 610
 611         nicvf_reg_write(nic, NIC_VF_INT, ~0UL);
 612         /* Enable Qset err interrupt */
 613         nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
 614
 615         /* Enable completion queue interrupt */
 616         for (qidx = 0; qidx < qs->cq_cnt; qidx++)
 617                 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
 618
 619         /* Enable RBDR threshold interrupt */
 620         for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
 621                 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx);
 622
 623         nic->drv_stats.txq_stop = 0;
 624         nic->drv_stats.txq_wake = 0;
 625
 626         /* Activate network interface */
 627         if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
 628
 629         /* Schedule callout to update stats */
 630         callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic);
 631
 632         return;
 633
 634 error:
 635         /* Something went very wrong. Disable this ifnet for good */
 636         if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 637 }
 638
 639 static void
 640 nicvf_if_init(void *if_softc)
 641 {
 642         struct nicvf *nic = if_softc;
 643
 644         NICVF_CORE_LOCK(nic);
 645         nicvf_if_init_locked(nic);
 646         NICVF_CORE_UNLOCK(nic);
 647 }
 648
 649 static int
 650 nicvf_if_transmit(struct ifnet *ifp, struct mbuf *mbuf)
 651 {
 652         struct nicvf *nic = if_getsoftc(ifp);
 653         struct queue_set *qs = nic->qs;
 654         struct snd_queue *sq;
 655         struct mbuf *mtmp;
 656         int qidx;
 657         int err = 0;
 658
 659
 660         if (__predict_false(qs == NULL)) {
 661                 panic("%s: missing queue set for %s", __func__,
 662                     device_get_nameunit(nic->dev));
 663         }
 664
 665         /* Select queue */
 666         if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE)
 667                 qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt;
 668         else
 669                 qidx = curcpu % qs->sq_cnt;
 670
 671         sq = &qs->sq[qidx];
 672
 673         if (mbuf->m_next != NULL &&
 674             (mbuf->m_pkthdr.csum_flags &
 675             (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)) != 0) {
 676                 if (M_WRITABLE(mbuf) == 0) {
 677                         mtmp = m_dup(mbuf, M_NOWAIT);
 678                         m_freem(mbuf);
 679                         if (mtmp == NULL)
 680                                 return (ENOBUFS);
 681                         mbuf = mtmp;
 682                 }
 683         }
 684
 685         err = drbr_enqueue(ifp, sq->br, mbuf);
 686         if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 687             IFF_DRV_RUNNING) || !nic->link_up || (err != 0)) {
 688                 /*
 689                  * Try to enqueue packet to the ring buffer.
 690                  * If the driver is not active, link down or enqueue operation
 691                  * failed, return with the appropriate error code.
 692                  */
 693                 return (err);
 694         }
 695
 696         if (NICVF_TX_TRYLOCK(sq) != 0) {
 697                 err = nicvf_xmit_locked(sq);
 698                 NICVF_TX_UNLOCK(sq);
 699                 return (err);
 700         } else
 701                 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task);
 702
 703         return (0);
 704 }
 705
 706 static void
 707 nicvf_if_qflush(struct ifnet *ifp)
 708 {
 709         struct nicvf *nic;
 710         struct queue_set *qs;
 711         struct snd_queue *sq;
 712         struct mbuf *mbuf;
 713         size_t idx;
 714
 715         nic = if_getsoftc(ifp);
 716         qs = nic->qs;
 717
 718         for (idx = 0; idx < qs->sq_cnt; idx++) {
 719                 sq = &qs->sq[idx];
 720                 NICVF_TX_LOCK(sq);
 721                 while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL)
 722                         m_freem(mbuf);
 723                 NICVF_TX_UNLOCK(sq);
 724         }
 725         if_qflush(ifp);
 726 }
 727
 728 static uint64_t
 729 nicvf_if_getcounter(struct ifnet *ifp, ift_counter cnt)
 730 {
 731         struct nicvf *nic;
 732         struct nicvf_hw_stats *hw_stats;
 733         struct nicvf_drv_stats *drv_stats;
 734
 735         nic = if_getsoftc(ifp);
 736         hw_stats = &nic->hw_stats;
 737         drv_stats = &nic->drv_stats;
 738
 739         switch (cnt) {
 740         case IFCOUNTER_IPACKETS:
 741                 return (drv_stats->rx_frames_ok);
 742         case IFCOUNTER_OPACKETS:
 743                 return (drv_stats->tx_frames_ok);
 744         case IFCOUNTER_IBYTES:
 745                 return (hw_stats->rx_bytes);
 746         case IFCOUNTER_OBYTES:
 747                 return (hw_stats->tx_bytes_ok);
 748         case IFCOUNTER_IMCASTS:
 749                 return (hw_stats->rx_mcast_frames);
 750         case IFCOUNTER_COLLISIONS:
 751                 return (0);
 752         case IFCOUNTER_IQDROPS:
 753                 return (drv_stats->rx_drops);
 754         case IFCOUNTER_OQDROPS:
 755                 return (drv_stats->tx_drops);
 756         default:
 757                 return (if_get_counter_default(ifp, cnt));
 758         }
 759
 760 }
 761
 762 static void
 763 nicvf_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 764 {
 765         struct nicvf *nic = if_getsoftc(ifp);
 766
 767         NICVF_CORE_LOCK(nic);
 768
 769         ifmr->ifm_status = IFM_AVALID;
 770         ifmr->ifm_active = IFM_ETHER;
 771
 772         if (nic->link_up) {
 773                 /* Device attached to working network */
 774                 ifmr->ifm_status |= IFM_ACTIVE;
 775         }
 776
 777         switch (nic->speed) {
 778         case SPEED_10:
 779                 ifmr->ifm_active |= IFM_10_T;
 780                 break;
 781         case SPEED_100:
 782                 ifmr->ifm_active |= IFM_100_TX;
 783                 break;
 784         case SPEED_1000:
 785                 ifmr->ifm_active |= IFM_1000_T;
 786                 break;
 787         case SPEED_10000:
 788                 ifmr->ifm_active |= IFM_10G_SR;
 789                 break;
 790         case SPEED_40000:
 791                 ifmr->ifm_active |= IFM_40G_CR4;
 792                 break;
 793         default:
 794                 ifmr->ifm_active |= IFM_AUTO;
 795                 break;
 796         }
 797
 798         if (nic->duplex)
 799                 ifmr->ifm_active |= IFM_FDX;
 800         else
 801                 ifmr->ifm_active |= IFM_HDX;
 802
 803         NICVF_CORE_UNLOCK(nic);
 804 }
 805
 806 static int
 807 nicvf_media_change(struct ifnet *ifp __unused)
 808 {
 809
 810         return (0);
 811 }
 812
 813 /* Register read/write APIs */
 814 void
 815 nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val)
 816 {
 817
 818         bus_write_8(nic->reg_base, offset, val);
 819 }
 820
 821 uint64_t
 822 nicvf_reg_read(struct nicvf *nic, uint64_t offset)
 823 {
 824
 825         return (bus_read_8(nic->reg_base, offset));
 826 }
 827
 828 void
 829 nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset,
 830     uint64_t qidx, uint64_t val)
 831 {
 832
 833         bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val);
 834 }
 835
 836 uint64_t
 837 nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset,
 838     uint64_t qidx)
 839 {
 840
 841         return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT)));
 842 }
 843
 844 /* VF -> PF mailbox communication */
 845 static void
 846 nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
 847 {
 848         uint64_t *msg = (uint64_t *)mbx;
 849
 850         nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
 851         nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
 852 }
 853
 854 int
 855 nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
 856 {
 857         int timeout = NIC_MBOX_MSG_TIMEOUT * 10;
 858         int sleep = 2;
 859
 860         NICVF_CORE_LOCK_ASSERT(nic);
 861
 862         nic->pf_acked = FALSE;
 863         nic->pf_nacked = FALSE;
 864
 865         nicvf_write_to_mbx(nic, mbx);
 866
 867         /* Wait for previous message to be acked, timeout 2sec */
 868         while (!nic->pf_acked) {
 869                 if (nic->pf_nacked)
 870                         return (EINVAL);
 871
 872                 DELAY(sleep * 1000);
 873
 874                 if (nic->pf_acked)
 875                         break;
 876                 timeout -= sleep;
 877                 if (!timeout) {
 878                         device_printf(nic->dev,
 879                                    "PF didn't ack to mbox msg %d from VF%d\n",
 880                                    (mbx->msg.msg & 0xFF), nic->vf_id);
 881
 882                         return (EBUSY);
 883                 }
 884         }
 885         return (0);
 886 }
 887
 888 /*
 889  * Checks if VF is able to comminicate with PF
 890  * and also gets the VNIC number this VF is associated to.
 891  */
 892 static int
 893 nicvf_check_pf_ready(struct nicvf *nic)
 894 {
 895         union nic_mbx mbx = {};
 896
 897         mbx.msg.msg = NIC_MBOX_MSG_READY;
 898         if (nicvf_send_msg_to_pf(nic, &mbx)) {
 899                 device_printf(nic->dev,
 900                            "PF didn't respond to READY msg\n");
 901                 return 0;
 902         }
 903
 904         return 1;
 905 }
 906
 907 static void
 908 nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
 909 {
 910
 911         if (bgx->rx)
 912                 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
 913         else
 914                 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
 915 }
 916
 917 static void
 918 nicvf_handle_mbx_intr(struct nicvf *nic)
 919 {
 920         union nic_mbx mbx = {};
 921         uint64_t *mbx_data;
 922         uint64_t mbx_addr;
 923         int i;
 924
 925         mbx_addr = NIC_VF_PF_MAILBOX_0_1;
 926         mbx_data = (uint64_t *)&mbx;
 927
 928         for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
 929                 *mbx_data = nicvf_reg_read(nic, mbx_addr);
 930                 mbx_data++;
 931                 mbx_addr += sizeof(uint64_t);
 932         }
 933
 934         switch (mbx.msg.msg) {
 935         case NIC_MBOX_MSG_READY:
 936                 nic->pf_acked = TRUE;
 937                 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
 938                 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
 939                 nic->node = mbx.nic_cfg.node_id;
 940                 memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN);
 941                 nic->loopback_supported = mbx.nic_cfg.loopback_supported;
 942                 nic->link_up = FALSE;
 943                 nic->duplex = 0;
 944                 nic->speed = 0;
 945                 break;
 946         case NIC_MBOX_MSG_ACK:
 947                 nic->pf_acked = TRUE;
 948                 break;
 949         case NIC_MBOX_MSG_NACK:
 950                 nic->pf_nacked = TRUE;
 951                 break;
 952         case NIC_MBOX_MSG_RSS_SIZE:
 953                 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
 954                 nic->pf_acked = TRUE;
 955                 break;
 956         case NIC_MBOX_MSG_BGX_STATS:
 957                 nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
 958                 nic->pf_acked = TRUE;
 959                 break;
 960         case NIC_MBOX_MSG_BGX_LINK_CHANGE:
 961                 nic->pf_acked = TRUE;
 962                 nic->link_up = mbx.link_status.link_up;
 963                 nic->duplex = mbx.link_status.duplex;
 964                 nic->speed = mbx.link_status.speed;
 965                 if (nic->link_up) {
 966                         if_setbaudrate(nic->ifp, nic->speed * 1000000);
 967                         if_link_state_change(nic->ifp, LINK_STATE_UP);
 968                 } else {
 969                         if_setbaudrate(nic->ifp, 0);
 970                         if_link_state_change(nic->ifp, LINK_STATE_DOWN);
 971                 }
 972                 break;
 973         default:
 974                 device_printf(nic->dev,
 975                            "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
 976                 break;
 977         }
 978         nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
 979 }
 980
 981 static int
 982 nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
 983 {
 984         union nic_mbx mbx = {};
 985
 986         mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
 987         mbx.frs.max_frs = mtu;
 988         mbx.frs.vf_id = nic->vf_id;
 989
 990         return nicvf_send_msg_to_pf(nic, &mbx);
 991 }
 992
 993 static int
 994 nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr)
 995 {
 996         union nic_mbx mbx = {};
 997
 998         mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
 999         mbx.mac.vf_id = nic->vf_id;
1000         memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN);
1001
1002         return (nicvf_send_msg_to_pf(nic, &mbx));
1003 }
1004
1005 static void
1006 nicvf_config_cpi(struct nicvf *nic)
1007 {
1008         union nic_mbx mbx = {};
1009
1010         mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
1011         mbx.cpi_cfg.vf_id = nic->vf_id;
1012         mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
1013         mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
1014
1015         nicvf_send_msg_to_pf(nic, &mbx);
1016 }
1017
1018 static void
1019 nicvf_get_rss_size(struct nicvf *nic)
1020 {
1021         union nic_mbx mbx = {};
1022
1023         mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
1024         mbx.rss_size.vf_id = nic->vf_id;
1025         nicvf_send_msg_to_pf(nic, &mbx);
1026 }
1027
1028 static void
1029 nicvf_config_rss(struct nicvf *nic)
1030 {
1031         union nic_mbx mbx = {};
1032         struct nicvf_rss_info *rss;
1033         int ind_tbl_len;
1034         int i, nextq;
1035
1036         rss = &nic->rss_info;
1037         ind_tbl_len = rss->rss_size;
1038         nextq = 0;
1039
1040         mbx.rss_cfg.vf_id = nic->vf_id;
1041         mbx.rss_cfg.hash_bits = rss->hash_bits;
1042         while (ind_tbl_len != 0) {
1043                 mbx.rss_cfg.tbl_offset = nextq;
1044                 mbx.rss_cfg.tbl_len = MIN(ind_tbl_len,
1045                     RSS_IND_TBL_LEN_PER_MBX_MSG);
1046                 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
1047                     NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
1048
1049                 for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
1050                         mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
1051
1052                 nicvf_send_msg_to_pf(nic, &mbx);
1053
1054                 ind_tbl_len -= mbx.rss_cfg.tbl_len;
1055         }
1056 }
1057
1058 static void
1059 nicvf_set_rss_key(struct nicvf *nic)
1060 {
1061         struct nicvf_rss_info *rss;
1062         uint64_t key_addr;
1063         int idx;
1064
1065         rss = &nic->rss_info;
1066         key_addr = NIC_VNIC_RSS_KEY_0_4;
1067
1068         for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
1069                 nicvf_reg_write(nic, key_addr, rss->key[idx]);
1070                 key_addr += sizeof(uint64_t);
1071         }
1072 }
1073
1074 static int
1075 nicvf_rss_init(struct nicvf *nic)
1076 {
1077         struct nicvf_rss_info *rss;
1078         int idx;
1079
1080         nicvf_get_rss_size(nic);
1081
1082         rss = &nic->rss_info;
1083         if (nic->cpi_alg != CPI_ALG_NONE) {
1084                 rss->enable = FALSE;
1085                 rss->hash_bits = 0;
1086                 return (ENXIO);
1087         }
1088
1089         rss->enable = TRUE;
1090
1091         /* Using the HW reset value for now */
1092         rss->key[0] = 0xFEED0BADFEED0BADUL;
1093         rss->key[1] = 0xFEED0BADFEED0BADUL;
1094         rss->key[2] = 0xFEED0BADFEED0BADUL;
1095         rss->key[3] = 0xFEED0BADFEED0BADUL;
1096         rss->key[4] = 0xFEED0BADFEED0BADUL;
1097
1098         nicvf_set_rss_key(nic);
1099
1100         rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
1101         nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
1102
1103         rss->hash_bits = fls(rss->rss_size) - 1;
1104         for (idx = 0; idx < rss->rss_size; idx++)
1105                 rss->ind_tbl[idx] = idx % nic->rx_queues;
1106
1107         nicvf_config_rss(nic);
1108
1109         return (0);
1110 }
1111
1112 static int
1113 nicvf_init_resources(struct nicvf *nic)
1114 {
1115         int err;
1116         union nic_mbx mbx = {};
1117
1118         mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
1119
1120         /* Enable Qset */
1121         nicvf_qset_config(nic, TRUE);
1122
1123         /* Initialize queues and HW for data transfer */
1124         err = nicvf_config_data_transfer(nic, TRUE);
1125         if (err) {
1126                 device_printf(nic->dev,
1127                     "Failed to alloc/config VF's QSet resources\n");
1128                 return (err);
1129         }
1130
1131         /* Send VF config done msg to PF */
1132         nicvf_write_to_mbx(nic, &mbx);
1133
1134         return (0);
1135 }
1136
1137 static void
1138 nicvf_misc_intr_handler(void *arg)
1139 {
1140         struct nicvf *nic = (struct nicvf *)arg;
1141         uint64_t intr;
1142
1143         intr = nicvf_reg_read(nic, NIC_VF_INT);
1144         /* Check for spurious interrupt */
1145         if (!(intr & NICVF_INTR_MBOX_MASK))
1146                 return;
1147
1148         nicvf_handle_mbx_intr(nic);
1149 }
1150
1151 static int
1152 nicvf_intr_handler(void *arg)
1153 {
1154         struct nicvf *nic;
1155         struct cmp_queue *cq;
1156         int qidx;
1157
1158         cq = (struct cmp_queue *)arg;
1159         nic = cq->nic;
1160         qidx = cq->idx;
1161
1162         /* Disable interrupts */
1163         nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
1164
1165         taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task);
1166
1167         /* Clear interrupt */
1168         nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
1169
1170         return (FILTER_HANDLED);
1171 }
1172
1173 static void
1174 nicvf_rbdr_intr_handler(void *arg)
1175 {
1176         struct nicvf *nic;
1177         struct queue_set *qs;
1178         struct rbdr *rbdr;
1179         int qidx;
1180
1181         nic = (struct nicvf *)arg;
1182
1183         /* Disable RBDR interrupt and schedule softirq */
1184         for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
1185                 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
1186                         continue;
1187                 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
1188
1189                 qs = nic->qs;
1190                 rbdr = &qs->rbdr[qidx];
1191                 taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait);
1192                 /* Clear interrupt */
1193                 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
1194         }
1195 }
1196
1197 static void
1198 nicvf_qs_err_intr_handler(void *arg)
1199 {
1200         struct nicvf *nic = (struct nicvf *)arg;
1201         struct queue_set *qs = nic->qs;
1202
1203         /* Disable Qset err interrupt and schedule softirq */
1204         nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
1205         taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task);
1206         nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
1207
1208 }
1209
1210 static int
1211 nicvf_enable_msix(struct nicvf *nic)
1212 {
1213         struct pci_devinfo *dinfo;
1214         int rid, count;
1215         int ret;
1216
1217         dinfo = device_get_ivars(nic->dev);
1218         rid = dinfo->cfg.msix.msix_table_bar;
1219         nic->msix_table_res =
1220             bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
1221         if (nic->msix_table_res == NULL) {
1222                 device_printf(nic->dev,
1223                     "Could not allocate memory for MSI-X table\n");
1224                 return (ENXIO);
1225         }
1226
1227         count = nic->num_vec = NIC_VF_MSIX_VECTORS;
1228
1229         ret = pci_alloc_msix(nic->dev, &count);
1230         if ((ret != 0) || (count != nic->num_vec)) {
1231                 device_printf(nic->dev,
1232                     "Request for #%d msix vectors failed, error: %d\n",
1233                     nic->num_vec, ret);
1234                 return (ret);
1235         }
1236
1237         nic->msix_enabled = 1;
1238         return (0);
1239 }
1240
1241 static void
1242 nicvf_disable_msix(struct nicvf *nic)
1243 {
1244
1245         if (nic->msix_enabled) {
1246                 pci_release_msi(nic->dev);
1247                 nic->msix_enabled = 0;
1248                 nic->num_vec = 0;
1249         }
1250 }
1251
1252 static void
1253 nicvf_release_all_interrupts(struct nicvf *nic)
1254 {
1255         struct resource *res;
1256         int irq;
1257         int err;
1258
1259         /* Free registered interrupts */
1260         for (irq = 0; irq < nic->num_vec; irq++) {
1261                 res = nic->msix_entries[irq].irq_res;
1262                 if (res == NULL)
1263                         continue;
1264                 /* Teardown interrupt first */
1265                 if (nic->msix_entries[irq].handle != NULL) {
1266                         err = bus_teardown_intr(nic->dev,
1267                             nic->msix_entries[irq].irq_res,
1268                             nic->msix_entries[irq].handle);
1269                         KASSERT(err == 0,
1270                             ("ERROR: Unable to teardown interrupt %d", irq));
1271                         nic->msix_entries[irq].handle = NULL;
1272                 }
1273
1274                 bus_release_resource(nic->dev, SYS_RES_IRQ,
1275                             rman_get_rid(res), nic->msix_entries[irq].irq_res);
1276                 nic->msix_entries[irq].irq_res = NULL;
1277         }
1278         /* Disable MSI-X */
1279         nicvf_disable_msix(nic);
1280 }
1281
1282 /*
1283  * Initialize MSIX vectors and register MISC interrupt.
1284  * Send READY message to PF to check if its alive
1285  */
1286 static int
1287 nicvf_allocate_misc_interrupt(struct nicvf *nic)
1288 {
1289         struct resource *res;
1290         int irq, rid;
1291         int ret = 0;
1292
1293         /* Return if mailbox interrupt is already registered */
1294         if (nic->msix_enabled)
1295                 return (0);
1296
1297         /* Enable MSI-X */
1298         if (nicvf_enable_msix(nic) != 0)
1299                 return (ENXIO);
1300
1301         irq = NICVF_INTR_ID_MISC;
1302         rid = irq + 1;
1303         nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
1304             SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
1305         if (nic->msix_entries[irq].irq_res == NULL) {
1306                 device_printf(nic->dev,
1307                     "Could not allocate Mbox interrupt for VF%d\n",
1308                     device_get_unit(nic->dev));
1309                 return (ENXIO);
1310         }
1311
1312         ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
1313             (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic,
1314             &nic->msix_entries[irq].handle);
1315         if (ret != 0) {
1316                 res = nic->msix_entries[irq].irq_res;
1317                 bus_release_resource(nic->dev, SYS_RES_IRQ,
1318                             rman_get_rid(res), res);
1319                 nic->msix_entries[irq].irq_res = NULL;
1320                 return (ret);
1321         }
1322
1323         return (0);
1324 }
1325
1326 static int
1327 nicvf_enable_misc_interrupt(struct nicvf *nic)
1328 {
1329
1330         /* Enable mailbox interrupt */
1331         nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
1332
1333         /* Check if VF is able to communicate with PF */
1334         if (!nicvf_check_pf_ready(nic)) {
1335                 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1336                 return (ENXIO);
1337         }
1338
1339         return (0);
1340 }
1341
1342 static void
1343 nicvf_release_net_interrupts(struct nicvf *nic)
1344 {
1345         struct resource *res;
1346         int irq;
1347         int err;
1348
1349         for_each_cq_irq(irq) {
1350                 res = nic->msix_entries[irq].irq_res;
1351                 if (res == NULL)
1352                         continue;
1353                 /* Teardown active interrupts first */
1354                 if (nic->msix_entries[irq].handle != NULL) {
1355                         err = bus_teardown_intr(nic->dev,
1356                             nic->msix_entries[irq].irq_res,
1357                             nic->msix_entries[irq].handle);
1358                         KASSERT(err == 0,
1359                             ("ERROR: Unable to teardown CQ interrupt %d",
1360                             (irq - NICVF_INTR_ID_CQ)));
1361                         if (err != 0)
1362                                 continue;
1363                 }
1364
1365                 /* Release resource */
1366                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
1367                     res);
1368                 nic->msix_entries[irq].irq_res = NULL;
1369         }
1370
1371         for_each_rbdr_irq(irq) {
1372                 res = nic->msix_entries[irq].irq_res;
1373                 if (res == NULL)
1374                         continue;
1375                 /* Teardown active interrupts first */
1376                 if (nic->msix_entries[irq].handle != NULL) {
1377                         err = bus_teardown_intr(nic->dev,
1378                             nic->msix_entries[irq].irq_res,
1379                             nic->msix_entries[irq].handle);
1380                         KASSERT(err == 0,
1381                             ("ERROR: Unable to teardown RDBR interrupt %d",
1382                             (irq - NICVF_INTR_ID_RBDR)));
1383                         if (err != 0)
1384                                 continue;
1385                 }
1386
1387                 /* Release resource */
1388                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
1389                     res);
1390                 nic->msix_entries[irq].irq_res = NULL;
1391         }
1392
1393         irq = NICVF_INTR_ID_QS_ERR;
1394         res = nic->msix_entries[irq].irq_res;
1395         if (res != NULL) {
1396                 /* Teardown active interrupts first */
1397                 if (nic->msix_entries[irq].handle != NULL) {
1398                         err = bus_teardown_intr(nic->dev,
1399                             nic->msix_entries[irq].irq_res,
1400                             nic->msix_entries[irq].handle);
1401                         KASSERT(err == 0,
1402                             ("ERROR: Unable to teardown QS Error interrupt %d",
1403                             irq));
1404                         if (err != 0)
1405                                 return;
1406                 }
1407
1408                 /* Release resource */
1409                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
1410                     res);
1411                 nic->msix_entries[irq].irq_res = NULL;
1412         }
1413 }
1414
1415 static int
1416 nicvf_allocate_net_interrupts(struct nicvf *nic)
1417 {
1418         u_int cpuid;
1419         int irq, rid;
1420         int qidx;
1421         int ret = 0;
1422
1423         /* MSI-X must be configured by now */
1424         if (!nic->msix_enabled) {
1425                 device_printf(nic->dev, "Cannot alloacte queue interrups. "
1426                     "MSI-X interrupts disabled.\n");
1427                 return (ENXIO);
1428         }
1429
1430         /* Register CQ interrupts */
1431         for_each_cq_irq(irq) {
1432                 if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt))
1433                         break;
1434
1435                 qidx = irq - NICVF_INTR_ID_CQ;
1436                 rid = irq + 1;
1437                 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
1438                     SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
1439                 if (nic->msix_entries[irq].irq_res == NULL) {
1440                         device_printf(nic->dev,
1441                             "Could not allocate CQ interrupt %d for VF%d\n",
1442                             (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
1443                         ret = ENXIO;
1444                         goto error;
1445                 }
1446                 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
1447                     (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler,
1448                     NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle);
1449                 if (ret != 0) {
1450                         device_printf(nic->dev,
1451                             "Could not setup CQ interrupt %d for VF%d\n",
1452                             (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
1453                         goto error;
1454                 }
1455                 cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx;
1456                 cpuid %= mp_ncpus;
1457                 /*
1458                  * Save CPU ID for later use when system-wide RSS is enabled.
1459                  * It will be used to pit the CQ task to the same CPU that got
1460                  * interrupted.
1461                  */
1462                 nic->qs->cq[qidx].cmp_cpuid = cpuid;
1463                 if (bootverbose) {
1464                         device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n",
1465                             qidx, cpuid);
1466                 }
1467                 /* Bind interrupts to the given CPU */
1468                 bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid);
1469         }
1470
1471         /* Register RBDR interrupt */
1472         for_each_rbdr_irq(irq) {
1473                 if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt))
1474                         break;
1475
1476                 rid = irq + 1;
1477                 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
1478                     SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
1479                 if (nic->msix_entries[irq].irq_res == NULL) {
1480                         device_printf(nic->dev,
1481                             "Could not allocate RBDR interrupt %d for VF%d\n",
1482                             (irq - NICVF_INTR_ID_RBDR),
1483                             device_get_unit(nic->dev));
1484                         ret = ENXIO;
1485                         goto error;
1486                 }
1487                 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
1488                     (INTR_MPSAFE | INTR_TYPE_NET), NULL,
1489                     nicvf_rbdr_intr_handler, nic,
1490                     &nic->msix_entries[irq].handle);
1491                 if (ret != 0) {
1492                         device_printf(nic->dev,
1493                             "Could not setup RBDR interrupt %d for VF%d\n",
1494                             (irq - NICVF_INTR_ID_RBDR),
1495                             device_get_unit(nic->dev));
1496                         goto error;
1497                 }
1498         }
1499
1500         /* Register QS error interrupt */
1501         irq = NICVF_INTR_ID_QS_ERR;
1502         rid = irq + 1;
1503         nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
1504             SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
1505         if (nic->msix_entries[irq].irq_res == NULL) {
1506                 device_printf(nic->dev,
1507                     "Could not allocate QS Error interrupt for VF%d\n",
1508                     device_get_unit(nic->dev));
1509                 ret = ENXIO;
1510                 goto error;
1511         }
1512         ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
1513             (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler,
1514             nic, &nic->msix_entries[irq].handle);
1515         if (ret != 0) {
1516                 device_printf(nic->dev,
1517                     "Could not setup QS Error interrupt for VF%d\n",
1518                     device_get_unit(nic->dev));
1519                 goto error;
1520         }
1521
1522         return (0);
1523 error:
1524         nicvf_release_net_interrupts(nic);
1525         return (ret);
1526 }
1527
1528 static int
1529 nicvf_stop_locked(struct nicvf *nic)
1530 {
1531         struct ifnet *ifp;
1532         int qidx;
1533         struct queue_set *qs = nic->qs;
1534         union nic_mbx mbx = {};
1535
1536         NICVF_CORE_LOCK_ASSERT(nic);
1537         /* Stop callout. Can block here since holding SX lock */
1538         callout_drain(&nic->stats_callout);
1539
1540         ifp = nic->ifp;
1541
1542         mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
1543         nicvf_send_msg_to_pf(nic, &mbx);
1544
1545         /* Disable RBDR & QS error interrupts */
1546         for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
1547                 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
1548                 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
1549         }
1550         nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
1551         nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
1552
1553         /* Deactivate network interface */
1554         if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
1555
1556         /* Free resources */
1557         nicvf_config_data_transfer(nic, FALSE);
1558
1559         /* Disable HW Qset */
1560         nicvf_qset_config(nic, FALSE);
1561
1562         /* disable mailbox interrupt */
1563         nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1564
1565         return (0);
1566 }
1567
1568 static void
1569 nicvf_update_stats(struct nicvf *nic)
1570 {
1571         int qidx;
1572         struct nicvf_hw_stats *stats = &nic->hw_stats;
1573         struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
1574         struct queue_set *qs = nic->qs;
1575
1576 #define GET_RX_STATS(reg) \
1577     nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3))
1578 #define GET_TX_STATS(reg) \
1579     nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3))
1580
1581         stats->rx_bytes = GET_RX_STATS(RX_OCTS);
1582         stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
1583         stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST);
1584         stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST);
1585         stats->rx_fcs_errors = GET_RX_STATS(RX_FCS);
1586         stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR);
1587         stats->rx_drop_red = GET_RX_STATS(RX_RED);
1588         stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS);
1589         stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN);
1590         stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS);
1591         stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST);
1592         stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST);
1593         stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST);
1594         stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST);
1595
1596         stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS);
1597         stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST);
1598         stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST);
1599         stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST);
1600         stats->tx_drops = GET_TX_STATS(TX_DROP);
1601
1602         drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok +
1603             stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok;
1604         drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun;
1605         drv_stats->tx_drops = stats->tx_drops;
1606
1607         /* Update RQ and SQ stats */
1608         for (qidx = 0; qidx < qs->rq_cnt; qidx++)
1609                 nicvf_update_rq_stats(nic, qidx);
1610         for (qidx = 0; qidx < qs->sq_cnt; qidx++)
1611                 nicvf_update_sq_stats(nic, qidx);
1612 }
1613
1614 static void
1615 nicvf_tick_stats(void *arg)
1616 {
1617         struct nicvf *nic;
1618
1619         nic = (struct nicvf *)arg;
1620
1621         /* Read the statistics */
1622         nicvf_update_stats(nic);
1623
1624         callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic);
1625 }