sys/dev/oce/oce_if.c

   1 /*-
   2  * Copyright (C) 2013 Emulex
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  *
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * 3. Neither the name of the Emulex Corporation nor the names of its
  16  *    contributors may be used to endorse or promote products derived from
  17  *    this software without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  23  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29  * POSSIBILITY OF SUCH DAMAGE.
  30  *
  31  * Contact Information:
  32  * freebsd-drivers@emulex.com
  33  *
  34  * Emulex
  35  * 3333 Susan Street
  36  * Costa Mesa, CA 92626
  37  */
  38
  39 /* $FreeBSD$ */
  40
  41 #include "opt_inet6.h"
  42 #include "opt_inet.h"
  43
  44 #include "oce_if.h"
  45 #include "oce_user.h"
  46
  47 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO)
  48
  49 /* UE Status Low CSR */
  50 static char *ue_status_low_desc[] = {
  51         "CEV",
  52         "CTX",
  53         "DBUF",
  54         "ERX",
  55         "Host",
  56         "MPU",
  57         "NDMA",
  58         "PTC ",
  59         "RDMA ",
  60         "RXF ",
  61         "RXIPS ",
  62         "RXULP0 ",
  63         "RXULP1 ",
  64         "RXULP2 ",
  65         "TIM ",
  66         "TPOST ",
  67         "TPRE ",
  68         "TXIPS ",
  69         "TXULP0 ",
  70         "TXULP1 ",
  71         "UC ",
  72         "WDMA ",
  73         "TXULP2 ",
  74         "HOST1 ",
  75         "P0_OB_LINK ",
  76         "P1_OB_LINK ",
  77         "HOST_GPIO ",
  78         "MBOX ",
  79         "AXGMAC0",
  80         "AXGMAC1",
  81         "JTAG",
  82         "MPU_INTPEND"
  83 };
  84
  85 /* UE Status High CSR */
  86 static char *ue_status_hi_desc[] = {
  87         "LPCMEMHOST",
  88         "MGMT_MAC",
  89         "PCS0ONLINE",
  90         "MPU_IRAM",
  91         "PCS1ONLINE",
  92         "PCTL0",
  93         "PCTL1",
  94         "PMEM",
  95         "RR",
  96         "TXPB",
  97         "RXPP",
  98         "XAUI",
  99         "TXP",
 100         "ARM",
 101         "IPC",
 102         "HOST2",
 103         "HOST3",
 104         "HOST4",
 105         "HOST5",
 106         "HOST6",
 107         "HOST7",
 108         "HOST8",
 109         "HOST9",
 110         "NETC",
 111         "Unknown",
 112         "Unknown",
 113         "Unknown",
 114         "Unknown",
 115         "Unknown",
 116         "Unknown",
 117         "Unknown",
 118         "Unknown"
 119 };
 120
 121 struct oce_common_cqe_info{
 122         uint8_t vtp:1;
 123         uint8_t l4_cksum_pass:1;
 124         uint8_t ip_cksum_pass:1;
 125         uint8_t ipv6_frame:1;
 126         uint8_t qnq:1;
 127         uint8_t rsvd:3;
 128         uint8_t num_frags;
 129         uint16_t pkt_size;
 130         uint16_t vtag;
 131 };
 132
 133
 134 /* Driver entry points prototypes */
 135 static int  oce_probe(device_t dev);
 136 static int  oce_attach(device_t dev);
 137 static int  oce_detach(device_t dev);
 138 static int  oce_shutdown(device_t dev);
 139 static int  oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
 140 static void oce_init(void *xsc);
 141 static int  oce_multiq_start(struct ifnet *ifp, struct mbuf *m);
 142 static void oce_multiq_flush(struct ifnet *ifp);
 143
 144 /* Driver interrupt routines protypes */
 145 static void oce_intr(void *arg, int pending);
 146 static int  oce_setup_intr(POCE_SOFTC sc);
 147 static int  oce_fast_isr(void *arg);
 148 static int  oce_alloc_intr(POCE_SOFTC sc, int vector,
 149                           void (*isr) (void *arg, int pending));
 150
 151 /* Media callbacks prototypes */
 152 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req);
 153 static int  oce_media_change(struct ifnet *ifp);
 154
 155 /* Transmit routines prototypes */
 156 static int  oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
 157 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
 158 static void oce_process_tx_completion(struct oce_wq *wq);
 159 static int  oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
 160                                  struct oce_wq *wq);
 161
 162 /* Receive routines prototypes */
 163 static int  oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
 164 static int  oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
 165 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
 166 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq);
 167 static uint16_t oce_rq_handler_lro(void *arg);
 168 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2);
 169 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2);
 170 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m);
 171
 172 /* Helper function prototypes in this file */
 173 static int  oce_attach_ifp(POCE_SOFTC sc);
 174 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
 175 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
 176 static int  oce_vid_config(POCE_SOFTC sc);
 177 static void oce_mac_addr_set(POCE_SOFTC sc);
 178 static int  oce_handle_passthrough(struct ifnet *ifp, caddr_t data);
 179 static void oce_local_timer(void *arg);
 180 static void oce_if_deactivate(POCE_SOFTC sc);
 181 static void oce_if_activate(POCE_SOFTC sc);
 182 static void setup_max_queues_want(POCE_SOFTC sc);
 183 static void update_queues_got(POCE_SOFTC sc);
 184 static void process_link_state(POCE_SOFTC sc,
 185                  struct oce_async_cqe_link_state *acqe);
 186 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
 187 static void oce_get_config(POCE_SOFTC sc);
 188 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
 189 static void oce_read_env_variables(POCE_SOFTC sc);
 190
 191
 192 /* IP specific */
 193 #if defined(INET6) || defined(INET)
 194 static int  oce_init_lro(POCE_SOFTC sc);
 195 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
 196 #endif
 197
 198 static device_method_t oce_dispatch[] = {
 199         DEVMETHOD(device_probe, oce_probe),
 200         DEVMETHOD(device_attach, oce_attach),
 201         DEVMETHOD(device_detach, oce_detach),
 202         DEVMETHOD(device_shutdown, oce_shutdown),
 203
 204         DEVMETHOD_END
 205 };
 206
 207 static driver_t oce_driver = {
 208         "oce",
 209         oce_dispatch,
 210         sizeof(OCE_SOFTC)
 211 };
 212 static devclass_t oce_devclass;
 213
 214
 215 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0);
 216 MODULE_DEPEND(oce, pci, 1, 1, 1);
 217 MODULE_DEPEND(oce, ether, 1, 1, 1);
 218 MODULE_VERSION(oce, 1);
 219
 220
 221 /* global vars */
 222 const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
 223
 224 /* Module capabilites and parameters */
 225 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
 226 uint32_t oce_enable_rss = OCE_MODCAP_RSS;
 227 uint32_t oce_rq_buf_size = 2048;
 228
 229 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
 230 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
 231
 232
 233 /* Supported devices table */
 234 static uint32_t supportedDevices[] =  {
 235         (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
 236         (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
 237         (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
 238         (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
 239         (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
 240         (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
 241 };
 242
 243 POCE_SOFTC softc_head = NULL;
 244 POCE_SOFTC softc_tail = NULL;
 245
 246 struct oce_rdma_if *oce_rdma_if = NULL;
 247
 248 /*****************************************************************************
 249  *                      Driver entry points functions                        *
 250  *****************************************************************************/
 251
 252 static int
 253 oce_probe(device_t dev)
 254 {
 255         uint16_t vendor = 0;
 256         uint16_t device = 0;
 257         int i = 0;
 258         char str[256] = {0};
 259         POCE_SOFTC sc;
 260
 261         sc = device_get_softc(dev);
 262         bzero(sc, sizeof(OCE_SOFTC));
 263         sc->dev = dev;
 264
 265         vendor = pci_get_vendor(dev);
 266         device = pci_get_device(dev);
 267
 268         for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
 269                 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
 270                         if (device == (supportedDevices[i] & 0xffff)) {
 271                                 sprintf(str, "%s:%s", "Emulex CNA NIC function",
 272                                         component_revision);
 273                                 device_set_desc_copy(dev, str);
 274
 275                                 switch (device) {
 276                                 case PCI_PRODUCT_BE2:
 277                                         sc->flags |= OCE_FLAGS_BE2;
 278                                         break;
 279                                 case PCI_PRODUCT_BE3:
 280                                         sc->flags |= OCE_FLAGS_BE3;
 281                                         break;
 282                                 case PCI_PRODUCT_XE201:
 283                                 case PCI_PRODUCT_XE201_VF:
 284                                         sc->flags |= OCE_FLAGS_XE201;
 285                                         break;
 286                                 case PCI_PRODUCT_SH:
 287                                         sc->flags |= OCE_FLAGS_SH;
 288                                         break;
 289                                 default:
 290                                         return ENXIO;
 291                                 }
 292                                 return BUS_PROBE_DEFAULT;
 293                         }
 294                 }
 295         }
 296
 297         return ENXIO;
 298 }
 299
 300
 301 static int
 302 oce_attach(device_t dev)
 303 {
 304         POCE_SOFTC sc;
 305         int rc = 0;
 306
 307         sc = device_get_softc(dev);
 308
 309         rc = oce_hw_pci_alloc(sc);
 310         if (rc)
 311                 return rc;
 312
 313         sc->tx_ring_size = OCE_TX_RING_SIZE;
 314         sc->rx_ring_size = OCE_RX_RING_SIZE;
 315         /* receive fragment size should be multiple of 2K */
 316         sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048);
 317         sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
 318         sc->promisc      = OCE_DEFAULT_PROMISCUOUS;
 319
 320         LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
 321         LOCK_CREATE(&sc->dev_lock,  "Device_lock");
 322
 323         /* initialise the hardware */
 324         rc = oce_hw_init(sc);
 325         if (rc)
 326                 goto pci_res_free;
 327
 328         oce_read_env_variables(sc);
 329
 330         oce_get_config(sc);
 331
 332         setup_max_queues_want(sc);
 333
 334         rc = oce_setup_intr(sc);
 335         if (rc)
 336                 goto mbox_free;
 337
 338         rc = oce_queue_init_all(sc);
 339         if (rc)
 340                 goto intr_free;
 341
 342         rc = oce_attach_ifp(sc);
 343         if (rc)
 344                 goto queues_free;
 345
 346 #if defined(INET6) || defined(INET)
 347         rc = oce_init_lro(sc);
 348         if (rc)
 349                 goto ifp_free;
 350 #endif
 351
 352         rc = oce_hw_start(sc);
 353         if (rc)
 354                 goto lro_free;
 355
 356         sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 357                                 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
 358         sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 359                                 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
 360
 361         rc = oce_stats_init(sc);
 362         if (rc)
 363                 goto vlan_free;
 364
 365         oce_add_sysctls(sc);
 366
 367         callout_init(&sc->timer, CALLOUT_MPSAFE);
 368         rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
 369         if (rc)
 370                 goto stats_free;
 371
 372         sc->next =NULL;
 373         if (softc_tail != NULL) {
 374           softc_tail->next = sc;
 375         } else {
 376           softc_head = sc;
 377         }
 378         softc_tail = sc;
 379
 380         return 0;
 381
 382 stats_free:
 383         callout_drain(&sc->timer);
 384         oce_stats_free(sc);
 385 vlan_free:
 386         if (sc->vlan_attach)
 387                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
 388         if (sc->vlan_detach)
 389                 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
 390         oce_hw_intr_disable(sc);
 391 lro_free:
 392 #if defined(INET6) || defined(INET)
 393         oce_free_lro(sc);
 394 ifp_free:
 395 #endif
 396         ether_ifdetach(sc->ifp);
 397         if_free(sc->ifp);
 398 queues_free:
 399         oce_queue_release_all(sc);
 400 intr_free:
 401         oce_intr_free(sc);
 402 mbox_free:
 403         oce_dma_free(sc, &sc->bsmbx);
 404 pci_res_free:
 405         oce_hw_pci_free(sc);
 406         LOCK_DESTROY(&sc->dev_lock);
 407         LOCK_DESTROY(&sc->bmbx_lock);
 408         return rc;
 409
 410 }
 411
 412
 413 static int
 414 oce_detach(device_t dev)
 415 {
 416         POCE_SOFTC sc = device_get_softc(dev);
 417         POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL;
 418
 419         poce_sc_tmp = softc_head;
 420         ppoce_sc_tmp1 = &softc_head;
 421         while (poce_sc_tmp != NULL) {
 422           if (poce_sc_tmp == sc) {
 423             *ppoce_sc_tmp1 = sc->next;
 424             if (sc->next == NULL) {
 425               softc_tail = poce_sc_tmp2;
 426             }
 427             break;
 428           }
 429           poce_sc_tmp2 = poce_sc_tmp;
 430           ppoce_sc_tmp1 = &poce_sc_tmp->next;
 431           poce_sc_tmp = poce_sc_tmp->next;
 432         }
 433
 434         LOCK(&sc->dev_lock);
 435         oce_if_deactivate(sc);
 436         UNLOCK(&sc->dev_lock);
 437
 438         callout_drain(&sc->timer);
 439
 440         if (sc->vlan_attach != NULL)
 441                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
 442         if (sc->vlan_detach != NULL)
 443                 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
 444
 445         ether_ifdetach(sc->ifp);
 446
 447         if_free(sc->ifp);
 448
 449         oce_hw_shutdown(sc);
 450
 451         bus_generic_detach(dev);
 452
 453         return 0;
 454 }
 455
 456
 457 static int
 458 oce_shutdown(device_t dev)
 459 {
 460         int rc;
 461
 462         rc = oce_detach(dev);
 463
 464         return rc;
 465 }
 466
 467
 468 static int
 469 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 470 {
 471         struct ifreq *ifr = (struct ifreq *)data;
 472         POCE_SOFTC sc = ifp->if_softc;
 473         int rc = 0;
 474         uint32_t u;
 475
 476         switch (command) {
 477
 478         case SIOCGIFMEDIA:
 479                 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
 480                 break;
 481
 482         case SIOCSIFMTU:
 483                 if (ifr->ifr_mtu > OCE_MAX_MTU)
 484                         rc = EINVAL;
 485                 else
 486                         ifp->if_mtu = ifr->ifr_mtu;
 487                 break;
 488
 489         case SIOCSIFFLAGS:
 490                 if (ifp->if_flags & IFF_UP) {
 491                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 492                                 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
 493                                 oce_init(sc);
 494                         }
 495                         device_printf(sc->dev, "Interface Up\n");
 496                 } else {
 497                         LOCK(&sc->dev_lock);
 498
 499                         sc->ifp->if_drv_flags &=
 500                             ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 501                         oce_if_deactivate(sc);
 502
 503                         UNLOCK(&sc->dev_lock);
 504
 505                         device_printf(sc->dev, "Interface Down\n");
 506                 }
 507
 508                 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) {
 509                         if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
 510                                 sc->promisc = TRUE;
 511                 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) {
 512                         if (!oce_rxf_set_promiscuous(sc, 0))
 513                                 sc->promisc = FALSE;
 514                 }
 515
 516                 break;
 517
 518         case SIOCADDMULTI:
 519         case SIOCDELMULTI:
 520                 rc = oce_hw_update_multicast(sc);
 521                 if (rc)
 522                         device_printf(sc->dev,
 523                                 "Update multicast address failed\n");
 524                 break;
 525
 526         case SIOCSIFCAP:
 527                 u = ifr->ifr_reqcap ^ ifp->if_capenable;
 528
 529                 if (u & IFCAP_TXCSUM) {
 530                         ifp->if_capenable ^= IFCAP_TXCSUM;
 531                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
 532
 533                         if (IFCAP_TSO & ifp->if_capenable &&
 534                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
 535                                 ifp->if_capenable &= ~IFCAP_TSO;
 536                                 ifp->if_hwassist &= ~CSUM_TSO;
 537                                 if_printf(ifp,
 538                                          "TSO disabled due to -txcsum.\n");
 539                         }
 540                 }
 541
 542                 if (u & IFCAP_RXCSUM)
 543                         ifp->if_capenable ^= IFCAP_RXCSUM;
 544
 545                 if (u & IFCAP_TSO4) {
 546                         ifp->if_capenable ^= IFCAP_TSO4;
 547
 548                         if (IFCAP_TSO & ifp->if_capenable) {
 549                                 if (IFCAP_TXCSUM & ifp->if_capenable)
 550                                         ifp->if_hwassist |= CSUM_TSO;
 551                                 else {
 552                                         ifp->if_capenable &= ~IFCAP_TSO;
 553                                         ifp->if_hwassist &= ~CSUM_TSO;
 554                                         if_printf(ifp,
 555                                             "Enable txcsum first.\n");
 556                                         rc = EAGAIN;
 557                                 }
 558                         } else
 559                                 ifp->if_hwassist &= ~CSUM_TSO;
 560                 }
 561
 562                 if (u & IFCAP_VLAN_HWTAGGING)
 563                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 564
 565                 if (u & IFCAP_VLAN_HWFILTER) {
 566                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 567                         oce_vid_config(sc);
 568                 }
 569 #if defined(INET6) || defined(INET)
 570                 if (u & IFCAP_LRO) {
 571                         ifp->if_capenable ^= IFCAP_LRO;
 572                         if(sc->enable_hwlro) {
 573                                 if(ifp->if_capenable & IFCAP_LRO) {
 574                                         rc = oce_mbox_nic_set_iface_lro_config(sc, 1);
 575                                 }else {
 576                                         rc = oce_mbox_nic_set_iface_lro_config(sc, 0);
 577                                 }
 578                         }
 579                 }
 580 #endif
 581
 582                 break;
 583
 584         case SIOCGPRIVATE_0:
 585                 rc = oce_handle_passthrough(ifp, data);
 586                 break;
 587         default:
 588                 rc = ether_ioctl(ifp, command, data);
 589                 break;
 590         }
 591
 592         return rc;
 593 }
 594
 595
 596 static void
 597 oce_init(void *arg)
 598 {
 599         POCE_SOFTC sc = arg;
 600
 601         LOCK(&sc->dev_lock);
 602
 603         if (sc->ifp->if_flags & IFF_UP) {
 604                 oce_if_deactivate(sc);
 605                 oce_if_activate(sc);
 606         }
 607
 608         UNLOCK(&sc->dev_lock);
 609
 610 }
 611
 612
 613 static int
 614 oce_multiq_start(struct ifnet *ifp, struct mbuf *m)
 615 {
 616         POCE_SOFTC sc = ifp->if_softc;
 617         struct oce_wq *wq = NULL;
 618         int queue_index = 0;
 619         int status = 0;
 620
 621         if (!sc->link_status)
 622                 return ENXIO;
 623
 624         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
 625                 queue_index = m->m_pkthdr.flowid % sc->nwqs;
 626
 627         wq = sc->wq[queue_index];
 628
 629         LOCK(&wq->tx_lock);
 630         status = oce_multiq_transmit(ifp, m, wq);
 631         UNLOCK(&wq->tx_lock);
 632
 633         return status;
 634
 635 }
 636
 637
 638 static void
 639 oce_multiq_flush(struct ifnet *ifp)
 640 {
 641         POCE_SOFTC sc = ifp->if_softc;
 642         struct mbuf     *m;
 643         int i = 0;
 644
 645         for (i = 0; i < sc->nwqs; i++) {
 646                 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
 647                         m_freem(m);
 648         }
 649         if_qflush(ifp);
 650 }
 651
 652
 653
 654 /*****************************************************************************
 655  *                   Driver interrupt routines functions                     *
 656  *****************************************************************************/
 657
 658 static void
 659 oce_intr(void *arg, int pending)
 660 {
 661
 662         POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
 663         POCE_SOFTC sc = ii->sc;
 664         struct oce_eq *eq = ii->eq;
 665         struct oce_eqe *eqe;
 666         struct oce_cq *cq = NULL;
 667         int i, num_eqes = 0;
 668
 669
 670         bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
 671                                  BUS_DMASYNC_POSTWRITE);
 672         do {
 673                 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
 674                 if (eqe->evnt == 0)
 675                         break;
 676                 eqe->evnt = 0;
 677                 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
 678                                         BUS_DMASYNC_POSTWRITE);
 679                 RING_GET(eq->ring, 1);
 680                 num_eqes++;
 681
 682         } while (TRUE);
 683
 684         if (!num_eqes)
 685                 goto eq_arm; /* Spurious */
 686
 687         /* Clear EQ entries, but dont arm */
 688         oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
 689
 690         /* Process TX, RX and MCC. But dont arm CQ*/
 691         for (i = 0; i < eq->cq_valid; i++) {
 692                 cq = eq->cq[i];
 693                 (*cq->cq_handler)(cq->cb_arg);
 694         }
 695
 696         /* Arm all cqs connected to this EQ */
 697         for (i = 0; i < eq->cq_valid; i++) {
 698                 cq = eq->cq[i];
 699                 oce_arm_cq(sc, cq->cq_id, 0, TRUE);
 700         }
 701
 702 eq_arm:
 703         oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
 704
 705         return;
 706 }
 707
 708
 709 static int
 710 oce_setup_intr(POCE_SOFTC sc)
 711 {
 712         int rc = 0, use_intx = 0;
 713         int vector = 0, req_vectors = 0;
 714         int tot_req_vectors, tot_vectors;
 715
 716         if (is_rss_enabled(sc))
 717                 req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
 718         else
 719                 req_vectors = 1;
 720
 721         tot_req_vectors = req_vectors;
 722         if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
 723           if (req_vectors > 1) {
 724             tot_req_vectors += OCE_RDMA_VECTORS;
 725             sc->roce_intr_count = OCE_RDMA_VECTORS;
 726           }
 727         }
 728
 729         if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
 730                 sc->intr_count = req_vectors;
 731                 tot_vectors = tot_req_vectors;
 732                 rc = pci_alloc_msix(sc->dev, &tot_vectors);
 733                 if (rc != 0) {
 734                         use_intx = 1;
 735                         pci_release_msi(sc->dev);
 736                 } else {
 737                   if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
 738                     if (tot_vectors < tot_req_vectors) {
 739                       if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) {
 740                         sc->roce_intr_count = (tot_vectors / 2);
 741                       }
 742                       sc->intr_count = tot_vectors - sc->roce_intr_count;
 743                     }
 744                   } else {
 745                     sc->intr_count = tot_vectors;
 746                   }
 747                   sc->flags |= OCE_FLAGS_USING_MSIX;
 748                 }
 749         } else
 750                 use_intx = 1;
 751
 752         if (use_intx)
 753                 sc->intr_count = 1;
 754
 755         /* Scale number of queues based on intr we got */
 756         update_queues_got(sc);
 757
 758         if (use_intx) {
 759                 device_printf(sc->dev, "Using legacy interrupt\n");
 760                 rc = oce_alloc_intr(sc, vector, oce_intr);
 761                 if (rc)
 762                         goto error;
 763         } else {
 764                 for (; vector < sc->intr_count; vector++) {
 765                         rc = oce_alloc_intr(sc, vector, oce_intr);
 766                         if (rc)
 767                                 goto error;
 768                 }
 769         }
 770
 771         return 0;
 772 error:
 773         oce_intr_free(sc);
 774         return rc;
 775 }
 776
 777
 778 static int
 779 oce_fast_isr(void *arg)
 780 {
 781         POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
 782         POCE_SOFTC sc = ii->sc;
 783
 784         if (ii->eq == NULL)
 785                 return FILTER_STRAY;
 786
 787         oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
 788
 789         taskqueue_enqueue(ii->tq, &ii->task);
 790
 791         ii->eq->intr++;
 792
 793         return FILTER_HANDLED;
 794 }
 795
 796
 797 static int
 798 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
 799 {
 800         POCE_INTR_INFO ii = &sc->intrs[vector];
 801         int rc = 0, rr;
 802
 803         if (vector >= OCE_MAX_EQ)
 804                 return (EINVAL);
 805
 806         /* Set the resource id for the interrupt.
 807          * MSIx is vector + 1 for the resource id,
 808          * INTx is 0 for the resource id.
 809          */
 810         if (sc->flags & OCE_FLAGS_USING_MSIX)
 811                 rr = vector + 1;
 812         else
 813                 rr = 0;
 814         ii->intr_res = bus_alloc_resource_any(sc->dev,
 815                                               SYS_RES_IRQ,
 816                                               &rr, RF_ACTIVE|RF_SHAREABLE);
 817         ii->irq_rr = rr;
 818         if (ii->intr_res == NULL) {
 819                 device_printf(sc->dev,
 820                           "Could not allocate interrupt\n");
 821                 rc = ENXIO;
 822                 return rc;
 823         }
 824
 825         TASK_INIT(&ii->task, 0, isr, ii);
 826         ii->vector = vector;
 827         sprintf(ii->task_name, "oce_task[%d]", ii->vector);
 828         ii->tq = taskqueue_create_fast(ii->task_name,
 829                         M_NOWAIT,
 830                         taskqueue_thread_enqueue,
 831                         &ii->tq);
 832         taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
 833                         device_get_nameunit(sc->dev));
 834
 835         ii->sc = sc;
 836         rc = bus_setup_intr(sc->dev,
 837                         ii->intr_res,
 838                         INTR_TYPE_NET,
 839                         oce_fast_isr, NULL, ii, &ii->tag);
 840         return rc;
 841
 842 }
 843
 844
 845 void
 846 oce_intr_free(POCE_SOFTC sc)
 847 {
 848         int i = 0;
 849
 850         for (i = 0; i < sc->intr_count; i++) {
 851
 852                 if (sc->intrs[i].tag != NULL)
 853                         bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
 854                                                 sc->intrs[i].tag);
 855                 if (sc->intrs[i].tq != NULL)
 856                         taskqueue_free(sc->intrs[i].tq);
 857
 858                 if (sc->intrs[i].intr_res != NULL)
 859                         bus_release_resource(sc->dev, SYS_RES_IRQ,
 860                                                 sc->intrs[i].irq_rr,
 861                                                 sc->intrs[i].intr_res);
 862                 sc->intrs[i].tag = NULL;
 863                 sc->intrs[i].intr_res = NULL;
 864         }
 865
 866         if (sc->flags & OCE_FLAGS_USING_MSIX)
 867                 pci_release_msi(sc->dev);
 868
 869 }
 870
 871
 872
 873 /******************************************************************************
 874 *                         Media callbacks functions                           *
 875 ******************************************************************************/
 876
 877 static void
 878 oce_media_status(struct ifnet *ifp, struct ifmediareq *req)
 879 {
 880         POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc;
 881
 882
 883         req->ifm_status = IFM_AVALID;
 884         req->ifm_active = IFM_ETHER;
 885
 886         if (sc->link_status == 1)
 887                 req->ifm_status |= IFM_ACTIVE;
 888         else
 889                 return;
 890
 891         switch (sc->link_speed) {
 892         case 1: /* 10 Mbps */
 893                 req->ifm_active |= IFM_10_T | IFM_FDX;
 894                 sc->speed = 10;
 895                 break;
 896         case 2: /* 100 Mbps */
 897                 req->ifm_active |= IFM_100_TX | IFM_FDX;
 898                 sc->speed = 100;
 899                 break;
 900         case 3: /* 1 Gbps */
 901                 req->ifm_active |= IFM_1000_T | IFM_FDX;
 902                 sc->speed = 1000;
 903                 break;
 904         case 4: /* 10 Gbps */
 905                 req->ifm_active |= IFM_10G_SR | IFM_FDX;
 906                 sc->speed = 10000;
 907                 break;
 908         case 5: /* 20 Gbps */
 909                 req->ifm_active |= IFM_10G_SR | IFM_FDX;
 910                 sc->speed = 20000;
 911                 break;
 912         case 6: /* 25 Gbps */
 913                 req->ifm_active |= IFM_10G_SR | IFM_FDX;
 914                 sc->speed = 25000;
 915                 break;
 916         case 7: /* 40 Gbps */
 917                 req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
 918                 sc->speed = 40000;
 919                 break;
 920         default:
 921                 sc->speed = 0;
 922                 break;
 923         }
 924
 925         return;
 926 }
 927
 928
 929 int
 930 oce_media_change(struct ifnet *ifp)
 931 {
 932         return 0;
 933 }
 934
 935
 936 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc,
 937                                 struct mbuf *m, boolean_t *os2bmc,
 938                                 struct mbuf **m_new)
 939 {
 940         struct ether_header *eh = NULL;
 941
 942         eh = mtod(m, struct ether_header *);
 943
 944         if (!is_os2bmc_enabled(sc) || *os2bmc) {
 945                 *os2bmc = FALSE;
 946                 goto done;
 947         }
 948         if (!ETHER_IS_MULTICAST(eh->ether_dhost))
 949                 goto done;
 950
 951         if (is_mc_allowed_on_bmc(sc, eh) ||
 952             is_bc_allowed_on_bmc(sc, eh) ||
 953             is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) {
 954                 *os2bmc = TRUE;
 955                 goto done;
 956         }
 957
 958         if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) {
 959                 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
 960                 uint8_t nexthdr = ip6->ip6_nxt;
 961                 if (nexthdr == IPPROTO_ICMPV6) {
 962                         struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1);
 963                         switch (icmp6->icmp6_type) {
 964                         case ND_ROUTER_ADVERT:
 965                                 *os2bmc = is_ipv6_ra_filt_enabled(sc);
 966                                 goto done;
 967                         case ND_NEIGHBOR_ADVERT:
 968                                 *os2bmc = is_ipv6_na_filt_enabled(sc);
 969                                 goto done;
 970                         default:
 971                                 break;
 972                         }
 973                 }
 974         }
 975
 976         if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) {
 977                 struct ip *ip = mtod(m, struct ip *);
 978                 int iphlen = ip->ip_hl << 2;
 979                 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen);
 980                 switch (uh->uh_dport) {
 981                 case DHCP_CLIENT_PORT:
 982                         *os2bmc = is_dhcp_client_filt_enabled(sc);
 983                         goto done;
 984                 case DHCP_SERVER_PORT:
 985                         *os2bmc = is_dhcp_srvr_filt_enabled(sc);
 986                         goto done;
 987                 case NET_BIOS_PORT1:
 988                 case NET_BIOS_PORT2:
 989                         *os2bmc = is_nbios_filt_enabled(sc);
 990                         goto done;
 991                 case DHCPV6_RAS_PORT:
 992                         *os2bmc = is_ipv6_ras_filt_enabled(sc);
 993                         goto done;
 994                 default:
 995                         break;
 996                 }
 997         }
 998 done:
 999         if (*os2bmc) {
1000                 *m_new = m_dup(m, M_NOWAIT);
1001                 if (!*m_new) {
1002                         *os2bmc = FALSE;
1003                         return;
1004                 }
1005                 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL);
1006         }
1007 }
1008
1009
1010
1011 /*****************************************************************************
1012  *                        Transmit routines functions                        *
1013  *****************************************************************************/
1014
1015 static int
1016 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
1017 {
1018         int rc = 0, i, retry_cnt = 0;
1019         bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
1020         struct mbuf *m, *m_temp, *m_new = NULL;
1021         struct oce_wq *wq = sc->wq[wq_index];
1022         struct oce_packet_desc *pd;
1023         struct oce_nic_hdr_wqe *nichdr;
1024         struct oce_nic_frag_wqe *nicfrag;
1025         struct ether_header *eh = NULL;
1026         int num_wqes;
1027         uint32_t reg_value;
1028         boolean_t complete = TRUE;
1029         boolean_t os2bmc = FALSE;
1030
1031         m = *mpp;
1032         if (!m)
1033                 return EINVAL;
1034
1035         if (!(m->m_flags & M_PKTHDR)) {
1036                 rc = ENXIO;
1037                 goto free_ret;
1038         }
1039
1040         /* Don't allow non-TSO packets longer than MTU */
1041         if (!is_tso_pkt(m)) {
1042                 eh = mtod(m, struct ether_header *);
1043                 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE))
1044                          goto free_ret;
1045         }
1046
1047         if(oce_tx_asic_stall_verify(sc, m)) {
1048                 m = oce_insert_vlan_tag(sc, m, &complete);
1049                 if(!m) {
1050                         device_printf(sc->dev, "Insertion unsuccessful\n");
1051                         return 0;
1052                 }
1053
1054         }
1055
1056         /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less
1057          * may cause a transmit stall on that port. So the work-around is to
1058          * pad short packets (<= 32 bytes) to a 36-byte length.
1059         */
1060         if(IS_SH(sc) || IS_XE201(sc) ) {
1061                 if(m->m_pkthdr.len <= 32) {
1062                         char buf[36];
1063                         bzero((void *)buf, 36);
1064                         m_append(m, (36 - m->m_pkthdr.len), buf);
1065                 }
1066         }
1067
1068 tx_start:
1069         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1070                 /* consolidate packet buffers for TSO/LSO segment offload */
1071 #if defined(INET6) || defined(INET)
1072                 m = oce_tso_setup(sc, mpp);
1073 #else
1074                 m = NULL;
1075 #endif
1076                 if (m == NULL) {
1077                         rc = ENXIO;
1078                         goto free_ret;
1079                 }
1080         }
1081
1082
1083         pd = &wq->pckts[wq->pkt_desc_head];
1084
1085 retry:
1086         rc = bus_dmamap_load_mbuf_sg(wq->tag,
1087                                      pd->map,
1088                                      m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
1089         if (rc == 0) {
1090                 num_wqes = pd->nsegs + 1;
1091                 if (IS_BE(sc) || IS_SH(sc)) {
1092                         /*Dummy required only for BE3.*/
1093                         if (num_wqes & 1)
1094                                 num_wqes++;
1095                 }
1096                 if (num_wqes >= RING_NUM_FREE(wq->ring)) {
1097                         bus_dmamap_unload(wq->tag, pd->map);
1098                         return EBUSY;
1099                 }
1100                 atomic_store_rel_int(&wq->pkt_desc_head,
1101                                      (wq->pkt_desc_head + 1) % \
1102                                       OCE_WQ_PACKET_ARRAY_SIZE);
1103                 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
1104                 pd->mbuf = m;
1105
1106                 nichdr =
1107                     RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
1108                 nichdr->u0.dw[0] = 0;
1109                 nichdr->u0.dw[1] = 0;
1110                 nichdr->u0.dw[2] = 0;
1111                 nichdr->u0.dw[3] = 0;
1112
1113                 nichdr->u0.s.complete = complete;
1114                 nichdr->u0.s.mgmt = os2bmc;
1115                 nichdr->u0.s.event = 1;
1116                 nichdr->u0.s.crc = 1;
1117                 nichdr->u0.s.forward = 0;
1118                 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
1119                 nichdr->u0.s.udpcs =
1120                         (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
1121                 nichdr->u0.s.tcpcs =
1122                         (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
1123                 nichdr->u0.s.num_wqe = num_wqes;
1124                 nichdr->u0.s.total_length = m->m_pkthdr.len;
1125
1126                 if (m->m_flags & M_VLANTAG) {
1127                         nichdr->u0.s.vlan = 1; /*Vlan present*/
1128                         nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
1129                 }
1130
1131                 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1132                         if (m->m_pkthdr.tso_segsz) {
1133                                 nichdr->u0.s.lso = 1;
1134                                 nichdr->u0.s.lso_mss  = m->m_pkthdr.tso_segsz;
1135                         }
1136                         if (!IS_BE(sc) || !IS_SH(sc))
1137                                 nichdr->u0.s.ipcs = 1;
1138                 }
1139
1140                 RING_PUT(wq->ring, 1);
1141                 atomic_add_int(&wq->ring->num_used, 1);
1142
1143                 for (i = 0; i < pd->nsegs; i++) {
1144                         nicfrag =
1145                             RING_GET_PRODUCER_ITEM_VA(wq->ring,
1146                                                       struct oce_nic_frag_wqe);
1147                         nicfrag->u0.s.rsvd0 = 0;
1148                         nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
1149                         nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
1150                         nicfrag->u0.s.frag_len = segs[i].ds_len;
1151                         pd->wqe_idx = wq->ring->pidx;
1152                         RING_PUT(wq->ring, 1);
1153                         atomic_add_int(&wq->ring->num_used, 1);
1154                 }
1155                 if (num_wqes > (pd->nsegs + 1)) {
1156                         nicfrag =
1157                             RING_GET_PRODUCER_ITEM_VA(wq->ring,
1158                                                       struct oce_nic_frag_wqe);
1159                         nicfrag->u0.dw[0] = 0;
1160                         nicfrag->u0.dw[1] = 0;
1161                         nicfrag->u0.dw[2] = 0;
1162                         nicfrag->u0.dw[3] = 0;
1163                         pd->wqe_idx = wq->ring->pidx;
1164                         RING_PUT(wq->ring, 1);
1165                         atomic_add_int(&wq->ring->num_used, 1);
1166                         pd->nsegs++;
1167                 }
1168
1169                 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
1170                 wq->tx_stats.tx_reqs++;
1171                 wq->tx_stats.tx_wrbs += num_wqes;
1172                 wq->tx_stats.tx_bytes += m->m_pkthdr.len;
1173                 wq->tx_stats.tx_pkts++;
1174
1175                 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
1176                                 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1177                 reg_value = (num_wqes << 16) | wq->wq_id;
1178
1179                 /* if os2bmc is not enabled or if the pkt is already tagged as
1180                    bmc, do nothing
1181                  */
1182                 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new);
1183
1184                 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
1185
1186         } else if (rc == EFBIG) {
1187                 if (retry_cnt == 0) {
1188                         m_temp = m_defrag(m, M_NOWAIT);
1189                         if (m_temp == NULL)
1190                                 goto free_ret;
1191                         m = m_temp;
1192                         *mpp = m_temp;
1193                         retry_cnt = retry_cnt + 1;
1194                         goto retry;
1195                 } else
1196                         goto free_ret;
1197         } else if (rc == ENOMEM)
1198                 return rc;
1199         else
1200                 goto free_ret;
1201
1202         if (os2bmc) {
1203                 m = m_new;
1204                 goto tx_start;
1205         }
1206
1207         return 0;
1208
1209 free_ret:
1210         m_freem(*mpp);
1211         *mpp = NULL;
1212         return rc;
1213 }
1214
1215
1216 static void
1217 oce_process_tx_completion(struct oce_wq *wq)
1218 {
1219         struct oce_packet_desc *pd;
1220         POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
1221         struct mbuf *m;
1222
1223         pd = &wq->pckts[wq->pkt_desc_tail];
1224         atomic_store_rel_int(&wq->pkt_desc_tail,
1225                              (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
1226         atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
1227         bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1228         bus_dmamap_unload(wq->tag, pd->map);
1229
1230         m = pd->mbuf;
1231         m_freem(m);
1232         pd->mbuf = NULL;
1233
1234
1235         if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) {
1236                 if (wq->ring->num_used < (wq->ring->num_items / 2)) {
1237                         sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
1238                         oce_tx_restart(sc, wq);
1239                 }
1240         }
1241 }
1242
1243
1244 static void
1245 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
1246 {
1247
1248         if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
1249                 return;
1250
1251 #if __FreeBSD_version >= 800000
1252         if (!drbr_empty(sc->ifp, wq->br))
1253 #else
1254         if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd))
1255 #endif
1256                 taskqueue_enqueue(taskqueue_swi, &wq->txtask);
1257
1258 }
1259
1260
1261 #if defined(INET6) || defined(INET)
1262 static struct mbuf *
1263 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
1264 {
1265         struct mbuf *m;
1266 #ifdef INET
1267         struct ip *ip;
1268 #endif
1269 #ifdef INET6
1270         struct ip6_hdr *ip6;
1271 #endif
1272         struct ether_vlan_header *eh;
1273         struct tcphdr *th;
1274         uint16_t etype;
1275         int total_len = 0, ehdrlen = 0;
1276
1277         m = *mpp;
1278
1279         if (M_WRITABLE(m) == 0) {
1280                 m = m_dup(*mpp, M_NOWAIT);
1281                 if (!m)
1282                         return NULL;
1283                 m_freem(*mpp);
1284                 *mpp = m;
1285         }
1286
1287         eh = mtod(m, struct ether_vlan_header *);
1288         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1289                 etype = ntohs(eh->evl_proto);
1290                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1291         } else {
1292                 etype = ntohs(eh->evl_encap_proto);
1293                 ehdrlen = ETHER_HDR_LEN;
1294         }
1295
1296         switch (etype) {
1297 #ifdef INET
1298         case ETHERTYPE_IP:
1299                 ip = (struct ip *)(m->m_data + ehdrlen);
1300                 if (ip->ip_p != IPPROTO_TCP)
1301                         return NULL;
1302                 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
1303
1304                 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
1305                 break;
1306 #endif
1307 #ifdef INET6
1308         case ETHERTYPE_IPV6:
1309                 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
1310                 if (ip6->ip6_nxt != IPPROTO_TCP)
1311                         return NULL;
1312                 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
1313
1314                 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
1315                 break;
1316 #endif
1317         default:
1318                 return NULL;
1319         }
1320
1321         m = m_pullup(m, total_len);
1322         if (!m)
1323                 return NULL;
1324         *mpp = m;
1325         return m;
1326
1327 }
1328 #endif /* INET6 || INET */
1329
1330 void
1331 oce_tx_task(void *arg, int npending)
1332 {
1333         struct oce_wq *wq = arg;
1334         POCE_SOFTC sc = wq->parent;
1335         struct ifnet *ifp = sc->ifp;
1336         int rc = 0;
1337
1338 #if __FreeBSD_version >= 800000
1339         LOCK(&wq->tx_lock);
1340         rc = oce_multiq_transmit(ifp, NULL, wq);
1341         if (rc) {
1342                 device_printf(sc->dev,
1343                                 "TX[%d] restart failed\n", wq->queue_index);
1344         }
1345         UNLOCK(&wq->tx_lock);
1346 #else
1347         oce_start(ifp);
1348 #endif
1349
1350 }
1351
1352
1353 void
1354 oce_start(struct ifnet *ifp)
1355 {
1356         POCE_SOFTC sc = ifp->if_softc;
1357         struct mbuf *m;
1358         int rc = 0;
1359         int def_q = 0; /* Defualt tx queue is 0*/
1360
1361         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1362                         IFF_DRV_RUNNING)
1363                 return;
1364
1365         if (!sc->link_status)
1366                 return;
1367
1368         do {
1369                 IF_DEQUEUE(&sc->ifp->if_snd, m);
1370                 if (m == NULL)
1371                         break;
1372
1373                 LOCK(&sc->wq[def_q]->tx_lock);
1374                 rc = oce_tx(sc, &m, def_q);
1375                 UNLOCK(&sc->wq[def_q]->tx_lock);
1376                 if (rc) {
1377                         if (m != NULL) {
1378                                 sc->wq[def_q]->tx_stats.tx_stops ++;
1379                                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1380                                 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1381                                 m = NULL;
1382                         }
1383                         break;
1384                 }
1385                 if (m != NULL)
1386                         ETHER_BPF_MTAP(ifp, m);
1387
1388         } while (TRUE);
1389
1390         return;
1391 }
1392
1393
1394 /* Handle the Completion Queue for transmit */
1395 uint16_t
1396 oce_wq_handler(void *arg)
1397 {
1398         struct oce_wq *wq = (struct oce_wq *)arg;
1399         POCE_SOFTC sc = wq->parent;
1400         struct oce_cq *cq = wq->cq;
1401         struct oce_nic_tx_cqe *cqe;
1402         int num_cqes = 0;
1403
1404         LOCK(&wq->tx_compl_lock);
1405         bus_dmamap_sync(cq->ring->dma.tag,
1406                         cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1407         cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1408         while (cqe->u0.dw[3]) {
1409                 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
1410
1411                 wq->ring->cidx = cqe->u0.s.wqe_index + 1;
1412                 if (wq->ring->cidx >= wq->ring->num_items)
1413                         wq->ring->cidx -= wq->ring->num_items;
1414
1415                 oce_process_tx_completion(wq);
1416                 wq->tx_stats.tx_compl++;
1417                 cqe->u0.dw[3] = 0;
1418                 RING_GET(cq->ring, 1);
1419                 bus_dmamap_sync(cq->ring->dma.tag,
1420                                 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1421                 cqe =
1422                     RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1423                 num_cqes++;
1424         }
1425
1426         if (num_cqes)
1427                 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
1428
1429         UNLOCK(&wq->tx_compl_lock);
1430         return num_cqes;
1431 }
1432
1433
1434 static int
1435 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq)
1436 {
1437         POCE_SOFTC sc = ifp->if_softc;
1438         int status = 0, queue_index = 0;
1439         struct mbuf *next = NULL;
1440         struct buf_ring *br = NULL;
1441
1442         br  = wq->br;
1443         queue_index = wq->queue_index;
1444
1445         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1446                 IFF_DRV_RUNNING) {
1447                 if (m != NULL)
1448                         status = drbr_enqueue(ifp, br, m);
1449                 return status;
1450         }
1451
1452         if (m != NULL) {
1453                 if ((status = drbr_enqueue(ifp, br, m)) != 0)
1454                         return status;
1455         }
1456         while ((next = drbr_peek(ifp, br)) != NULL) {
1457                 if (oce_tx(sc, &next, queue_index)) {
1458                         if (next == NULL) {
1459                                 drbr_advance(ifp, br);
1460                         } else {
1461                                 drbr_putback(ifp, br, next);
1462                                 wq->tx_stats.tx_stops ++;
1463                                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1464                         }
1465                         break;
1466                 }
1467                 drbr_advance(ifp, br);
1468                 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len);
1469                 if (next->m_flags & M_MCAST)
1470                         if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
1471                 ETHER_BPF_MTAP(ifp, next);
1472         }
1473
1474         return 0;
1475 }
1476
1477
1478
1479
1480 /*****************************************************************************
1481  *                          Receive  routines functions                      *
1482  *****************************************************************************/
1483
1484 static void
1485 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2)
1486 {
1487         uint32_t *p;
1488         struct ether_header *eh = NULL;
1489         struct tcphdr *tcp_hdr = NULL;
1490         struct ip *ip4_hdr = NULL;
1491         struct ip6_hdr *ip6 = NULL;
1492         uint32_t payload_len = 0;
1493
1494         eh = mtod(m, struct ether_header *);
1495         /* correct IP header */
1496         if(!cqe2->ipv6_frame) {
1497                 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header));
1498                 ip4_hdr->ip_ttl = cqe2->frame_lifespan;
1499                 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header));
1500                 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip));
1501         }else {
1502                 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header));
1503                 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan;
1504                 payload_len = cqe2->coalesced_size - sizeof(struct ether_header)
1505                                                 - sizeof(struct ip6_hdr);
1506                 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len);
1507                 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr));
1508         }
1509
1510         /* correct tcp header */
1511         tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num);
1512         if(cqe2->push) {
1513                 tcp_hdr->th_flags |= TH_PUSH;
1514         }
1515         tcp_hdr->th_win = htons(cqe2->tcp_window);
1516         tcp_hdr->th_sum = 0xffff;
1517         if(cqe2->ts_opt) {
1518                 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2);
1519                 *p = cqe1->tcp_timestamp_val;
1520                 *(p+1) = cqe1->tcp_timestamp_ecr;
1521         }
1522
1523         return;
1524 }
1525
1526 static void
1527 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m)
1528 {
1529         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1530         uint32_t i = 0, frag_len = 0;
1531         uint32_t len = cqe_info->pkt_size;
1532         struct oce_packet_desc *pd;
1533         struct mbuf *tail = NULL;
1534
1535         for (i = 0; i < cqe_info->num_frags; i++) {
1536                 if (rq->ring->cidx == rq->ring->pidx) {
1537                         device_printf(sc->dev,
1538                                   "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n");
1539                         return;
1540                 }
1541                 pd = &rq->pckts[rq->ring->cidx];
1542
1543                 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1544                 bus_dmamap_unload(rq->tag, pd->map);
1545                 RING_GET(rq->ring, 1);
1546                 rq->pending--;
1547
1548                 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
1549                 pd->mbuf->m_len = frag_len;
1550
1551                 if (tail != NULL) {
1552                         /* additional fragments */
1553                         pd->mbuf->m_flags &= ~M_PKTHDR;
1554                         tail->m_next = pd->mbuf;
1555                         if(rq->islro)
1556                                 tail->m_nextpkt = NULL;
1557                         tail = pd->mbuf;
1558                 } else {
1559                         /* first fragment, fill out much of the packet header */
1560                         pd->mbuf->m_pkthdr.len = len;
1561                         if(rq->islro)
1562                                 pd->mbuf->m_nextpkt = NULL;
1563                         pd->mbuf->m_pkthdr.csum_flags = 0;
1564                         if (IF_CSUM_ENABLED(sc)) {
1565                                 if (cqe_info->l4_cksum_pass) {
1566                                         if(!cqe_info->ipv6_frame) { /* IPV4 */
1567                                                 pd->mbuf->m_pkthdr.csum_flags |=
1568                                                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1569                                         }else { /* IPV6 frame */
1570                                                 if(rq->islro) {
1571                                                         pd->mbuf->m_pkthdr.csum_flags |=
1572                                                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1573                                                 }
1574                                         }
1575                                         pd->mbuf->m_pkthdr.csum_data = 0xffff;
1576                                 }
1577                                 if (cqe_info->ip_cksum_pass) {
1578                                         pd->mbuf->m_pkthdr.csum_flags |=
1579                                                (CSUM_IP_CHECKED|CSUM_IP_VALID);
1580                                 }
1581                         }
1582                         *m = tail = pd->mbuf;
1583                }
1584                 pd->mbuf = NULL;
1585                 len -= frag_len;
1586         }
1587
1588         return;
1589 }
1590
1591 static void
1592 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2)
1593 {
1594         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1595         struct nic_hwlro_cqe_part1 *cqe1 = NULL;
1596         struct mbuf *m = NULL;
1597         struct oce_common_cqe_info cq_info;
1598
1599         /* parse cqe */
1600         if(cqe2 == NULL) {
1601                 cq_info.pkt_size =  cqe->pkt_size;
1602                 cq_info.vtag = cqe->vlan_tag;
1603                 cq_info.l4_cksum_pass = cqe->l4_cksum_pass;
1604                 cq_info.ip_cksum_pass = cqe->ip_cksum_pass;
1605                 cq_info.ipv6_frame = cqe->ipv6_frame;
1606                 cq_info.vtp = cqe->vtp;
1607                 cq_info.qnq = cqe->qnq;
1608         }else {
1609                 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe;
1610                 cq_info.pkt_size =  cqe2->coalesced_size;
1611                 cq_info.vtag = cqe2->vlan_tag;
1612                 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass;
1613                 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass;
1614                 cq_info.ipv6_frame = cqe2->ipv6_frame;
1615                 cq_info.vtp = cqe2->vtp;
1616                 cq_info.qnq = cqe1->qnq;
1617         }
1618
1619         cq_info.vtag = BSWAP_16(cq_info.vtag);
1620
1621         cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size;
1622         if(cq_info.pkt_size % rq->cfg.frag_size)
1623                 cq_info.num_frags++;
1624
1625         oce_rx_mbuf_chain(rq, &cq_info, &m);
1626
1627         if (m) {
1628                 if(cqe2) {
1629                         //assert(cqe2->valid != 0);
1630
1631                         //assert(cqe2->cqe_type != 2);
1632                         oce_correct_header(m, cqe1, cqe2);
1633                 }
1634
1635                 m->m_pkthdr.rcvif = sc->ifp;
1636 #if __FreeBSD_version >= 800000
1637                 if (rq->queue_index)
1638                         m->m_pkthdr.flowid = (rq->queue_index - 1);
1639                 else
1640                         m->m_pkthdr.flowid = rq->queue_index;
1641                 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1642 #endif
1643                 /* This deternies if vlan tag is Valid */
1644                 if (cq_info.vtp) {
1645                         if (sc->function_mode & FNM_FLEX10_MODE) {
1646                                 /* FLEX10. If QnQ is not set, neglect VLAN */
1647                                 if (cq_info.qnq) {
1648                                         m->m_pkthdr.ether_vtag = cq_info.vtag;
1649                                         m->m_flags |= M_VLANTAG;
1650                                 }
1651                         } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK))  {
1652                                 /* In UMC mode generally pvid will be striped by
1653                                    hw. But in some cases we have seen it comes
1654                                    with pvid. So if pvid == vlan, neglect vlan.
1655                                  */
1656                                 m->m_pkthdr.ether_vtag = cq_info.vtag;
1657                                 m->m_flags |= M_VLANTAG;
1658                         }
1659                 }
1660                 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1661
1662                 (*sc->ifp->if_input) (sc->ifp, m);
1663
1664                 /* Update rx stats per queue */
1665                 rq->rx_stats.rx_pkts++;
1666                 rq->rx_stats.rx_bytes += cq_info.pkt_size;
1667                 rq->rx_stats.rx_frags += cq_info.num_frags;
1668                 rq->rx_stats.rx_ucast_pkts++;
1669         }
1670         return;
1671 }
1672
1673 static void
1674 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
1675 {
1676         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1677         int len;
1678         struct mbuf *m = NULL;
1679         struct oce_common_cqe_info cq_info;
1680         uint16_t vtag = 0;
1681
1682         /* Is it a flush compl that has no data */
1683         if(!cqe->u0.s.num_fragments)
1684                 goto exit;
1685
1686         len = cqe->u0.s.pkt_size;
1687         if (!len) {
1688                 /*partial DMA workaround for Lancer*/
1689                 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1690                 goto exit;
1691         }
1692
1693         if (!oce_cqe_portid_valid(sc, cqe)) {
1694                 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1695                 goto exit;
1696         }
1697
1698          /* Get vlan_tag value */
1699         if(IS_BE(sc) || IS_SH(sc))
1700                 vtag = BSWAP_16(cqe->u0.s.vlan_tag);
1701         else
1702                 vtag = cqe->u0.s.vlan_tag;
1703
1704         cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass;
1705         cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass;
1706         cq_info.ipv6_frame = cqe->u0.s.ip_ver;
1707         cq_info.num_frags = cqe->u0.s.num_fragments;
1708         cq_info.pkt_size = cqe->u0.s.pkt_size;
1709
1710         oce_rx_mbuf_chain(rq, &cq_info, &m);
1711
1712         if (m) {
1713                 m->m_pkthdr.rcvif = sc->ifp;
1714 #if __FreeBSD_version >= 800000
1715                 if (rq->queue_index)
1716                         m->m_pkthdr.flowid = (rq->queue_index - 1);
1717                 else
1718                         m->m_pkthdr.flowid = rq->queue_index;
1719                 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1720 #endif
1721                 /* This deternies if vlan tag is Valid */
1722                 if (oce_cqe_vtp_valid(sc, cqe)) {
1723                         if (sc->function_mode & FNM_FLEX10_MODE) {
1724                                 /* FLEX10. If QnQ is not set, neglect VLAN */
1725                                 if (cqe->u0.s.qnq) {
1726                                         m->m_pkthdr.ether_vtag = vtag;
1727                                         m->m_flags |= M_VLANTAG;
1728                                 }
1729                         } else if (sc->pvid != (vtag & VLAN_VID_MASK))  {
1730                                 /* In UMC mode generally pvid will be striped by
1731                                    hw. But in some cases we have seen it comes
1732                                    with pvid. So if pvid == vlan, neglect vlan.
1733                                 */
1734                                 m->m_pkthdr.ether_vtag = vtag;
1735                                 m->m_flags |= M_VLANTAG;
1736                         }
1737                 }
1738
1739                 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1740 #if defined(INET6) || defined(INET)
1741                 /* Try to queue to LRO */
1742                 if (IF_LRO_ENABLED(sc) &&
1743                     (cqe->u0.s.ip_cksum_pass) &&
1744                     (cqe->u0.s.l4_cksum_pass) &&
1745                     (!cqe->u0.s.ip_ver)       &&
1746                     (rq->lro.lro_cnt != 0)) {
1747
1748                         if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
1749                                 rq->lro_pkts_queued ++;
1750                                 goto post_done;
1751                         }
1752                         /* If LRO posting fails then try to post to STACK */
1753                 }
1754 #endif
1755
1756                 (*sc->ifp->if_input) (sc->ifp, m);
1757 #if defined(INET6) || defined(INET)
1758 post_done:
1759 #endif
1760                 /* Update rx stats per queue */
1761                 rq->rx_stats.rx_pkts++;
1762                 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
1763                 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
1764                 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
1765                         rq->rx_stats.rx_mcast_pkts++;
1766                 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
1767                         rq->rx_stats.rx_ucast_pkts++;
1768         }
1769 exit:
1770         return;
1771 }
1772
1773
1774 void
1775 oce_discard_rx_comp(struct oce_rq *rq, int num_frags)
1776 {
1777         uint32_t i = 0;
1778         struct oce_packet_desc *pd;
1779         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1780
1781         for (i = 0; i < num_frags; i++) {
1782                 if (rq->ring->cidx == rq->ring->pidx) {
1783                         device_printf(sc->dev,
1784                                 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n");
1785                         return;
1786                 }
1787                 pd = &rq->pckts[rq->ring->cidx];
1788                 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1789                 bus_dmamap_unload(rq->tag, pd->map);
1790                 if (pd->mbuf != NULL) {
1791                         m_freem(pd->mbuf);
1792                         pd->mbuf = NULL;
1793                 }
1794
1795                 RING_GET(rq->ring, 1);
1796                 rq->pending--;
1797         }
1798 }
1799
1800
1801 static int
1802 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1803 {
1804         struct oce_nic_rx_cqe_v1 *cqe_v1;
1805         int vtp = 0;
1806
1807         if (sc->be3_native) {
1808                 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1809                 vtp =  cqe_v1->u0.s.vlan_tag_present;
1810         } else
1811                 vtp = cqe->u0.s.vlan_tag_present;
1812
1813         return vtp;
1814
1815 }
1816
1817
1818 static int
1819 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1820 {
1821         struct oce_nic_rx_cqe_v1 *cqe_v1;
1822         int port_id = 0;
1823
1824         if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
1825                 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1826                 port_id =  cqe_v1->u0.s.port;
1827                 if (sc->port_id != port_id)
1828                         return 0;
1829         } else
1830                 ;/* For BE3 legacy and Lancer this is dummy */
1831
1832         return 1;
1833
1834 }
1835
1836 #if defined(INET6) || defined(INET)
1837 void
1838 oce_rx_flush_lro(struct oce_rq *rq)
1839 {
1840         struct lro_ctrl *lro = &rq->lro;
1841         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1842
1843         if (!IF_LRO_ENABLED(sc))
1844                 return;
1845
1846         tcp_lro_flush_all(lro);
1847         rq->lro_pkts_queued = 0;
1848
1849         return;
1850 }
1851
1852
1853 static int
1854 oce_init_lro(POCE_SOFTC sc)
1855 {
1856         struct lro_ctrl *lro = NULL;
1857         int i = 0, rc = 0;
1858
1859         for (i = 0; i < sc->nrqs; i++) {
1860                 lro = &sc->rq[i]->lro;
1861                 rc = tcp_lro_init(lro);
1862                 if (rc != 0) {
1863                         device_printf(sc->dev, "LRO init failed\n");
1864                         return rc;
1865                 }
1866                 lro->ifp = sc->ifp;
1867         }
1868
1869         return rc;
1870 }
1871
1872
1873 void
1874 oce_free_lro(POCE_SOFTC sc)
1875 {
1876         struct lro_ctrl *lro = NULL;
1877         int i = 0;
1878
1879         for (i = 0; i < sc->nrqs; i++) {
1880                 lro = &sc->rq[i]->lro;
1881                 if (lro)
1882                         tcp_lro_free(lro);
1883         }
1884 }
1885 #endif
1886
1887 int
1888 oce_alloc_rx_bufs(struct oce_rq *rq, int count)
1889 {
1890         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1891         int i, in, rc;
1892         struct oce_packet_desc *pd;
1893         bus_dma_segment_t segs[6];
1894         int nsegs, added = 0;
1895         struct oce_nic_rqe *rqe;
1896         pd_rxulp_db_t rxdb_reg;
1897         uint32_t val = 0;
1898         uint32_t oce_max_rq_posts = 64;
1899
1900         bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
1901         for (i = 0; i < count; i++) {
1902                 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE;
1903
1904                 pd = &rq->pckts[rq->ring->pidx];
1905                 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size);
1906                 if (pd->mbuf == NULL) {
1907                         device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size);
1908                         break;
1909                 }
1910                 pd->mbuf->m_nextpkt = NULL;
1911
1912                 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size;
1913
1914                 rc = bus_dmamap_load_mbuf_sg(rq->tag,
1915                                              pd->map,
1916                                              pd->mbuf,
1917                                              segs, &nsegs, BUS_DMA_NOWAIT);
1918                 if (rc) {
1919                         m_free(pd->mbuf);
1920                         device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc);
1921                         break;
1922                 }
1923
1924                 if (nsegs != 1) {
1925                         i--;
1926                         continue;
1927                 }
1928
1929                 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
1930
1931                 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
1932                 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
1933                 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
1934                 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
1935                 RING_PUT(rq->ring, 1);
1936                 added++;
1937                 rq->pending++;
1938         }
1939         oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS;
1940         if (added != 0) {
1941                 for (i = added / oce_max_rq_posts; i > 0; i--) {
1942                         rxdb_reg.bits.num_posted = oce_max_rq_posts;
1943                         rxdb_reg.bits.qid = rq->rq_id;
1944                         if(rq->islro) {
1945                                 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1946                                 val |= oce_max_rq_posts << 16;
1947                                 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1948                         }else {
1949                                 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1950                         }
1951                         added -= oce_max_rq_posts;
1952                 }
1953                 if (added > 0) {
1954                         rxdb_reg.bits.qid = rq->rq_id;
1955                         rxdb_reg.bits.num_posted = added;
1956                         if(rq->islro) {
1957                                 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1958                                 val |= added << 16;
1959                                 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1960                         }else {
1961                                 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1962                         }
1963                 }
1964         }
1965
1966         return 0;
1967 }
1968
1969 static void
1970 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq)
1971 {
1972         if (num_cqes) {
1973                 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE);
1974                 if(!sc->enable_hwlro) {
1975                         if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1)
1976                                 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1));
1977                 }else {
1978                         if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64)
1979                                 oce_alloc_rx_bufs(rq, 64);
1980                 }
1981         }
1982
1983         return;
1984 }
1985
1986 uint16_t
1987 oce_rq_handler_lro(void *arg)
1988 {
1989         struct oce_rq *rq = (struct oce_rq *)arg;
1990         struct oce_cq *cq = rq->cq;
1991         POCE_SOFTC sc = rq->parent;
1992         struct nic_hwlro_singleton_cqe *cqe;
1993         struct nic_hwlro_cqe_part2 *cqe2;
1994         int num_cqes = 0;
1995
1996         LOCK(&rq->rx_lock);
1997         bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1998         cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
1999         while (cqe->valid) {
2000                 if(cqe->cqe_type == 0) { /* singleton cqe */
2001                         /* we should not get singleton cqe after cqe1 on same rq */
2002                         if(rq->cqe_firstpart != NULL) {
2003                                 device_printf(sc->dev, "Got singleton cqe after cqe1 \n");
2004                                 goto exit_rq_handler_lro;
2005                         }
2006                         if(cqe->error != 0) {
2007                                 rq->rx_stats.rxcp_err++;
2008                                 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2009                         }
2010                         oce_rx_lro(rq, cqe, NULL);
2011                         rq->rx_stats.rx_compl++;
2012                         cqe->valid = 0;
2013                         RING_GET(cq->ring, 1);
2014                         num_cqes++;
2015                         if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2016                                 break;
2017                 }else if(cqe->cqe_type == 0x1) { /* first part */
2018                         /* we should not get cqe1 after cqe1 on same rq */
2019                         if(rq->cqe_firstpart != NULL) {
2020                                 device_printf(sc->dev, "Got cqe1 after cqe1 \n");
2021                                 goto exit_rq_handler_lro;
2022                         }
2023                         rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe;
2024                         RING_GET(cq->ring, 1);
2025                 }else if(cqe->cqe_type == 0x2) { /* second part */
2026                         cqe2 = (struct nic_hwlro_cqe_part2 *)cqe;
2027                         if(cqe2->error != 0) {
2028                                 rq->rx_stats.rxcp_err++;
2029                                 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2030                         }
2031                         /* We should not get cqe2 without cqe1 */
2032                         if(rq->cqe_firstpart == NULL) {
2033                                 device_printf(sc->dev, "Got cqe2 without cqe1 \n");
2034                                 goto exit_rq_handler_lro;
2035                         }
2036                         oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2);
2037
2038                         rq->rx_stats.rx_compl++;
2039                         rq->cqe_firstpart->valid = 0;
2040                         cqe2->valid = 0;
2041                         rq->cqe_firstpart = NULL;
2042
2043                         RING_GET(cq->ring, 1);
2044                         num_cqes += 2;
2045                         if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2046                                 break;
2047                 }
2048
2049                 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2050                 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2051         }
2052         oce_check_rx_bufs(sc, num_cqes, rq);
2053 exit_rq_handler_lro:
2054         UNLOCK(&rq->rx_lock);
2055         return 0;
2056 }
2057
2058 /* Handle the Completion Queue for receive */
2059 uint16_t
2060 oce_rq_handler(void *arg)
2061 {
2062         struct oce_rq *rq = (struct oce_rq *)arg;
2063         struct oce_cq *cq = rq->cq;
2064         POCE_SOFTC sc = rq->parent;
2065         struct oce_nic_rx_cqe *cqe;
2066         int num_cqes = 0;
2067
2068         if(rq->islro) {
2069                 oce_rq_handler_lro(arg);
2070                 return 0;
2071         }
2072         LOCK(&rq->rx_lock);
2073         bus_dmamap_sync(cq->ring->dma.tag,
2074                         cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2075         cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2076         while (cqe->u0.dw[2]) {
2077                 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
2078
2079                 if (cqe->u0.s.error == 0) {
2080                         oce_rx(rq, cqe);
2081                 } else {
2082                         rq->rx_stats.rxcp_err++;
2083                         if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2084                         /* Post L3/L4 errors to stack.*/
2085                         oce_rx(rq, cqe);
2086                 }
2087                 rq->rx_stats.rx_compl++;
2088                 cqe->u0.dw[2] = 0;
2089
2090 #if defined(INET6) || defined(INET)
2091                 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
2092                         oce_rx_flush_lro(rq);
2093                 }
2094 #endif
2095
2096                 RING_GET(cq->ring, 1);
2097                 bus_dmamap_sync(cq->ring->dma.tag,
2098                                 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2099                 cqe =
2100                     RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2101                 num_cqes++;
2102                 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2103                         break;
2104         }
2105
2106 #if defined(INET6) || defined(INET)
2107         if (IF_LRO_ENABLED(sc))
2108                 oce_rx_flush_lro(rq);
2109 #endif
2110
2111         oce_check_rx_bufs(sc, num_cqes, rq);
2112         UNLOCK(&rq->rx_lock);
2113         return 0;
2114
2115 }
2116
2117
2118
2119
2120 /*****************************************************************************
2121  *                 Helper function prototypes in this file                   *
2122  *****************************************************************************/
2123
2124 static int
2125 oce_attach_ifp(POCE_SOFTC sc)
2126 {
2127
2128         sc->ifp = if_alloc(IFT_ETHER);
2129         if (!sc->ifp)
2130                 return ENOMEM;
2131
2132         ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
2133         ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2134         ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2135
2136         sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST;
2137         sc->ifp->if_ioctl = oce_ioctl;
2138         sc->ifp->if_start = oce_start;
2139         sc->ifp->if_init = oce_init;
2140         sc->ifp->if_mtu = ETHERMTU;
2141         sc->ifp->if_softc = sc;
2142 #if __FreeBSD_version >= 800000
2143         sc->ifp->if_transmit = oce_multiq_start;
2144         sc->ifp->if_qflush = oce_multiq_flush;
2145 #endif
2146
2147         if_initname(sc->ifp,
2148                     device_get_name(sc->dev), device_get_unit(sc->dev));
2149
2150         sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1;
2151         IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen);
2152         IFQ_SET_READY(&sc->ifp->if_snd);
2153
2154         sc->ifp->if_hwassist = OCE_IF_HWASSIST;
2155         sc->ifp->if_hwassist |= CSUM_TSO;
2156         sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP);
2157
2158         sc->ifp->if_capabilities = OCE_IF_CAPABILITIES;
2159         sc->ifp->if_capabilities |= IFCAP_HWCSUM;
2160         sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2161
2162 #if defined(INET6) || defined(INET)
2163         sc->ifp->if_capabilities |= IFCAP_TSO;
2164         sc->ifp->if_capabilities |= IFCAP_LRO;
2165         sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
2166 #endif
2167
2168         sc->ifp->if_capenable = sc->ifp->if_capabilities;
2169         sc->ifp->if_baudrate = IF_Gbps(10);
2170
2171 #if __FreeBSD_version >= 1000000
2172         sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2173         sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS;
2174         sc->ifp->if_hw_tsomaxsegsize = 4096;
2175 #endif
2176
2177         ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
2178
2179         return 0;
2180 }
2181
2182
2183 static void
2184 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2185 {
2186         POCE_SOFTC sc = ifp->if_softc;
2187
2188         if (ifp->if_softc !=  arg)
2189                 return;
2190         if ((vtag == 0) || (vtag > 4095))
2191                 return;
2192
2193         sc->vlan_tag[vtag] = 1;
2194         sc->vlans_added++;
2195         if (sc->vlans_added <= (sc->max_vlans + 1))
2196                 oce_vid_config(sc);
2197 }
2198
2199
2200 static void
2201 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2202 {
2203         POCE_SOFTC sc = ifp->if_softc;
2204
2205         if (ifp->if_softc !=  arg)
2206                 return;
2207         if ((vtag == 0) || (vtag > 4095))
2208                 return;
2209
2210         sc->vlan_tag[vtag] = 0;
2211         sc->vlans_added--;
2212         oce_vid_config(sc);
2213 }
2214
2215
2216 /*
2217  * A max of 64 vlans can be configured in BE. If the user configures
2218  * more, place the card in vlan promiscuous mode.
2219  */
2220 static int
2221 oce_vid_config(POCE_SOFTC sc)
2222 {
2223         struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
2224         uint16_t ntags = 0, i;
2225         int status = 0;
2226
2227         if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
2228                         (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) {
2229                 for (i = 0; i < MAX_VLANS; i++) {
2230                         if (sc->vlan_tag[i]) {
2231                                 vtags[ntags].vtag = i;
2232                                 ntags++;
2233                         }
2234                 }
2235                 if (ntags)
2236                         status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2237                                                 vtags, ntags, 1, 0);
2238         } else
2239                 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2240                                                 NULL, 0, 1, 1);
2241         return status;
2242 }
2243
2244
2245 static void
2246 oce_mac_addr_set(POCE_SOFTC sc)
2247 {
2248         uint32_t old_pmac_id = sc->pmac_id;
2249         int status = 0;
2250
2251
2252         status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2253                          sc->macaddr.size_of_struct);
2254         if (!status)
2255                 return;
2256
2257         status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)),
2258                                         sc->if_id, &sc->pmac_id);
2259         if (!status) {
2260                 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
2261                 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2262                                  sc->macaddr.size_of_struct);
2263         }
2264         if (status)
2265                 device_printf(sc->dev, "Failed update macaddress\n");
2266
2267 }
2268
2269
2270 static int
2271 oce_handle_passthrough(struct ifnet *ifp, caddr_t data)
2272 {
2273         POCE_SOFTC sc = ifp->if_softc;
2274         struct ifreq *ifr = (struct ifreq *)data;
2275         int rc = ENXIO;
2276         char cookie[32] = {0};
2277         void *priv_data = (void *)ifr->ifr_data;
2278         void *ioctl_ptr;
2279         uint32_t req_size;
2280         struct mbx_hdr req;
2281         OCE_DMA_MEM dma_mem;
2282         struct mbx_common_get_cntl_attr *fw_cmd;
2283
2284         if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
2285                 return EFAULT;
2286
2287         if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
2288                 return EINVAL;
2289
2290         ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
2291         if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
2292                 return EFAULT;
2293
2294         req_size = le32toh(req.u0.req.request_length);
2295         if (req_size > 65536)
2296                 return EINVAL;
2297
2298         req_size += sizeof(struct mbx_hdr);
2299         rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
2300         if (rc)
2301                 return ENOMEM;
2302
2303         if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
2304                 rc = EFAULT;
2305                 goto dma_free;
2306         }
2307
2308         rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
2309         if (rc) {
2310                 rc = EIO;
2311                 goto dma_free;
2312         }
2313
2314         if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size))
2315                 rc =  EFAULT;
2316
2317         /*
2318            firmware is filling all the attributes for this ioctl except
2319            the driver version..so fill it
2320          */
2321         if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
2322                 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr;
2323                 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
2324                         COMPONENT_REVISION, strlen(COMPONENT_REVISION));
2325         }
2326
2327 dma_free:
2328         oce_dma_free(sc, &dma_mem);
2329         return rc;
2330
2331 }
2332
2333 static void
2334 oce_eqd_set_periodic(POCE_SOFTC sc)
2335 {
2336         struct oce_set_eqd set_eqd[OCE_MAX_EQ];
2337         struct oce_aic_obj *aic;
2338         struct oce_eq *eqo;
2339         uint64_t now = 0, delta;
2340         int eqd, i, num = 0;
2341         uint32_t tx_reqs = 0, rxpkts = 0, pps;
2342         struct oce_wq *wq;
2343         struct oce_rq *rq;
2344
2345         #define ticks_to_msecs(t)       (1000 * (t) / hz)
2346
2347         for (i = 0 ; i < sc->neqs; i++) {
2348                 eqo = sc->eq[i];
2349                 aic = &sc->aic_obj[i];
2350                 /* When setting the static eq delay from the user space */
2351                 if (!aic->enable) {
2352                         if (aic->ticks)
2353                                 aic->ticks = 0;
2354                         eqd = aic->et_eqd;
2355                         goto modify_eqd;
2356                 }
2357
2358                 rq = sc->rq[i];
2359                 rxpkts = rq->rx_stats.rx_pkts;
2360                 wq = sc->wq[i];
2361                 tx_reqs = wq->tx_stats.tx_reqs;
2362                 now = ticks;
2363
2364                 if (!aic->ticks || now < aic->ticks ||
2365                     rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) {
2366                         aic->prev_rxpkts = rxpkts;
2367                         aic->prev_txreqs = tx_reqs;
2368                         aic->ticks = now;
2369                         continue;
2370                 }
2371
2372                 delta = ticks_to_msecs(now - aic->ticks);
2373
2374                 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) +
2375                       (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta);
2376                 eqd = (pps / 15000) << 2;
2377                 if (eqd < 8)
2378                         eqd = 0;
2379
2380                 /* Make sure that the eq delay is in the known range */
2381                 eqd = min(eqd, aic->max_eqd);
2382                 eqd = max(eqd, aic->min_eqd);
2383
2384                 aic->prev_rxpkts = rxpkts;
2385                 aic->prev_txreqs = tx_reqs;
2386                 aic->ticks = now;
2387
2388 modify_eqd:
2389                 if (eqd != aic->cur_eqd) {
2390                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
2391                         set_eqd[num].eq_id = eqo->eq_id;
2392                         aic->cur_eqd = eqd;
2393                         num++;
2394                 }
2395         }
2396
2397         /* Is there atleast one eq that needs to be modified? */
2398         for(i = 0; i < num; i += 8) {
2399                 if((num - i) >=8 )
2400                         oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8);
2401                 else
2402                         oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i));
2403         }
2404
2405 }
2406
2407 static void oce_detect_hw_error(POCE_SOFTC sc)
2408 {
2409
2410         uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
2411         uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2412         uint32_t i;
2413
2414         if (sc->hw_error)
2415                 return;
2416
2417         if (IS_XE201(sc)) {
2418                 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
2419                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2420                         sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
2421                         sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
2422                 }
2423         } else {
2424                 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
2425                 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
2426                 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
2427                 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
2428
2429                 ue_low = (ue_low & ~ue_low_mask);
2430                 ue_high = (ue_high & ~ue_high_mask);
2431         }
2432
2433         /* On certain platforms BE hardware can indicate spurious UEs.
2434          * Allow the h/w to stop working completely in case of a real UE.
2435          * Hence not setting the hw_error for UE detection.
2436          */
2437         if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2438                 sc->hw_error = TRUE;
2439                 device_printf(sc->dev, "Error detected in the card\n");
2440         }
2441
2442         if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2443                 device_printf(sc->dev,
2444                                 "ERR: sliport status 0x%x\n", sliport_status);
2445                 device_printf(sc->dev,
2446                                 "ERR: sliport error1 0x%x\n", sliport_err1);
2447                 device_printf(sc->dev,
2448                                 "ERR: sliport error2 0x%x\n", sliport_err2);
2449         }
2450
2451         if (ue_low) {
2452                 for (i = 0; ue_low; ue_low >>= 1, i++) {
2453                         if (ue_low & 1)
2454                                 device_printf(sc->dev, "UE: %s bit set\n",
2455                                                         ue_status_low_desc[i]);
2456                 }
2457         }
2458
2459         if (ue_high) {
2460                 for (i = 0; ue_high; ue_high >>= 1, i++) {
2461                         if (ue_high & 1)
2462                                 device_printf(sc->dev, "UE: %s bit set\n",
2463                                                         ue_status_hi_desc[i]);
2464                 }
2465         }
2466
2467 }
2468
2469
2470 static void
2471 oce_local_timer(void *arg)
2472 {
2473         POCE_SOFTC sc = arg;
2474         int i = 0;
2475
2476         oce_detect_hw_error(sc);
2477         oce_refresh_nic_stats(sc);
2478         oce_refresh_queue_stats(sc);
2479         oce_mac_addr_set(sc);
2480
2481         /* TX Watch Dog*/
2482         for (i = 0; i < sc->nwqs; i++)
2483                 oce_tx_restart(sc, sc->wq[i]);
2484
2485         /* calculate and set the eq delay for optimal interrupt rate */
2486         if (IS_BE(sc) || IS_SH(sc))
2487                 oce_eqd_set_periodic(sc);
2488
2489         callout_reset(&sc->timer, hz, oce_local_timer, sc);
2490 }
2491
2492 static void
2493 oce_tx_compl_clean(POCE_SOFTC sc)
2494 {
2495         struct oce_wq *wq;
2496         int i = 0, timeo = 0, num_wqes = 0;
2497         int pending_txqs = sc->nwqs;
2498
2499         /* Stop polling for compls when HW has been silent for 10ms or
2500          * hw_error or no outstanding completions expected
2501          */
2502         do {
2503                 pending_txqs = sc->nwqs;
2504
2505                 for_all_wq_queues(sc, wq, i) {
2506                         num_wqes = oce_wq_handler(wq);
2507
2508                         if(num_wqes)
2509                                 timeo = 0;
2510
2511                         if(!wq->ring->num_used)
2512                                 pending_txqs--;
2513                 }
2514
2515                 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error)
2516                         break;
2517
2518                 DELAY(1000);
2519         } while (TRUE);
2520
2521         for_all_wq_queues(sc, wq, i) {
2522                 while(wq->ring->num_used) {
2523                         LOCK(&wq->tx_compl_lock);
2524                         oce_process_tx_completion(wq);
2525                         UNLOCK(&wq->tx_compl_lock);
2526                 }
2527         }
2528
2529 }
2530
2531 /* NOTE : This should only be called holding
2532  *        DEVICE_LOCK.
2533  */
2534 static void
2535 oce_if_deactivate(POCE_SOFTC sc)
2536 {
2537         int i;
2538         struct oce_rq *rq;
2539         struct oce_wq *wq;
2540         struct oce_eq *eq;
2541
2542         sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2543
2544         oce_tx_compl_clean(sc);
2545
2546         /* Stop intrs and finish any bottom halves pending */
2547         oce_hw_intr_disable(sc);
2548
2549         /* Since taskqueue_drain takes a Gaint Lock, We should not acquire
2550            any other lock. So unlock device lock and require after
2551            completing taskqueue_drain.
2552         */
2553         UNLOCK(&sc->dev_lock);
2554         for (i = 0; i < sc->intr_count; i++) {
2555                 if (sc->intrs[i].tq != NULL) {
2556                         taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
2557                 }
2558         }
2559         LOCK(&sc->dev_lock);
2560
2561         /* Delete RX queue in card with flush param */
2562         oce_stop_rx(sc);
2563
2564         /* Invalidate any pending cq and eq entries*/
2565         for_all_evnt_queues(sc, eq, i)
2566                 oce_drain_eq(eq);
2567         for_all_rq_queues(sc, rq, i)
2568                 oce_drain_rq_cq(rq);
2569         for_all_wq_queues(sc, wq, i)
2570                 oce_drain_wq_cq(wq);
2571
2572         /* But still we need to get MCC aync events.
2573            So enable intrs and also arm first EQ
2574         */
2575         oce_hw_intr_enable(sc);
2576         oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
2577
2578         DELAY(10);
2579 }
2580
2581
2582 static void
2583 oce_if_activate(POCE_SOFTC sc)
2584 {
2585         struct oce_eq *eq;
2586         struct oce_rq *rq;
2587         struct oce_wq *wq;
2588         int i, rc = 0;
2589
2590         sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
2591
2592         oce_hw_intr_disable(sc);
2593
2594         oce_start_rx(sc);
2595
2596         for_all_rq_queues(sc, rq, i) {
2597                 rc = oce_start_rq(rq);
2598                 if (rc)
2599                         device_printf(sc->dev, "Unable to start RX\n");
2600         }
2601
2602         for_all_wq_queues(sc, wq, i) {
2603                 rc = oce_start_wq(wq);
2604                 if (rc)
2605                         device_printf(sc->dev, "Unable to start TX\n");
2606         }
2607
2608
2609         for_all_evnt_queues(sc, eq, i)
2610                 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
2611
2612         oce_hw_intr_enable(sc);
2613
2614 }
2615
2616 static void
2617 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
2618 {
2619         /* Update Link status */
2620         if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
2621              ASYNC_EVENT_LINK_UP) {
2622                 sc->link_status = ASYNC_EVENT_LINK_UP;
2623                 if_link_state_change(sc->ifp, LINK_STATE_UP);
2624         } else {
2625                 sc->link_status = ASYNC_EVENT_LINK_DOWN;
2626                 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2627         }
2628 }
2629
2630
2631 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc,
2632                                          struct oce_async_evt_grp5_os2bmc *evt)
2633 {
2634         DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc));
2635         if (evt->u.s.mgmt_enable)
2636                 sc->flags |= OCE_FLAGS_OS2BMC;
2637         else
2638                 return;
2639
2640         sc->bmc_filt_mask = evt->u.s.arp_filter;
2641         sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1);
2642         sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2);
2643         sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3);
2644         sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4);
2645         sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5);
2646         sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6);
2647         sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7);
2648         sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8);
2649 }
2650
2651
2652 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe)
2653 {
2654         struct oce_async_event_grp5_pvid_state *gcqe;
2655         struct oce_async_evt_grp5_os2bmc *bmccqe;
2656
2657         switch (cqe->u0.s.async_type) {
2658         case ASYNC_EVENT_PVID_STATE:
2659                 /* GRP5 PVID */
2660                 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe;
2661                 if (gcqe->enabled)
2662                         sc->pvid = gcqe->tag & VLAN_VID_MASK;
2663                 else
2664                         sc->pvid = 0;
2665                 break;
2666         case ASYNC_EVENT_OS2BMC:
2667                 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe;
2668                 oce_async_grp5_osbmc_process(sc, bmccqe);
2669                 break;
2670         default:
2671                 break;
2672         }
2673 }
2674
2675 /* Handle the Completion Queue for the Mailbox/Async notifications */
2676 uint16_t
2677 oce_mq_handler(void *arg)
2678 {
2679         struct oce_mq *mq = (struct oce_mq *)arg;
2680         POCE_SOFTC sc = mq->parent;
2681         struct oce_cq *cq = mq->cq;
2682         int num_cqes = 0, evt_type = 0, optype = 0;
2683         struct oce_mq_cqe *cqe;
2684         struct oce_async_cqe_link_state *acqe;
2685         struct oce_async_event_qnq *dbgcqe;
2686
2687
2688         bus_dmamap_sync(cq->ring->dma.tag,
2689                         cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2690         cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2691
2692         while (cqe->u0.dw[3]) {
2693                 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
2694                 if (cqe->u0.s.async_event) {
2695                         evt_type = cqe->u0.s.event_type;
2696                         optype = cqe->u0.s.async_type;
2697                         if (evt_type  == ASYNC_EVENT_CODE_LINK_STATE) {
2698                                 /* Link status evt */
2699                                 acqe = (struct oce_async_cqe_link_state *)cqe;
2700                                 process_link_state(sc, acqe);
2701                         } else if (evt_type == ASYNC_EVENT_GRP5) {
2702                                 oce_process_grp5_events(sc, cqe);
2703                         } else if (evt_type == ASYNC_EVENT_CODE_DEBUG &&
2704                                         optype == ASYNC_EVENT_DEBUG_QNQ) {
2705                                 dbgcqe =  (struct oce_async_event_qnq *)cqe;
2706                                 if(dbgcqe->valid)
2707                                         sc->qnqid = dbgcqe->vlan_tag;
2708                                 sc->qnq_debug_event = TRUE;
2709                         }
2710                 }
2711                 cqe->u0.dw[3] = 0;
2712                 RING_GET(cq->ring, 1);
2713                 bus_dmamap_sync(cq->ring->dma.tag,
2714                                 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2715                 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2716                 num_cqes++;
2717         }
2718
2719         if (num_cqes)
2720                 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
2721
2722         return 0;
2723 }
2724
2725
2726 static void
2727 setup_max_queues_want(POCE_SOFTC sc)
2728 {
2729         /* Check if it is FLEX machine. Is so dont use RSS */
2730         if ((sc->function_mode & FNM_FLEX10_MODE) ||
2731             (sc->function_mode & FNM_UMC_MODE)    ||
2732             (sc->function_mode & FNM_VNIC_MODE)   ||
2733             (!is_rss_enabled(sc))                 ||
2734             IS_BE2(sc)) {
2735                 sc->nrqs = 1;
2736                 sc->nwqs = 1;
2737         } else {
2738                 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2739                 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
2740         }
2741
2742         if (IS_BE2(sc) && is_rss_enabled(sc))
2743                 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2744 }
2745
2746
2747 static void
2748 update_queues_got(POCE_SOFTC sc)
2749 {
2750         if (is_rss_enabled(sc)) {
2751                 sc->nrqs = sc->intr_count + 1;
2752                 sc->nwqs = sc->intr_count;
2753         } else {
2754                 sc->nrqs = 1;
2755                 sc->nwqs = 1;
2756         }
2757
2758         if (IS_BE2(sc))
2759                 sc->nwqs = 1;
2760 }
2761
2762 static int
2763 oce_check_ipv6_ext_hdr(struct mbuf *m)
2764 {
2765         struct ether_header *eh = mtod(m, struct ether_header *);
2766         caddr_t m_datatemp = m->m_data;
2767
2768         if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
2769                 m->m_data += sizeof(struct ether_header);
2770                 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
2771
2772                 if((ip6->ip6_nxt != IPPROTO_TCP) && \
2773                                 (ip6->ip6_nxt != IPPROTO_UDP)){
2774                         struct ip6_ext *ip6e = NULL;
2775                         m->m_data += sizeof(struct ip6_hdr);
2776
2777                         ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
2778                         if(ip6e->ip6e_len == 0xff) {
2779                                 m->m_data = m_datatemp;
2780                                 return TRUE;
2781                         }
2782                 }
2783                 m->m_data = m_datatemp;
2784         }
2785         return FALSE;
2786 }
2787
2788 static int
2789 is_be3_a1(POCE_SOFTC sc)
2790 {
2791         if((sc->flags & OCE_FLAGS_BE3)  && ((sc->asic_revision & 0xFF) < 2)) {
2792                 return TRUE;
2793         }
2794         return FALSE;
2795 }
2796
2797 static struct mbuf *
2798 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
2799 {
2800         uint16_t vlan_tag = 0;
2801
2802         if(!M_WRITABLE(m))
2803                 return NULL;
2804
2805         /* Embed vlan tag in the packet if it is not part of it */
2806         if(m->m_flags & M_VLANTAG) {
2807                 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
2808                 m->m_flags &= ~M_VLANTAG;
2809         }
2810
2811         /* if UMC, ignore vlan tag insertion and instead insert pvid */
2812         if(sc->pvid) {
2813                 if(!vlan_tag)
2814                         vlan_tag = sc->pvid;
2815                 if (complete)
2816                         *complete = FALSE;
2817         }
2818
2819         if(vlan_tag) {
2820                 m = ether_vlanencap(m, vlan_tag);
2821         }
2822
2823         if(sc->qnqid) {
2824                 m = ether_vlanencap(m, sc->qnqid);
2825
2826                 if (complete)
2827                         *complete = FALSE;
2828         }
2829         return m;
2830 }
2831
2832 static int
2833 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
2834 {
2835         if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
2836                         oce_check_ipv6_ext_hdr(m)) {
2837                 return TRUE;
2838         }
2839         return FALSE;
2840 }
2841
2842 static void
2843 oce_get_config(POCE_SOFTC sc)
2844 {
2845         int rc = 0;
2846         uint32_t max_rss = 0;
2847
2848         if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
2849                 max_rss = OCE_LEGACY_MODE_RSS;
2850         else
2851                 max_rss = OCE_MAX_RSS;
2852
2853         if (!IS_BE(sc)) {
2854                 rc = oce_get_profile_config(sc, max_rss);
2855                 if (rc) {
2856                         sc->nwqs = OCE_MAX_WQ;
2857                         sc->nrssqs = max_rss;
2858                         sc->nrqs = sc->nrssqs + 1;
2859                 }
2860         }
2861         else { /* For BE3 don't rely on fw for determining the resources */
2862                 sc->nrssqs = max_rss;
2863                 sc->nrqs = sc->nrssqs + 1;
2864                 sc->nwqs = OCE_MAX_WQ;
2865                 sc->max_vlans = MAX_VLANFILTER_SIZE;
2866         }
2867 }
2868
2869 static void
2870 oce_rdma_close(void)
2871 {
2872   if (oce_rdma_if != NULL) {
2873     oce_rdma_if = NULL;
2874   }
2875 }
2876
2877 static void
2878 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr)
2879 {
2880   memcpy(macaddr, sc->macaddr.mac_addr, 6);
2881 }
2882
2883 int
2884 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if)
2885 {
2886   POCE_SOFTC sc;
2887   struct oce_dev_info di;
2888   int i;
2889
2890   if ((rdma_info == NULL) || (rdma_if == NULL)) {
2891     return -EINVAL;
2892   }
2893
2894   if ((rdma_info->size != OCE_RDMA_INFO_SIZE) ||
2895       (rdma_if->size != OCE_RDMA_IF_SIZE)) {
2896     return -ENXIO;
2897   }
2898
2899   rdma_info->close = oce_rdma_close;
2900   rdma_info->mbox_post = oce_mbox_post;
2901   rdma_info->common_req_hdr_init = mbx_common_req_hdr_init;
2902   rdma_info->get_mac_addr = oce_get_mac_addr;
2903
2904   oce_rdma_if = rdma_if;
2905
2906   sc = softc_head;
2907   while (sc != NULL) {
2908     if (oce_rdma_if->announce != NULL) {
2909       memset(&di, 0, sizeof(di));
2910       di.dev = sc->dev;
2911       di.softc = sc;
2912       di.ifp = sc->ifp;
2913       di.db_bhandle = sc->db_bhandle;
2914       di.db_btag = sc->db_btag;
2915       di.db_page_size = 4096;
2916       if (sc->flags & OCE_FLAGS_USING_MSIX) {
2917         di.intr_mode = OCE_INTERRUPT_MODE_MSIX;
2918       } else if (sc->flags & OCE_FLAGS_USING_MSI) {
2919         di.intr_mode = OCE_INTERRUPT_MODE_MSI;
2920       } else {
2921         di.intr_mode = OCE_INTERRUPT_MODE_INTX;
2922       }
2923       di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk
2924       if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) {
2925         di.msix.num_vectors = sc->intr_count + sc->roce_intr_count;
2926         di.msix.start_vector = sc->intr_count;
2927         for (i=0; i<di.msix.num_vectors; i++) {
2928           di.msix.vector_list[i] = sc->intrs[i].vector;
2929         }
2930       } else {
2931       }
2932       memcpy(di.mac_addr, sc->macaddr.mac_addr, 6);
2933       di.vendor_id = pci_get_vendor(sc->dev);
2934       di.dev_id = pci_get_device(sc->dev);
2935
2936       if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
2937           di.flags  |= OCE_RDMA_INFO_RDMA_SUPPORTED;
2938       }
2939
2940       rdma_if->announce(&di);
2941       sc = sc->next;
2942     }
2943   }
2944
2945   return 0;
2946 }
2947
2948 static void
2949 oce_read_env_variables( POCE_SOFTC sc )
2950 {
2951         char *value = NULL;
2952         int rc = 0;
2953
2954         /* read if user wants to enable hwlro or swlro */
2955         //value = getenv("oce_enable_hwlro");
2956         if(value && IS_SH(sc)) {
2957                 sc->enable_hwlro = strtol(value, NULL, 10);
2958                 if(sc->enable_hwlro) {
2959                         rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL);
2960                         if(rc) {
2961                                 device_printf(sc->dev, "no hardware lro support\n");
2962                                 device_printf(sc->dev, "software lro enabled\n");
2963                                 sc->enable_hwlro = 0;
2964                         }else {
2965                                 device_printf(sc->dev, "hardware lro enabled\n");
2966                                 oce_max_rsp_handled = 32;
2967                         }
2968                 }else {
2969                         device_printf(sc->dev, "software lro enabled\n");
2970                 }
2971         }else {
2972                 sc->enable_hwlro = 0;
2973         }
2974
2975         /* read mbuf size */
2976         //value = getenv("oce_rq_buf_size");
2977         if(value && IS_SH(sc)) {
2978                 oce_rq_buf_size = strtol(value, NULL, 10);
2979                 switch(oce_rq_buf_size) {
2980                 case 2048:
2981                 case 4096:
2982                 case 9216:
2983                 case 16384:
2984                         break;
2985
2986                 default:
2987                         device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n");
2988                         oce_rq_buf_size = 2048;
2989                 }
2990         }
2991
2992         return;
2993 }