sys/dev/oce/oce_if.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (C) 2013 Emulex
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  *
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * 3. Neither the name of the Emulex Corporation nor the names of its
  18  *    contributors may be used to endorse or promote products derived from
  19  *    this software without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  22  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  25  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  31  * POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * Contact Information:
  34  * freebsd-drivers@emulex.com
  35  *
  36  * Emulex
  37  * 3333 Susan Street
  38  * Costa Mesa, CA 92626
  39  */
  40
  41 /* $FreeBSD$ */
  42
  43 #include "opt_inet6.h"
  44 #include "opt_inet.h"
  45
  46 #include "oce_if.h"
  47 #include "oce_user.h"
  48
  49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO)
  50
  51 /* UE Status Low CSR */
  52 static char *ue_status_low_desc[] = {
  53         "CEV",
  54         "CTX",
  55         "DBUF",
  56         "ERX",
  57         "Host",
  58         "MPU",
  59         "NDMA",
  60         "PTC ",
  61         "RDMA ",
  62         "RXF ",
  63         "RXIPS ",
  64         "RXULP0 ",
  65         "RXULP1 ",
  66         "RXULP2 ",
  67         "TIM ",
  68         "TPOST ",
  69         "TPRE ",
  70         "TXIPS ",
  71         "TXULP0 ",
  72         "TXULP1 ",
  73         "UC ",
  74         "WDMA ",
  75         "TXULP2 ",
  76         "HOST1 ",
  77         "P0_OB_LINK ",
  78         "P1_OB_LINK ",
  79         "HOST_GPIO ",
  80         "MBOX ",
  81         "AXGMAC0",
  82         "AXGMAC1",
  83         "JTAG",
  84         "MPU_INTPEND"
  85 };
  86
  87 /* UE Status High CSR */
  88 static char *ue_status_hi_desc[] = {
  89         "LPCMEMHOST",
  90         "MGMT_MAC",
  91         "PCS0ONLINE",
  92         "MPU_IRAM",
  93         "PCS1ONLINE",
  94         "PCTL0",
  95         "PCTL1",
  96         "PMEM",
  97         "RR",
  98         "TXPB",
  99         "RXPP",
 100         "XAUI",
 101         "TXP",
 102         "ARM",
 103         "IPC",
 104         "HOST2",
 105         "HOST3",
 106         "HOST4",
 107         "HOST5",
 108         "HOST6",
 109         "HOST7",
 110         "HOST8",
 111         "HOST9",
 112         "NETC",
 113         "Unknown",
 114         "Unknown",
 115         "Unknown",
 116         "Unknown",
 117         "Unknown",
 118         "Unknown",
 119         "Unknown",
 120         "Unknown"
 121 };
 122
 123 struct oce_common_cqe_info{
 124         uint8_t vtp:1;
 125         uint8_t l4_cksum_pass:1;
 126         uint8_t ip_cksum_pass:1;
 127         uint8_t ipv6_frame:1;
 128         uint8_t qnq:1;
 129         uint8_t rsvd:3;
 130         uint8_t num_frags;
 131         uint16_t pkt_size;
 132         uint16_t vtag;
 133 };
 134
 135
 136 /* Driver entry points prototypes */
 137 static int  oce_probe(device_t dev);
 138 static int  oce_attach(device_t dev);
 139 static int  oce_detach(device_t dev);
 140 static int  oce_shutdown(device_t dev);
 141 static int  oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
 142 static void oce_init(void *xsc);
 143 static int  oce_multiq_start(struct ifnet *ifp, struct mbuf *m);
 144 static void oce_multiq_flush(struct ifnet *ifp);
 145
 146 /* Driver interrupt routines protypes */
 147 static void oce_intr(void *arg, int pending);
 148 static int  oce_setup_intr(POCE_SOFTC sc);
 149 static int  oce_fast_isr(void *arg);
 150 static int  oce_alloc_intr(POCE_SOFTC sc, int vector,
 151                           void (*isr) (void *arg, int pending));
 152
 153 /* Media callbacks prototypes */
 154 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req);
 155 static int  oce_media_change(struct ifnet *ifp);
 156
 157 /* Transmit routines prototypes */
 158 static int  oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
 159 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
 160 static void oce_process_tx_completion(struct oce_wq *wq);
 161 static int  oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
 162                                  struct oce_wq *wq);
 163
 164 /* Receive routines prototypes */
 165 static int  oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
 166 static int  oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
 167 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
 168 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq);
 169 static uint16_t oce_rq_handler_lro(void *arg);
 170 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2);
 171 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2);
 172 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m);
 173
 174 /* Helper function prototypes in this file */
 175 static int  oce_attach_ifp(POCE_SOFTC sc);
 176 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
 177 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
 178 static int  oce_vid_config(POCE_SOFTC sc);
 179 static void oce_mac_addr_set(POCE_SOFTC sc);
 180 static int  oce_handle_passthrough(struct ifnet *ifp, caddr_t data);
 181 static void oce_local_timer(void *arg);
 182 static void oce_if_deactivate(POCE_SOFTC sc);
 183 static void oce_if_activate(POCE_SOFTC sc);
 184 static void setup_max_queues_want(POCE_SOFTC sc);
 185 static void update_queues_got(POCE_SOFTC sc);
 186 static void process_link_state(POCE_SOFTC sc,
 187                  struct oce_async_cqe_link_state *acqe);
 188 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
 189 static void oce_get_config(POCE_SOFTC sc);
 190 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
 191 static void oce_read_env_variables(POCE_SOFTC sc);
 192
 193
 194 /* IP specific */
 195 #if defined(INET6) || defined(INET)
 196 static int  oce_init_lro(POCE_SOFTC sc);
 197 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
 198 #endif
 199
 200 static device_method_t oce_dispatch[] = {
 201         DEVMETHOD(device_probe, oce_probe),
 202         DEVMETHOD(device_attach, oce_attach),
 203         DEVMETHOD(device_detach, oce_detach),
 204         DEVMETHOD(device_shutdown, oce_shutdown),
 205
 206         DEVMETHOD_END
 207 };
 208
 209 static driver_t oce_driver = {
 210         "oce",
 211         oce_dispatch,
 212         sizeof(OCE_SOFTC)
 213 };
 214 static devclass_t oce_devclass;
 215
 216
 217 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0);
 218 MODULE_DEPEND(oce, pci, 1, 1, 1);
 219 MODULE_DEPEND(oce, ether, 1, 1, 1);
 220 MODULE_VERSION(oce, 1);
 221
 222
 223 /* global vars */
 224 const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
 225
 226 /* Module capabilites and parameters */
 227 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
 228 uint32_t oce_enable_rss = OCE_MODCAP_RSS;
 229 uint32_t oce_rq_buf_size = 2048;
 230
 231 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
 232 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
 233
 234
 235 /* Supported devices table */
 236 static uint32_t supportedDevices[] =  {
 237         (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
 238         (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
 239         (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
 240         (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
 241         (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
 242         (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
 243 };
 244
 245 POCE_SOFTC softc_head = NULL;
 246 POCE_SOFTC softc_tail = NULL;
 247
 248 struct oce_rdma_if *oce_rdma_if = NULL;
 249
 250 /*****************************************************************************
 251  *                      Driver entry points functions                        *
 252  *****************************************************************************/
 253
 254 static int
 255 oce_probe(device_t dev)
 256 {
 257         uint16_t vendor = 0;
 258         uint16_t device = 0;
 259         int i = 0;
 260         char str[256] = {0};
 261         POCE_SOFTC sc;
 262
 263         sc = device_get_softc(dev);
 264         bzero(sc, sizeof(OCE_SOFTC));
 265         sc->dev = dev;
 266
 267         vendor = pci_get_vendor(dev);
 268         device = pci_get_device(dev);
 269
 270         for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
 271                 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
 272                         if (device == (supportedDevices[i] & 0xffff)) {
 273                                 sprintf(str, "%s:%s", "Emulex CNA NIC function",
 274                                         component_revision);
 275                                 device_set_desc_copy(dev, str);
 276
 277                                 switch (device) {
 278                                 case PCI_PRODUCT_BE2:
 279                                         sc->flags |= OCE_FLAGS_BE2;
 280                                         break;
 281                                 case PCI_PRODUCT_BE3:
 282                                         sc->flags |= OCE_FLAGS_BE3;
 283                                         break;
 284                                 case PCI_PRODUCT_XE201:
 285                                 case PCI_PRODUCT_XE201_VF:
 286                                         sc->flags |= OCE_FLAGS_XE201;
 287                                         break;
 288                                 case PCI_PRODUCT_SH:
 289                                         sc->flags |= OCE_FLAGS_SH;
 290                                         break;
 291                                 default:
 292                                         return ENXIO;
 293                                 }
 294                                 return BUS_PROBE_DEFAULT;
 295                         }
 296                 }
 297         }
 298
 299         return ENXIO;
 300 }
 301
 302
 303 static int
 304 oce_attach(device_t dev)
 305 {
 306         POCE_SOFTC sc;
 307         int rc = 0;
 308
 309         sc = device_get_softc(dev);
 310
 311         rc = oce_hw_pci_alloc(sc);
 312         if (rc)
 313                 return rc;
 314
 315         sc->tx_ring_size = OCE_TX_RING_SIZE;
 316         sc->rx_ring_size = OCE_RX_RING_SIZE;
 317         /* receive fragment size should be multiple of 2K */
 318         sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048);
 319         sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
 320         sc->promisc      = OCE_DEFAULT_PROMISCUOUS;
 321
 322         LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
 323         LOCK_CREATE(&sc->dev_lock,  "Device_lock");
 324
 325         /* initialise the hardware */
 326         rc = oce_hw_init(sc);
 327         if (rc)
 328                 goto pci_res_free;
 329
 330         oce_read_env_variables(sc);
 331
 332         oce_get_config(sc);
 333
 334         setup_max_queues_want(sc);
 335
 336         rc = oce_setup_intr(sc);
 337         if (rc)
 338                 goto mbox_free;
 339
 340         rc = oce_queue_init_all(sc);
 341         if (rc)
 342                 goto intr_free;
 343
 344         rc = oce_attach_ifp(sc);
 345         if (rc)
 346                 goto queues_free;
 347
 348 #if defined(INET6) || defined(INET)
 349         rc = oce_init_lro(sc);
 350         if (rc)
 351                 goto ifp_free;
 352 #endif
 353
 354         rc = oce_hw_start(sc);
 355         if (rc)
 356                 goto lro_free;
 357
 358         sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 359                                 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
 360         sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 361                                 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
 362
 363         rc = oce_stats_init(sc);
 364         if (rc)
 365                 goto vlan_free;
 366
 367         oce_add_sysctls(sc);
 368
 369         callout_init(&sc->timer, CALLOUT_MPSAFE);
 370         rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
 371         if (rc)
 372                 goto stats_free;
 373
 374         sc->next =NULL;
 375         if (softc_tail != NULL) {
 376           softc_tail->next = sc;
 377         } else {
 378           softc_head = sc;
 379         }
 380         softc_tail = sc;
 381
 382         return 0;
 383
 384 stats_free:
 385         callout_drain(&sc->timer);
 386         oce_stats_free(sc);
 387 vlan_free:
 388         if (sc->vlan_attach)
 389                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
 390         if (sc->vlan_detach)
 391                 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
 392         oce_hw_intr_disable(sc);
 393 lro_free:
 394 #if defined(INET6) || defined(INET)
 395         oce_free_lro(sc);
 396 ifp_free:
 397 #endif
 398         ether_ifdetach(sc->ifp);
 399         if_free(sc->ifp);
 400 queues_free:
 401         oce_queue_release_all(sc);
 402 intr_free:
 403         oce_intr_free(sc);
 404 mbox_free:
 405         oce_dma_free(sc, &sc->bsmbx);
 406 pci_res_free:
 407         oce_hw_pci_free(sc);
 408         LOCK_DESTROY(&sc->dev_lock);
 409         LOCK_DESTROY(&sc->bmbx_lock);
 410         return rc;
 411
 412 }
 413
 414
 415 static int
 416 oce_detach(device_t dev)
 417 {
 418         POCE_SOFTC sc = device_get_softc(dev);
 419         POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL;
 420
 421         poce_sc_tmp = softc_head;
 422         ppoce_sc_tmp1 = &softc_head;
 423         while (poce_sc_tmp != NULL) {
 424           if (poce_sc_tmp == sc) {
 425             *ppoce_sc_tmp1 = sc->next;
 426             if (sc->next == NULL) {
 427               softc_tail = poce_sc_tmp2;
 428             }
 429             break;
 430           }
 431           poce_sc_tmp2 = poce_sc_tmp;
 432           ppoce_sc_tmp1 = &poce_sc_tmp->next;
 433           poce_sc_tmp = poce_sc_tmp->next;
 434         }
 435
 436         LOCK(&sc->dev_lock);
 437         oce_if_deactivate(sc);
 438         UNLOCK(&sc->dev_lock);
 439
 440         callout_drain(&sc->timer);
 441
 442         if (sc->vlan_attach != NULL)
 443                 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
 444         if (sc->vlan_detach != NULL)
 445                 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
 446
 447         ether_ifdetach(sc->ifp);
 448
 449         if_free(sc->ifp);
 450
 451         oce_hw_shutdown(sc);
 452
 453         bus_generic_detach(dev);
 454
 455         return 0;
 456 }
 457
 458
 459 static int
 460 oce_shutdown(device_t dev)
 461 {
 462         int rc;
 463
 464         rc = oce_detach(dev);
 465
 466         return rc;
 467 }
 468
 469
 470 static int
 471 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 472 {
 473         struct ifreq *ifr = (struct ifreq *)data;
 474         POCE_SOFTC sc = ifp->if_softc;
 475         int rc = 0;
 476         uint32_t u;
 477
 478         switch (command) {
 479
 480         case SIOCGIFMEDIA:
 481                 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
 482                 break;
 483
 484         case SIOCSIFMTU:
 485                 if (ifr->ifr_mtu > OCE_MAX_MTU)
 486                         rc = EINVAL;
 487                 else
 488                         ifp->if_mtu = ifr->ifr_mtu;
 489                 break;
 490
 491         case SIOCSIFFLAGS:
 492                 if (ifp->if_flags & IFF_UP) {
 493                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 494                                 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
 495                                 oce_init(sc);
 496                         }
 497                         device_printf(sc->dev, "Interface Up\n");
 498                 } else {
 499                         LOCK(&sc->dev_lock);
 500
 501                         sc->ifp->if_drv_flags &=
 502                             ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 503                         oce_if_deactivate(sc);
 504
 505                         UNLOCK(&sc->dev_lock);
 506
 507                         device_printf(sc->dev, "Interface Down\n");
 508                 }
 509
 510                 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) {
 511                         if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
 512                                 sc->promisc = TRUE;
 513                 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) {
 514                         if (!oce_rxf_set_promiscuous(sc, 0))
 515                                 sc->promisc = FALSE;
 516                 }
 517
 518                 break;
 519
 520         case SIOCADDMULTI:
 521         case SIOCDELMULTI:
 522                 rc = oce_hw_update_multicast(sc);
 523                 if (rc)
 524                         device_printf(sc->dev,
 525                                 "Update multicast address failed\n");
 526                 break;
 527
 528         case SIOCSIFCAP:
 529                 u = ifr->ifr_reqcap ^ ifp->if_capenable;
 530
 531                 if (u & IFCAP_TXCSUM) {
 532                         ifp->if_capenable ^= IFCAP_TXCSUM;
 533                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
 534
 535                         if (IFCAP_TSO & ifp->if_capenable &&
 536                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
 537                                 ifp->if_capenable &= ~IFCAP_TSO;
 538                                 ifp->if_hwassist &= ~CSUM_TSO;
 539                                 if_printf(ifp,
 540                                          "TSO disabled due to -txcsum.\n");
 541                         }
 542                 }
 543
 544                 if (u & IFCAP_RXCSUM)
 545                         ifp->if_capenable ^= IFCAP_RXCSUM;
 546
 547                 if (u & IFCAP_TSO4) {
 548                         ifp->if_capenable ^= IFCAP_TSO4;
 549
 550                         if (IFCAP_TSO & ifp->if_capenable) {
 551                                 if (IFCAP_TXCSUM & ifp->if_capenable)
 552                                         ifp->if_hwassist |= CSUM_TSO;
 553                                 else {
 554                                         ifp->if_capenable &= ~IFCAP_TSO;
 555                                         ifp->if_hwassist &= ~CSUM_TSO;
 556                                         if_printf(ifp,
 557                                             "Enable txcsum first.\n");
 558                                         rc = EAGAIN;
 559                                 }
 560                         } else
 561                                 ifp->if_hwassist &= ~CSUM_TSO;
 562                 }
 563
 564                 if (u & IFCAP_VLAN_HWTAGGING)
 565                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 566
 567                 if (u & IFCAP_VLAN_HWFILTER) {
 568                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 569                         oce_vid_config(sc);
 570                 }
 571 #if defined(INET6) || defined(INET)
 572                 if (u & IFCAP_LRO) {
 573                         ifp->if_capenable ^= IFCAP_LRO;
 574                         if(sc->enable_hwlro) {
 575                                 if(ifp->if_capenable & IFCAP_LRO) {
 576                                         rc = oce_mbox_nic_set_iface_lro_config(sc, 1);
 577                                 }else {
 578                                         rc = oce_mbox_nic_set_iface_lro_config(sc, 0);
 579                                 }
 580                         }
 581                 }
 582 #endif
 583
 584                 break;
 585
 586         case SIOCGPRIVATE_0:
 587                 rc = oce_handle_passthrough(ifp, data);
 588                 break;
 589         default:
 590                 rc = ether_ioctl(ifp, command, data);
 591                 break;
 592         }
 593
 594         return rc;
 595 }
 596
 597
 598 static void
 599 oce_init(void *arg)
 600 {
 601         POCE_SOFTC sc = arg;
 602
 603         LOCK(&sc->dev_lock);
 604
 605         if (sc->ifp->if_flags & IFF_UP) {
 606                 oce_if_deactivate(sc);
 607                 oce_if_activate(sc);
 608         }
 609
 610         UNLOCK(&sc->dev_lock);
 611
 612 }
 613
 614
 615 static int
 616 oce_multiq_start(struct ifnet *ifp, struct mbuf *m)
 617 {
 618         POCE_SOFTC sc = ifp->if_softc;
 619         struct oce_wq *wq = NULL;
 620         int queue_index = 0;
 621         int status = 0;
 622
 623         if (!sc->link_status)
 624                 return ENXIO;
 625
 626         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
 627                 queue_index = m->m_pkthdr.flowid % sc->nwqs;
 628
 629         wq = sc->wq[queue_index];
 630
 631         LOCK(&wq->tx_lock);
 632         status = oce_multiq_transmit(ifp, m, wq);
 633         UNLOCK(&wq->tx_lock);
 634
 635         return status;
 636
 637 }
 638
 639
 640 static void
 641 oce_multiq_flush(struct ifnet *ifp)
 642 {
 643         POCE_SOFTC sc = ifp->if_softc;
 644         struct mbuf     *m;
 645         int i = 0;
 646
 647         for (i = 0; i < sc->nwqs; i++) {
 648                 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
 649                         m_freem(m);
 650         }
 651         if_qflush(ifp);
 652 }
 653
 654
 655
 656 /*****************************************************************************
 657  *                   Driver interrupt routines functions                     *
 658  *****************************************************************************/
 659
 660 static void
 661 oce_intr(void *arg, int pending)
 662 {
 663
 664         POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
 665         POCE_SOFTC sc = ii->sc;
 666         struct oce_eq *eq = ii->eq;
 667         struct oce_eqe *eqe;
 668         struct oce_cq *cq = NULL;
 669         int i, num_eqes = 0;
 670
 671
 672         bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
 673                                  BUS_DMASYNC_POSTWRITE);
 674         do {
 675                 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
 676                 if (eqe->evnt == 0)
 677                         break;
 678                 eqe->evnt = 0;
 679                 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
 680                                         BUS_DMASYNC_POSTWRITE);
 681                 RING_GET(eq->ring, 1);
 682                 num_eqes++;
 683
 684         } while (TRUE);
 685
 686         if (!num_eqes)
 687                 goto eq_arm; /* Spurious */
 688
 689         /* Clear EQ entries, but dont arm */
 690         oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
 691
 692         /* Process TX, RX and MCC. But dont arm CQ*/
 693         for (i = 0; i < eq->cq_valid; i++) {
 694                 cq = eq->cq[i];
 695                 (*cq->cq_handler)(cq->cb_arg);
 696         }
 697
 698         /* Arm all cqs connected to this EQ */
 699         for (i = 0; i < eq->cq_valid; i++) {
 700                 cq = eq->cq[i];
 701                 oce_arm_cq(sc, cq->cq_id, 0, TRUE);
 702         }
 703
 704 eq_arm:
 705         oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
 706
 707         return;
 708 }
 709
 710
 711 static int
 712 oce_setup_intr(POCE_SOFTC sc)
 713 {
 714         int rc = 0, use_intx = 0;
 715         int vector = 0, req_vectors = 0;
 716         int tot_req_vectors, tot_vectors;
 717
 718         if (is_rss_enabled(sc))
 719                 req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
 720         else
 721                 req_vectors = 1;
 722
 723         tot_req_vectors = req_vectors;
 724         if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
 725           if (req_vectors > 1) {
 726             tot_req_vectors += OCE_RDMA_VECTORS;
 727             sc->roce_intr_count = OCE_RDMA_VECTORS;
 728           }
 729         }
 730
 731         if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
 732                 sc->intr_count = req_vectors;
 733                 tot_vectors = tot_req_vectors;
 734                 rc = pci_alloc_msix(sc->dev, &tot_vectors);
 735                 if (rc != 0) {
 736                         use_intx = 1;
 737                         pci_release_msi(sc->dev);
 738                 } else {
 739                   if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
 740                     if (tot_vectors < tot_req_vectors) {
 741                       if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) {
 742                         sc->roce_intr_count = (tot_vectors / 2);
 743                       }
 744                       sc->intr_count = tot_vectors - sc->roce_intr_count;
 745                     }
 746                   } else {
 747                     sc->intr_count = tot_vectors;
 748                   }
 749                   sc->flags |= OCE_FLAGS_USING_MSIX;
 750                 }
 751         } else
 752                 use_intx = 1;
 753
 754         if (use_intx)
 755                 sc->intr_count = 1;
 756
 757         /* Scale number of queues based on intr we got */
 758         update_queues_got(sc);
 759
 760         if (use_intx) {
 761                 device_printf(sc->dev, "Using legacy interrupt\n");
 762                 rc = oce_alloc_intr(sc, vector, oce_intr);
 763                 if (rc)
 764                         goto error;
 765         } else {
 766                 for (; vector < sc->intr_count; vector++) {
 767                         rc = oce_alloc_intr(sc, vector, oce_intr);
 768                         if (rc)
 769                                 goto error;
 770                 }
 771         }
 772
 773         return 0;
 774 error:
 775         oce_intr_free(sc);
 776         return rc;
 777 }
 778
 779
 780 static int
 781 oce_fast_isr(void *arg)
 782 {
 783         POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
 784         POCE_SOFTC sc = ii->sc;
 785
 786         if (ii->eq == NULL)
 787                 return FILTER_STRAY;
 788
 789         oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
 790
 791         taskqueue_enqueue(ii->tq, &ii->task);
 792
 793         ii->eq->intr++;
 794
 795         return FILTER_HANDLED;
 796 }
 797
 798
 799 static int
 800 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
 801 {
 802         POCE_INTR_INFO ii = &sc->intrs[vector];
 803         int rc = 0, rr;
 804
 805         if (vector >= OCE_MAX_EQ)
 806                 return (EINVAL);
 807
 808         /* Set the resource id for the interrupt.
 809          * MSIx is vector + 1 for the resource id,
 810          * INTx is 0 for the resource id.
 811          */
 812         if (sc->flags & OCE_FLAGS_USING_MSIX)
 813                 rr = vector + 1;
 814         else
 815                 rr = 0;
 816         ii->intr_res = bus_alloc_resource_any(sc->dev,
 817                                               SYS_RES_IRQ,
 818                                               &rr, RF_ACTIVE|RF_SHAREABLE);
 819         ii->irq_rr = rr;
 820         if (ii->intr_res == NULL) {
 821                 device_printf(sc->dev,
 822                           "Could not allocate interrupt\n");
 823                 rc = ENXIO;
 824                 return rc;
 825         }
 826
 827         TASK_INIT(&ii->task, 0, isr, ii);
 828         ii->vector = vector;
 829         sprintf(ii->task_name, "oce_task[%d]", ii->vector);
 830         ii->tq = taskqueue_create_fast(ii->task_name,
 831                         M_NOWAIT,
 832                         taskqueue_thread_enqueue,
 833                         &ii->tq);
 834         taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
 835                         device_get_nameunit(sc->dev));
 836
 837         ii->sc = sc;
 838         rc = bus_setup_intr(sc->dev,
 839                         ii->intr_res,
 840                         INTR_TYPE_NET,
 841                         oce_fast_isr, NULL, ii, &ii->tag);
 842         return rc;
 843
 844 }
 845
 846
 847 void
 848 oce_intr_free(POCE_SOFTC sc)
 849 {
 850         int i = 0;
 851
 852         for (i = 0; i < sc->intr_count; i++) {
 853
 854                 if (sc->intrs[i].tag != NULL)
 855                         bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
 856                                                 sc->intrs[i].tag);
 857                 if (sc->intrs[i].tq != NULL)
 858                         taskqueue_free(sc->intrs[i].tq);
 859
 860                 if (sc->intrs[i].intr_res != NULL)
 861                         bus_release_resource(sc->dev, SYS_RES_IRQ,
 862                                                 sc->intrs[i].irq_rr,
 863                                                 sc->intrs[i].intr_res);
 864                 sc->intrs[i].tag = NULL;
 865                 sc->intrs[i].intr_res = NULL;
 866         }
 867
 868         if (sc->flags & OCE_FLAGS_USING_MSIX)
 869                 pci_release_msi(sc->dev);
 870
 871 }
 872
 873
 874
 875 /******************************************************************************
 876 *                         Media callbacks functions                           *
 877 ******************************************************************************/
 878
 879 static void
 880 oce_media_status(struct ifnet *ifp, struct ifmediareq *req)
 881 {
 882         POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc;
 883
 884
 885         req->ifm_status = IFM_AVALID;
 886         req->ifm_active = IFM_ETHER;
 887
 888         if (sc->link_status == 1)
 889                 req->ifm_status |= IFM_ACTIVE;
 890         else
 891                 return;
 892
 893         switch (sc->link_speed) {
 894         case 1: /* 10 Mbps */
 895                 req->ifm_active |= IFM_10_T | IFM_FDX;
 896                 sc->speed = 10;
 897                 break;
 898         case 2: /* 100 Mbps */
 899                 req->ifm_active |= IFM_100_TX | IFM_FDX;
 900                 sc->speed = 100;
 901                 break;
 902         case 3: /* 1 Gbps */
 903                 req->ifm_active |= IFM_1000_T | IFM_FDX;
 904                 sc->speed = 1000;
 905                 break;
 906         case 4: /* 10 Gbps */
 907                 req->ifm_active |= IFM_10G_SR | IFM_FDX;
 908                 sc->speed = 10000;
 909                 break;
 910         case 5: /* 20 Gbps */
 911                 req->ifm_active |= IFM_10G_SR | IFM_FDX;
 912                 sc->speed = 20000;
 913                 break;
 914         case 6: /* 25 Gbps */
 915                 req->ifm_active |= IFM_10G_SR | IFM_FDX;
 916                 sc->speed = 25000;
 917                 break;
 918         case 7: /* 40 Gbps */
 919                 req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
 920                 sc->speed = 40000;
 921                 break;
 922         default:
 923                 sc->speed = 0;
 924                 break;
 925         }
 926
 927         return;
 928 }
 929
 930
 931 int
 932 oce_media_change(struct ifnet *ifp)
 933 {
 934         return 0;
 935 }
 936
 937
 938 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc,
 939                                 struct mbuf *m, boolean_t *os2bmc,
 940                                 struct mbuf **m_new)
 941 {
 942         struct ether_header *eh = NULL;
 943
 944         eh = mtod(m, struct ether_header *);
 945
 946         if (!is_os2bmc_enabled(sc) || *os2bmc) {
 947                 *os2bmc = FALSE;
 948                 goto done;
 949         }
 950         if (!ETHER_IS_MULTICAST(eh->ether_dhost))
 951                 goto done;
 952
 953         if (is_mc_allowed_on_bmc(sc, eh) ||
 954             is_bc_allowed_on_bmc(sc, eh) ||
 955             is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) {
 956                 *os2bmc = TRUE;
 957                 goto done;
 958         }
 959
 960         if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) {
 961                 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
 962                 uint8_t nexthdr = ip6->ip6_nxt;
 963                 if (nexthdr == IPPROTO_ICMPV6) {
 964                         struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1);
 965                         switch (icmp6->icmp6_type) {
 966                         case ND_ROUTER_ADVERT:
 967                                 *os2bmc = is_ipv6_ra_filt_enabled(sc);
 968                                 goto done;
 969                         case ND_NEIGHBOR_ADVERT:
 970                                 *os2bmc = is_ipv6_na_filt_enabled(sc);
 971                                 goto done;
 972                         default:
 973                                 break;
 974                         }
 975                 }
 976         }
 977
 978         if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) {
 979                 struct ip *ip = mtod(m, struct ip *);
 980                 int iphlen = ip->ip_hl << 2;
 981                 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen);
 982                 switch (uh->uh_dport) {
 983                 case DHCP_CLIENT_PORT:
 984                         *os2bmc = is_dhcp_client_filt_enabled(sc);
 985                         goto done;
 986                 case DHCP_SERVER_PORT:
 987                         *os2bmc = is_dhcp_srvr_filt_enabled(sc);
 988                         goto done;
 989                 case NET_BIOS_PORT1:
 990                 case NET_BIOS_PORT2:
 991                         *os2bmc = is_nbios_filt_enabled(sc);
 992                         goto done;
 993                 case DHCPV6_RAS_PORT:
 994                         *os2bmc = is_ipv6_ras_filt_enabled(sc);
 995                         goto done;
 996                 default:
 997                         break;
 998                 }
 999         }
1000 done:
1001         if (*os2bmc) {
1002                 *m_new = m_dup(m, M_NOWAIT);
1003                 if (!*m_new) {
1004                         *os2bmc = FALSE;
1005                         return;
1006                 }
1007                 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL);
1008         }
1009 }
1010
1011
1012
1013 /*****************************************************************************
1014  *                        Transmit routines functions                        *
1015  *****************************************************************************/
1016
1017 static int
1018 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
1019 {
1020         int rc = 0, i, retry_cnt = 0;
1021         bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
1022         struct mbuf *m, *m_temp, *m_new = NULL;
1023         struct oce_wq *wq = sc->wq[wq_index];
1024         struct oce_packet_desc *pd;
1025         struct oce_nic_hdr_wqe *nichdr;
1026         struct oce_nic_frag_wqe *nicfrag;
1027         struct ether_header *eh = NULL;
1028         int num_wqes;
1029         uint32_t reg_value;
1030         boolean_t complete = TRUE;
1031         boolean_t os2bmc = FALSE;
1032
1033         m = *mpp;
1034         if (!m)
1035                 return EINVAL;
1036
1037         if (!(m->m_flags & M_PKTHDR)) {
1038                 rc = ENXIO;
1039                 goto free_ret;
1040         }
1041
1042         /* Don't allow non-TSO packets longer than MTU */
1043         if (!is_tso_pkt(m)) {
1044                 eh = mtod(m, struct ether_header *);
1045                 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE))
1046                          goto free_ret;
1047         }
1048
1049         if(oce_tx_asic_stall_verify(sc, m)) {
1050                 m = oce_insert_vlan_tag(sc, m, &complete);
1051                 if(!m) {
1052                         device_printf(sc->dev, "Insertion unsuccessful\n");
1053                         return 0;
1054                 }
1055
1056         }
1057
1058         /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less
1059          * may cause a transmit stall on that port. So the work-around is to
1060          * pad short packets (<= 32 bytes) to a 36-byte length.
1061         */
1062         if(IS_SH(sc) || IS_XE201(sc) ) {
1063                 if(m->m_pkthdr.len <= 32) {
1064                         char buf[36];
1065                         bzero((void *)buf, 36);
1066                         m_append(m, (36 - m->m_pkthdr.len), buf);
1067                 }
1068         }
1069
1070 tx_start:
1071         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1072                 /* consolidate packet buffers for TSO/LSO segment offload */
1073 #if defined(INET6) || defined(INET)
1074                 m = oce_tso_setup(sc, mpp);
1075 #else
1076                 m = NULL;
1077 #endif
1078                 if (m == NULL) {
1079                         rc = ENXIO;
1080                         goto free_ret;
1081                 }
1082         }
1083
1084
1085         pd = &wq->pckts[wq->pkt_desc_head];
1086
1087 retry:
1088         rc = bus_dmamap_load_mbuf_sg(wq->tag,
1089                                      pd->map,
1090                                      m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
1091         if (rc == 0) {
1092                 num_wqes = pd->nsegs + 1;
1093                 if (IS_BE(sc) || IS_SH(sc)) {
1094                         /*Dummy required only for BE3.*/
1095                         if (num_wqes & 1)
1096                                 num_wqes++;
1097                 }
1098                 if (num_wqes >= RING_NUM_FREE(wq->ring)) {
1099                         bus_dmamap_unload(wq->tag, pd->map);
1100                         return EBUSY;
1101                 }
1102                 atomic_store_rel_int(&wq->pkt_desc_head,
1103                                      (wq->pkt_desc_head + 1) % \
1104                                       OCE_WQ_PACKET_ARRAY_SIZE);
1105                 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
1106                 pd->mbuf = m;
1107
1108                 nichdr =
1109                     RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
1110                 nichdr->u0.dw[0] = 0;
1111                 nichdr->u0.dw[1] = 0;
1112                 nichdr->u0.dw[2] = 0;
1113                 nichdr->u0.dw[3] = 0;
1114
1115                 nichdr->u0.s.complete = complete;
1116                 nichdr->u0.s.mgmt = os2bmc;
1117                 nichdr->u0.s.event = 1;
1118                 nichdr->u0.s.crc = 1;
1119                 nichdr->u0.s.forward = 0;
1120                 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
1121                 nichdr->u0.s.udpcs =
1122                         (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
1123                 nichdr->u0.s.tcpcs =
1124                         (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
1125                 nichdr->u0.s.num_wqe = num_wqes;
1126                 nichdr->u0.s.total_length = m->m_pkthdr.len;
1127
1128                 if (m->m_flags & M_VLANTAG) {
1129                         nichdr->u0.s.vlan = 1; /*Vlan present*/
1130                         nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
1131                 }
1132
1133                 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1134                         if (m->m_pkthdr.tso_segsz) {
1135                                 nichdr->u0.s.lso = 1;
1136                                 nichdr->u0.s.lso_mss  = m->m_pkthdr.tso_segsz;
1137                         }
1138                         if (!IS_BE(sc) || !IS_SH(sc))
1139                                 nichdr->u0.s.ipcs = 1;
1140                 }
1141
1142                 RING_PUT(wq->ring, 1);
1143                 atomic_add_int(&wq->ring->num_used, 1);
1144
1145                 for (i = 0; i < pd->nsegs; i++) {
1146                         nicfrag =
1147                             RING_GET_PRODUCER_ITEM_VA(wq->ring,
1148                                                       struct oce_nic_frag_wqe);
1149                         nicfrag->u0.s.rsvd0 = 0;
1150                         nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
1151                         nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
1152                         nicfrag->u0.s.frag_len = segs[i].ds_len;
1153                         pd->wqe_idx = wq->ring->pidx;
1154                         RING_PUT(wq->ring, 1);
1155                         atomic_add_int(&wq->ring->num_used, 1);
1156                 }
1157                 if (num_wqes > (pd->nsegs + 1)) {
1158                         nicfrag =
1159                             RING_GET_PRODUCER_ITEM_VA(wq->ring,
1160                                                       struct oce_nic_frag_wqe);
1161                         nicfrag->u0.dw[0] = 0;
1162                         nicfrag->u0.dw[1] = 0;
1163                         nicfrag->u0.dw[2] = 0;
1164                         nicfrag->u0.dw[3] = 0;
1165                         pd->wqe_idx = wq->ring->pidx;
1166                         RING_PUT(wq->ring, 1);
1167                         atomic_add_int(&wq->ring->num_used, 1);
1168                         pd->nsegs++;
1169                 }
1170
1171                 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
1172                 wq->tx_stats.tx_reqs++;
1173                 wq->tx_stats.tx_wrbs += num_wqes;
1174                 wq->tx_stats.tx_bytes += m->m_pkthdr.len;
1175                 wq->tx_stats.tx_pkts++;
1176
1177                 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
1178                                 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1179                 reg_value = (num_wqes << 16) | wq->wq_id;
1180
1181                 /* if os2bmc is not enabled or if the pkt is already tagged as
1182                    bmc, do nothing
1183                  */
1184                 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new);
1185
1186                 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
1187
1188         } else if (rc == EFBIG) {
1189                 if (retry_cnt == 0) {
1190                         m_temp = m_defrag(m, M_NOWAIT);
1191                         if (m_temp == NULL)
1192                                 goto free_ret;
1193                         m = m_temp;
1194                         *mpp = m_temp;
1195                         retry_cnt = retry_cnt + 1;
1196                         goto retry;
1197                 } else
1198                         goto free_ret;
1199         } else if (rc == ENOMEM)
1200                 return rc;
1201         else
1202                 goto free_ret;
1203
1204         if (os2bmc) {
1205                 m = m_new;
1206                 goto tx_start;
1207         }
1208
1209         return 0;
1210
1211 free_ret:
1212         m_freem(*mpp);
1213         *mpp = NULL;
1214         return rc;
1215 }
1216
1217
1218 static void
1219 oce_process_tx_completion(struct oce_wq *wq)
1220 {
1221         struct oce_packet_desc *pd;
1222         POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
1223         struct mbuf *m;
1224
1225         pd = &wq->pckts[wq->pkt_desc_tail];
1226         atomic_store_rel_int(&wq->pkt_desc_tail,
1227                              (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
1228         atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
1229         bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1230         bus_dmamap_unload(wq->tag, pd->map);
1231
1232         m = pd->mbuf;
1233         m_freem(m);
1234         pd->mbuf = NULL;
1235
1236
1237         if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) {
1238                 if (wq->ring->num_used < (wq->ring->num_items / 2)) {
1239                         sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
1240                         oce_tx_restart(sc, wq);
1241                 }
1242         }
1243 }
1244
1245
1246 static void
1247 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
1248 {
1249
1250         if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
1251                 return;
1252
1253 #if __FreeBSD_version >= 800000
1254         if (!drbr_empty(sc->ifp, wq->br))
1255 #else
1256         if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd))
1257 #endif
1258                 taskqueue_enqueue(taskqueue_swi, &wq->txtask);
1259
1260 }
1261
1262
1263 #if defined(INET6) || defined(INET)
1264 static struct mbuf *
1265 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
1266 {
1267         struct mbuf *m;
1268 #ifdef INET
1269         struct ip *ip;
1270 #endif
1271 #ifdef INET6
1272         struct ip6_hdr *ip6;
1273 #endif
1274         struct ether_vlan_header *eh;
1275         struct tcphdr *th;
1276         uint16_t etype;
1277         int total_len = 0, ehdrlen = 0;
1278
1279         m = *mpp;
1280
1281         if (M_WRITABLE(m) == 0) {
1282                 m = m_dup(*mpp, M_NOWAIT);
1283                 if (!m)
1284                         return NULL;
1285                 m_freem(*mpp);
1286                 *mpp = m;
1287         }
1288
1289         eh = mtod(m, struct ether_vlan_header *);
1290         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1291                 etype = ntohs(eh->evl_proto);
1292                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1293         } else {
1294                 etype = ntohs(eh->evl_encap_proto);
1295                 ehdrlen = ETHER_HDR_LEN;
1296         }
1297
1298         switch (etype) {
1299 #ifdef INET
1300         case ETHERTYPE_IP:
1301                 ip = (struct ip *)(m->m_data + ehdrlen);
1302                 if (ip->ip_p != IPPROTO_TCP)
1303                         return NULL;
1304                 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
1305
1306                 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
1307                 break;
1308 #endif
1309 #ifdef INET6
1310         case ETHERTYPE_IPV6:
1311                 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
1312                 if (ip6->ip6_nxt != IPPROTO_TCP)
1313                         return NULL;
1314                 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
1315
1316                 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
1317                 break;
1318 #endif
1319         default:
1320                 return NULL;
1321         }
1322
1323         m = m_pullup(m, total_len);
1324         if (!m)
1325                 return NULL;
1326         *mpp = m;
1327         return m;
1328
1329 }
1330 #endif /* INET6 || INET */
1331
1332 void
1333 oce_tx_task(void *arg, int npending)
1334 {
1335         struct oce_wq *wq = arg;
1336         POCE_SOFTC sc = wq->parent;
1337         struct ifnet *ifp = sc->ifp;
1338         int rc = 0;
1339
1340 #if __FreeBSD_version >= 800000
1341         LOCK(&wq->tx_lock);
1342         rc = oce_multiq_transmit(ifp, NULL, wq);
1343         if (rc) {
1344                 device_printf(sc->dev,
1345                                 "TX[%d] restart failed\n", wq->queue_index);
1346         }
1347         UNLOCK(&wq->tx_lock);
1348 #else
1349         oce_start(ifp);
1350 #endif
1351
1352 }
1353
1354
1355 void
1356 oce_start(struct ifnet *ifp)
1357 {
1358         POCE_SOFTC sc = ifp->if_softc;
1359         struct mbuf *m;
1360         int rc = 0;
1361         int def_q = 0; /* Defualt tx queue is 0*/
1362
1363         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1364                         IFF_DRV_RUNNING)
1365                 return;
1366
1367         if (!sc->link_status)
1368                 return;
1369
1370         do {
1371                 IF_DEQUEUE(&sc->ifp->if_snd, m);
1372                 if (m == NULL)
1373                         break;
1374
1375                 LOCK(&sc->wq[def_q]->tx_lock);
1376                 rc = oce_tx(sc, &m, def_q);
1377                 UNLOCK(&sc->wq[def_q]->tx_lock);
1378                 if (rc) {
1379                         if (m != NULL) {
1380                                 sc->wq[def_q]->tx_stats.tx_stops ++;
1381                                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1382                                 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1383                                 m = NULL;
1384                         }
1385                         break;
1386                 }
1387                 if (m != NULL)
1388                         ETHER_BPF_MTAP(ifp, m);
1389
1390         } while (TRUE);
1391
1392         return;
1393 }
1394
1395
1396 /* Handle the Completion Queue for transmit */
1397 uint16_t
1398 oce_wq_handler(void *arg)
1399 {
1400         struct oce_wq *wq = (struct oce_wq *)arg;
1401         POCE_SOFTC sc = wq->parent;
1402         struct oce_cq *cq = wq->cq;
1403         struct oce_nic_tx_cqe *cqe;
1404         int num_cqes = 0;
1405
1406         LOCK(&wq->tx_compl_lock);
1407         bus_dmamap_sync(cq->ring->dma.tag,
1408                         cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1409         cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1410         while (cqe->u0.dw[3]) {
1411                 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
1412
1413                 wq->ring->cidx = cqe->u0.s.wqe_index + 1;
1414                 if (wq->ring->cidx >= wq->ring->num_items)
1415                         wq->ring->cidx -= wq->ring->num_items;
1416
1417                 oce_process_tx_completion(wq);
1418                 wq->tx_stats.tx_compl++;
1419                 cqe->u0.dw[3] = 0;
1420                 RING_GET(cq->ring, 1);
1421                 bus_dmamap_sync(cq->ring->dma.tag,
1422                                 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1423                 cqe =
1424                     RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1425                 num_cqes++;
1426         }
1427
1428         if (num_cqes)
1429                 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
1430
1431         UNLOCK(&wq->tx_compl_lock);
1432         return num_cqes;
1433 }
1434
1435
1436 static int
1437 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq)
1438 {
1439         POCE_SOFTC sc = ifp->if_softc;
1440         int status = 0, queue_index = 0;
1441         struct mbuf *next = NULL;
1442         struct buf_ring *br = NULL;
1443
1444         br  = wq->br;
1445         queue_index = wq->queue_index;
1446
1447         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1448                 IFF_DRV_RUNNING) {
1449                 if (m != NULL)
1450                         status = drbr_enqueue(ifp, br, m);
1451                 return status;
1452         }
1453
1454         if (m != NULL) {
1455                 if ((status = drbr_enqueue(ifp, br, m)) != 0)
1456                         return status;
1457         }
1458         while ((next = drbr_peek(ifp, br)) != NULL) {
1459                 if (oce_tx(sc, &next, queue_index)) {
1460                         if (next == NULL) {
1461                                 drbr_advance(ifp, br);
1462                         } else {
1463                                 drbr_putback(ifp, br, next);
1464                                 wq->tx_stats.tx_stops ++;
1465                                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1466                         }
1467                         break;
1468                 }
1469                 drbr_advance(ifp, br);
1470                 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len);
1471                 if (next->m_flags & M_MCAST)
1472                         if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
1473                 ETHER_BPF_MTAP(ifp, next);
1474         }
1475
1476         return 0;
1477 }
1478
1479
1480
1481
1482 /*****************************************************************************
1483  *                          Receive  routines functions                      *
1484  *****************************************************************************/
1485
1486 static void
1487 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2)
1488 {
1489         uint32_t *p;
1490         struct ether_header *eh = NULL;
1491         struct tcphdr *tcp_hdr = NULL;
1492         struct ip *ip4_hdr = NULL;
1493         struct ip6_hdr *ip6 = NULL;
1494         uint32_t payload_len = 0;
1495
1496         eh = mtod(m, struct ether_header *);
1497         /* correct IP header */
1498         if(!cqe2->ipv6_frame) {
1499                 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header));
1500                 ip4_hdr->ip_ttl = cqe2->frame_lifespan;
1501                 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header));
1502                 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip));
1503         }else {
1504                 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header));
1505                 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan;
1506                 payload_len = cqe2->coalesced_size - sizeof(struct ether_header)
1507                                                 - sizeof(struct ip6_hdr);
1508                 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len);
1509                 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr));
1510         }
1511
1512         /* correct tcp header */
1513         tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num);
1514         if(cqe2->push) {
1515                 tcp_hdr->th_flags |= TH_PUSH;
1516         }
1517         tcp_hdr->th_win = htons(cqe2->tcp_window);
1518         tcp_hdr->th_sum = 0xffff;
1519         if(cqe2->ts_opt) {
1520                 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2);
1521                 *p = cqe1->tcp_timestamp_val;
1522                 *(p+1) = cqe1->tcp_timestamp_ecr;
1523         }
1524
1525         return;
1526 }
1527
1528 static void
1529 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m)
1530 {
1531         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1532         uint32_t i = 0, frag_len = 0;
1533         uint32_t len = cqe_info->pkt_size;
1534         struct oce_packet_desc *pd;
1535         struct mbuf *tail = NULL;
1536
1537         for (i = 0; i < cqe_info->num_frags; i++) {
1538                 if (rq->ring->cidx == rq->ring->pidx) {
1539                         device_printf(sc->dev,
1540                                   "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n");
1541                         return;
1542                 }
1543                 pd = &rq->pckts[rq->ring->cidx];
1544
1545                 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1546                 bus_dmamap_unload(rq->tag, pd->map);
1547                 RING_GET(rq->ring, 1);
1548                 rq->pending--;
1549
1550                 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
1551                 pd->mbuf->m_len = frag_len;
1552
1553                 if (tail != NULL) {
1554                         /* additional fragments */
1555                         pd->mbuf->m_flags &= ~M_PKTHDR;
1556                         tail->m_next = pd->mbuf;
1557                         if(rq->islro)
1558                                 tail->m_nextpkt = NULL;
1559                         tail = pd->mbuf;
1560                 } else {
1561                         /* first fragment, fill out much of the packet header */
1562                         pd->mbuf->m_pkthdr.len = len;
1563                         if(rq->islro)
1564                                 pd->mbuf->m_nextpkt = NULL;
1565                         pd->mbuf->m_pkthdr.csum_flags = 0;
1566                         if (IF_CSUM_ENABLED(sc)) {
1567                                 if (cqe_info->l4_cksum_pass) {
1568                                         if(!cqe_info->ipv6_frame) { /* IPV4 */
1569                                                 pd->mbuf->m_pkthdr.csum_flags |=
1570                                                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1571                                         }else { /* IPV6 frame */
1572                                                 if(rq->islro) {
1573                                                         pd->mbuf->m_pkthdr.csum_flags |=
1574                                                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1575                                                 }
1576                                         }
1577                                         pd->mbuf->m_pkthdr.csum_data = 0xffff;
1578                                 }
1579                                 if (cqe_info->ip_cksum_pass) {
1580                                         pd->mbuf->m_pkthdr.csum_flags |=
1581                                                (CSUM_IP_CHECKED|CSUM_IP_VALID);
1582                                 }
1583                         }
1584                         *m = tail = pd->mbuf;
1585                }
1586                 pd->mbuf = NULL;
1587                 len -= frag_len;
1588         }
1589
1590         return;
1591 }
1592
1593 static void
1594 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2)
1595 {
1596         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1597         struct nic_hwlro_cqe_part1 *cqe1 = NULL;
1598         struct mbuf *m = NULL;
1599         struct oce_common_cqe_info cq_info;
1600
1601         /* parse cqe */
1602         if(cqe2 == NULL) {
1603                 cq_info.pkt_size =  cqe->pkt_size;
1604                 cq_info.vtag = cqe->vlan_tag;
1605                 cq_info.l4_cksum_pass = cqe->l4_cksum_pass;
1606                 cq_info.ip_cksum_pass = cqe->ip_cksum_pass;
1607                 cq_info.ipv6_frame = cqe->ipv6_frame;
1608                 cq_info.vtp = cqe->vtp;
1609                 cq_info.qnq = cqe->qnq;
1610         }else {
1611                 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe;
1612                 cq_info.pkt_size =  cqe2->coalesced_size;
1613                 cq_info.vtag = cqe2->vlan_tag;
1614                 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass;
1615                 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass;
1616                 cq_info.ipv6_frame = cqe2->ipv6_frame;
1617                 cq_info.vtp = cqe2->vtp;
1618                 cq_info.qnq = cqe1->qnq;
1619         }
1620
1621         cq_info.vtag = BSWAP_16(cq_info.vtag);
1622
1623         cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size;
1624         if(cq_info.pkt_size % rq->cfg.frag_size)
1625                 cq_info.num_frags++;
1626
1627         oce_rx_mbuf_chain(rq, &cq_info, &m);
1628
1629         if (m) {
1630                 if(cqe2) {
1631                         //assert(cqe2->valid != 0);
1632
1633                         //assert(cqe2->cqe_type != 2);
1634                         oce_correct_header(m, cqe1, cqe2);
1635                 }
1636
1637                 m->m_pkthdr.rcvif = sc->ifp;
1638 #if __FreeBSD_version >= 800000
1639                 if (rq->queue_index)
1640                         m->m_pkthdr.flowid = (rq->queue_index - 1);
1641                 else
1642                         m->m_pkthdr.flowid = rq->queue_index;
1643                 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1644 #endif
1645                 /* This deternies if vlan tag is Valid */
1646                 if (cq_info.vtp) {
1647                         if (sc->function_mode & FNM_FLEX10_MODE) {
1648                                 /* FLEX10. If QnQ is not set, neglect VLAN */
1649                                 if (cq_info.qnq) {
1650                                         m->m_pkthdr.ether_vtag = cq_info.vtag;
1651                                         m->m_flags |= M_VLANTAG;
1652                                 }
1653                         } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK))  {
1654                                 /* In UMC mode generally pvid will be striped by
1655                                    hw. But in some cases we have seen it comes
1656                                    with pvid. So if pvid == vlan, neglect vlan.
1657                                  */
1658                                 m->m_pkthdr.ether_vtag = cq_info.vtag;
1659                                 m->m_flags |= M_VLANTAG;
1660                         }
1661                 }
1662                 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1663
1664                 (*sc->ifp->if_input) (sc->ifp, m);
1665
1666                 /* Update rx stats per queue */
1667                 rq->rx_stats.rx_pkts++;
1668                 rq->rx_stats.rx_bytes += cq_info.pkt_size;
1669                 rq->rx_stats.rx_frags += cq_info.num_frags;
1670                 rq->rx_stats.rx_ucast_pkts++;
1671         }
1672         return;
1673 }
1674
1675 static void
1676 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
1677 {
1678         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1679         int len;
1680         struct mbuf *m = NULL;
1681         struct oce_common_cqe_info cq_info;
1682         uint16_t vtag = 0;
1683
1684         /* Is it a flush compl that has no data */
1685         if(!cqe->u0.s.num_fragments)
1686                 goto exit;
1687
1688         len = cqe->u0.s.pkt_size;
1689         if (!len) {
1690                 /*partial DMA workaround for Lancer*/
1691                 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1692                 goto exit;
1693         }
1694
1695         if (!oce_cqe_portid_valid(sc, cqe)) {
1696                 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1697                 goto exit;
1698         }
1699
1700          /* Get vlan_tag value */
1701         if(IS_BE(sc) || IS_SH(sc))
1702                 vtag = BSWAP_16(cqe->u0.s.vlan_tag);
1703         else
1704                 vtag = cqe->u0.s.vlan_tag;
1705
1706         cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass;
1707         cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass;
1708         cq_info.ipv6_frame = cqe->u0.s.ip_ver;
1709         cq_info.num_frags = cqe->u0.s.num_fragments;
1710         cq_info.pkt_size = cqe->u0.s.pkt_size;
1711
1712         oce_rx_mbuf_chain(rq, &cq_info, &m);
1713
1714         if (m) {
1715                 m->m_pkthdr.rcvif = sc->ifp;
1716 #if __FreeBSD_version >= 800000
1717                 if (rq->queue_index)
1718                         m->m_pkthdr.flowid = (rq->queue_index - 1);
1719                 else
1720                         m->m_pkthdr.flowid = rq->queue_index;
1721                 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1722 #endif
1723                 /* This deternies if vlan tag is Valid */
1724                 if (oce_cqe_vtp_valid(sc, cqe)) {
1725                         if (sc->function_mode & FNM_FLEX10_MODE) {
1726                                 /* FLEX10. If QnQ is not set, neglect VLAN */
1727                                 if (cqe->u0.s.qnq) {
1728                                         m->m_pkthdr.ether_vtag = vtag;
1729                                         m->m_flags |= M_VLANTAG;
1730                                 }
1731                         } else if (sc->pvid != (vtag & VLAN_VID_MASK))  {
1732                                 /* In UMC mode generally pvid will be striped by
1733                                    hw. But in some cases we have seen it comes
1734                                    with pvid. So if pvid == vlan, neglect vlan.
1735                                 */
1736                                 m->m_pkthdr.ether_vtag = vtag;
1737                                 m->m_flags |= M_VLANTAG;
1738                         }
1739                 }
1740
1741                 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1742 #if defined(INET6) || defined(INET)
1743                 /* Try to queue to LRO */
1744                 if (IF_LRO_ENABLED(sc) &&
1745                     (cqe->u0.s.ip_cksum_pass) &&
1746                     (cqe->u0.s.l4_cksum_pass) &&
1747                     (!cqe->u0.s.ip_ver)       &&
1748                     (rq->lro.lro_cnt != 0)) {
1749
1750                         if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
1751                                 rq->lro_pkts_queued ++;
1752                                 goto post_done;
1753                         }
1754                         /* If LRO posting fails then try to post to STACK */
1755                 }
1756 #endif
1757
1758                 (*sc->ifp->if_input) (sc->ifp, m);
1759 #if defined(INET6) || defined(INET)
1760 post_done:
1761 #endif
1762                 /* Update rx stats per queue */
1763                 rq->rx_stats.rx_pkts++;
1764                 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
1765                 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
1766                 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
1767                         rq->rx_stats.rx_mcast_pkts++;
1768                 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
1769                         rq->rx_stats.rx_ucast_pkts++;
1770         }
1771 exit:
1772         return;
1773 }
1774
1775
1776 void
1777 oce_discard_rx_comp(struct oce_rq *rq, int num_frags)
1778 {
1779         uint32_t i = 0;
1780         struct oce_packet_desc *pd;
1781         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1782
1783         for (i = 0; i < num_frags; i++) {
1784                 if (rq->ring->cidx == rq->ring->pidx) {
1785                         device_printf(sc->dev,
1786                                 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n");
1787                         return;
1788                 }
1789                 pd = &rq->pckts[rq->ring->cidx];
1790                 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1791                 bus_dmamap_unload(rq->tag, pd->map);
1792                 if (pd->mbuf != NULL) {
1793                         m_freem(pd->mbuf);
1794                         pd->mbuf = NULL;
1795                 }
1796
1797                 RING_GET(rq->ring, 1);
1798                 rq->pending--;
1799         }
1800 }
1801
1802
1803 static int
1804 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1805 {
1806         struct oce_nic_rx_cqe_v1 *cqe_v1;
1807         int vtp = 0;
1808
1809         if (sc->be3_native) {
1810                 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1811                 vtp =  cqe_v1->u0.s.vlan_tag_present;
1812         } else
1813                 vtp = cqe->u0.s.vlan_tag_present;
1814
1815         return vtp;
1816
1817 }
1818
1819
1820 static int
1821 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1822 {
1823         struct oce_nic_rx_cqe_v1 *cqe_v1;
1824         int port_id = 0;
1825
1826         if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
1827                 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1828                 port_id =  cqe_v1->u0.s.port;
1829                 if (sc->port_id != port_id)
1830                         return 0;
1831         } else
1832                 ;/* For BE3 legacy and Lancer this is dummy */
1833
1834         return 1;
1835
1836 }
1837
1838 #if defined(INET6) || defined(INET)
1839 void
1840 oce_rx_flush_lro(struct oce_rq *rq)
1841 {
1842         struct lro_ctrl *lro = &rq->lro;
1843         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1844
1845         if (!IF_LRO_ENABLED(sc))
1846                 return;
1847
1848         tcp_lro_flush_all(lro);
1849         rq->lro_pkts_queued = 0;
1850
1851         return;
1852 }
1853
1854
1855 static int
1856 oce_init_lro(POCE_SOFTC sc)
1857 {
1858         struct lro_ctrl *lro = NULL;
1859         int i = 0, rc = 0;
1860
1861         for (i = 0; i < sc->nrqs; i++) {
1862                 lro = &sc->rq[i]->lro;
1863                 rc = tcp_lro_init(lro);
1864                 if (rc != 0) {
1865                         device_printf(sc->dev, "LRO init failed\n");
1866                         return rc;
1867                 }
1868                 lro->ifp = sc->ifp;
1869         }
1870
1871         return rc;
1872 }
1873
1874
1875 void
1876 oce_free_lro(POCE_SOFTC sc)
1877 {
1878         struct lro_ctrl *lro = NULL;
1879         int i = 0;
1880
1881         for (i = 0; i < sc->nrqs; i++) {
1882                 lro = &sc->rq[i]->lro;
1883                 if (lro)
1884                         tcp_lro_free(lro);
1885         }
1886 }
1887 #endif
1888
1889 int
1890 oce_alloc_rx_bufs(struct oce_rq *rq, int count)
1891 {
1892         POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1893         int i, in, rc;
1894         struct oce_packet_desc *pd;
1895         bus_dma_segment_t segs[6];
1896         int nsegs, added = 0;
1897         struct oce_nic_rqe *rqe;
1898         pd_rxulp_db_t rxdb_reg;
1899         uint32_t val = 0;
1900         uint32_t oce_max_rq_posts = 64;
1901
1902         bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
1903         for (i = 0; i < count; i++) {
1904                 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE;
1905
1906                 pd = &rq->pckts[rq->ring->pidx];
1907                 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size);
1908                 if (pd->mbuf == NULL) {
1909                         device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size);
1910                         break;
1911                 }
1912                 pd->mbuf->m_nextpkt = NULL;
1913
1914                 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size;
1915
1916                 rc = bus_dmamap_load_mbuf_sg(rq->tag,
1917                                              pd->map,
1918                                              pd->mbuf,
1919                                              segs, &nsegs, BUS_DMA_NOWAIT);
1920                 if (rc) {
1921                         m_free(pd->mbuf);
1922                         device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc);
1923                         break;
1924                 }
1925
1926                 if (nsegs != 1) {
1927                         i--;
1928                         continue;
1929                 }
1930
1931                 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
1932
1933                 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
1934                 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
1935                 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
1936                 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
1937                 RING_PUT(rq->ring, 1);
1938                 added++;
1939                 rq->pending++;
1940         }
1941         oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS;
1942         if (added != 0) {
1943                 for (i = added / oce_max_rq_posts; i > 0; i--) {
1944                         rxdb_reg.bits.num_posted = oce_max_rq_posts;
1945                         rxdb_reg.bits.qid = rq->rq_id;
1946                         if(rq->islro) {
1947                                 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1948                                 val |= oce_max_rq_posts << 16;
1949                                 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1950                         }else {
1951                                 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1952                         }
1953                         added -= oce_max_rq_posts;
1954                 }
1955                 if (added > 0) {
1956                         rxdb_reg.bits.qid = rq->rq_id;
1957                         rxdb_reg.bits.num_posted = added;
1958                         if(rq->islro) {
1959                                 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1960                                 val |= added << 16;
1961                                 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1962                         }else {
1963                                 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1964                         }
1965                 }
1966         }
1967
1968         return 0;
1969 }
1970
1971 static void
1972 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq)
1973 {
1974         if (num_cqes) {
1975                 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE);
1976                 if(!sc->enable_hwlro) {
1977                         if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1)
1978                                 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1));
1979                 }else {
1980                         if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64)
1981                                 oce_alloc_rx_bufs(rq, 64);
1982                 }
1983         }
1984
1985         return;
1986 }
1987
1988 uint16_t
1989 oce_rq_handler_lro(void *arg)
1990 {
1991         struct oce_rq *rq = (struct oce_rq *)arg;
1992         struct oce_cq *cq = rq->cq;
1993         POCE_SOFTC sc = rq->parent;
1994         struct nic_hwlro_singleton_cqe *cqe;
1995         struct nic_hwlro_cqe_part2 *cqe2;
1996         int num_cqes = 0;
1997
1998         LOCK(&rq->rx_lock);
1999         bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2000         cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2001         while (cqe->valid) {
2002                 if(cqe->cqe_type == 0) { /* singleton cqe */
2003                         /* we should not get singleton cqe after cqe1 on same rq */
2004                         if(rq->cqe_firstpart != NULL) {
2005                                 device_printf(sc->dev, "Got singleton cqe after cqe1 \n");
2006                                 goto exit_rq_handler_lro;
2007                         }
2008                         if(cqe->error != 0) {
2009                                 rq->rx_stats.rxcp_err++;
2010                                 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2011                         }
2012                         oce_rx_lro(rq, cqe, NULL);
2013                         rq->rx_stats.rx_compl++;
2014                         cqe->valid = 0;
2015                         RING_GET(cq->ring, 1);
2016                         num_cqes++;
2017                         if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2018                                 break;
2019                 }else if(cqe->cqe_type == 0x1) { /* first part */
2020                         /* we should not get cqe1 after cqe1 on same rq */
2021                         if(rq->cqe_firstpart != NULL) {
2022                                 device_printf(sc->dev, "Got cqe1 after cqe1 \n");
2023                                 goto exit_rq_handler_lro;
2024                         }
2025                         rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe;
2026                         RING_GET(cq->ring, 1);
2027                 }else if(cqe->cqe_type == 0x2) { /* second part */
2028                         cqe2 = (struct nic_hwlro_cqe_part2 *)cqe;
2029                         if(cqe2->error != 0) {
2030                                 rq->rx_stats.rxcp_err++;
2031                                 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2032                         }
2033                         /* We should not get cqe2 without cqe1 */
2034                         if(rq->cqe_firstpart == NULL) {
2035                                 device_printf(sc->dev, "Got cqe2 without cqe1 \n");
2036                                 goto exit_rq_handler_lro;
2037                         }
2038                         oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2);
2039
2040                         rq->rx_stats.rx_compl++;
2041                         rq->cqe_firstpart->valid = 0;
2042                         cqe2->valid = 0;
2043                         rq->cqe_firstpart = NULL;
2044
2045                         RING_GET(cq->ring, 1);
2046                         num_cqes += 2;
2047                         if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2048                                 break;
2049                 }
2050
2051                 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2052                 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2053         }
2054         oce_check_rx_bufs(sc, num_cqes, rq);
2055 exit_rq_handler_lro:
2056         UNLOCK(&rq->rx_lock);
2057         return 0;
2058 }
2059
2060 /* Handle the Completion Queue for receive */
2061 uint16_t
2062 oce_rq_handler(void *arg)
2063 {
2064         struct oce_rq *rq = (struct oce_rq *)arg;
2065         struct oce_cq *cq = rq->cq;
2066         POCE_SOFTC sc = rq->parent;
2067         struct oce_nic_rx_cqe *cqe;
2068         int num_cqes = 0;
2069
2070         if(rq->islro) {
2071                 oce_rq_handler_lro(arg);
2072                 return 0;
2073         }
2074         LOCK(&rq->rx_lock);
2075         bus_dmamap_sync(cq->ring->dma.tag,
2076                         cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2077         cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2078         while (cqe->u0.dw[2]) {
2079                 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
2080
2081                 if (cqe->u0.s.error == 0) {
2082                         oce_rx(rq, cqe);
2083                 } else {
2084                         rq->rx_stats.rxcp_err++;
2085                         if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2086                         /* Post L3/L4 errors to stack.*/
2087                         oce_rx(rq, cqe);
2088                 }
2089                 rq->rx_stats.rx_compl++;
2090                 cqe->u0.dw[2] = 0;
2091
2092 #if defined(INET6) || defined(INET)
2093                 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
2094                         oce_rx_flush_lro(rq);
2095                 }
2096 #endif
2097
2098                 RING_GET(cq->ring, 1);
2099                 bus_dmamap_sync(cq->ring->dma.tag,
2100                                 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2101                 cqe =
2102                     RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2103                 num_cqes++;
2104                 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2105                         break;
2106         }
2107
2108 #if defined(INET6) || defined(INET)
2109         if (IF_LRO_ENABLED(sc))
2110                 oce_rx_flush_lro(rq);
2111 #endif
2112
2113         oce_check_rx_bufs(sc, num_cqes, rq);
2114         UNLOCK(&rq->rx_lock);
2115         return 0;
2116
2117 }
2118
2119
2120
2121
2122 /*****************************************************************************
2123  *                 Helper function prototypes in this file                   *
2124  *****************************************************************************/
2125
2126 static int
2127 oce_attach_ifp(POCE_SOFTC sc)
2128 {
2129
2130         sc->ifp = if_alloc(IFT_ETHER);
2131         if (!sc->ifp)
2132                 return ENOMEM;
2133
2134         ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
2135         ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2136         ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2137
2138         sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST;
2139         sc->ifp->if_ioctl = oce_ioctl;
2140         sc->ifp->if_start = oce_start;
2141         sc->ifp->if_init = oce_init;
2142         sc->ifp->if_mtu = ETHERMTU;
2143         sc->ifp->if_softc = sc;
2144 #if __FreeBSD_version >= 800000
2145         sc->ifp->if_transmit = oce_multiq_start;
2146         sc->ifp->if_qflush = oce_multiq_flush;
2147 #endif
2148
2149         if_initname(sc->ifp,
2150                     device_get_name(sc->dev), device_get_unit(sc->dev));
2151
2152         sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1;
2153         IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen);
2154         IFQ_SET_READY(&sc->ifp->if_snd);
2155
2156         sc->ifp->if_hwassist = OCE_IF_HWASSIST;
2157         sc->ifp->if_hwassist |= CSUM_TSO;
2158         sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP);
2159
2160         sc->ifp->if_capabilities = OCE_IF_CAPABILITIES;
2161         sc->ifp->if_capabilities |= IFCAP_HWCSUM;
2162         sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2163
2164 #if defined(INET6) || defined(INET)
2165         sc->ifp->if_capabilities |= IFCAP_TSO;
2166         sc->ifp->if_capabilities |= IFCAP_LRO;
2167         sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
2168 #endif
2169
2170         sc->ifp->if_capenable = sc->ifp->if_capabilities;
2171         sc->ifp->if_baudrate = IF_Gbps(10);
2172
2173 #if __FreeBSD_version >= 1000000
2174         sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2175         sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS;
2176         sc->ifp->if_hw_tsomaxsegsize = 4096;
2177 #endif
2178
2179         ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
2180
2181         return 0;
2182 }
2183
2184
2185 static void
2186 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2187 {
2188         POCE_SOFTC sc = ifp->if_softc;
2189
2190         if (ifp->if_softc !=  arg)
2191                 return;
2192         if ((vtag == 0) || (vtag > 4095))
2193                 return;
2194
2195         sc->vlan_tag[vtag] = 1;
2196         sc->vlans_added++;
2197         if (sc->vlans_added <= (sc->max_vlans + 1))
2198                 oce_vid_config(sc);
2199 }
2200
2201
2202 static void
2203 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2204 {
2205         POCE_SOFTC sc = ifp->if_softc;
2206
2207         if (ifp->if_softc !=  arg)
2208                 return;
2209         if ((vtag == 0) || (vtag > 4095))
2210                 return;
2211
2212         sc->vlan_tag[vtag] = 0;
2213         sc->vlans_added--;
2214         oce_vid_config(sc);
2215 }
2216
2217
2218 /*
2219  * A max of 64 vlans can be configured in BE. If the user configures
2220  * more, place the card in vlan promiscuous mode.
2221  */
2222 static int
2223 oce_vid_config(POCE_SOFTC sc)
2224 {
2225         struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
2226         uint16_t ntags = 0, i;
2227         int status = 0;
2228
2229         if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
2230                         (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) {
2231                 for (i = 0; i < MAX_VLANS; i++) {
2232                         if (sc->vlan_tag[i]) {
2233                                 vtags[ntags].vtag = i;
2234                                 ntags++;
2235                         }
2236                 }
2237                 if (ntags)
2238                         status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2239                                                 vtags, ntags, 1, 0);
2240         } else
2241                 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2242                                                 NULL, 0, 1, 1);
2243         return status;
2244 }
2245
2246
2247 static void
2248 oce_mac_addr_set(POCE_SOFTC sc)
2249 {
2250         uint32_t old_pmac_id = sc->pmac_id;
2251         int status = 0;
2252
2253
2254         status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2255                          sc->macaddr.size_of_struct);
2256         if (!status)
2257                 return;
2258
2259         status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)),
2260                                         sc->if_id, &sc->pmac_id);
2261         if (!status) {
2262                 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
2263                 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2264                                  sc->macaddr.size_of_struct);
2265         }
2266         if (status)
2267                 device_printf(sc->dev, "Failed update macaddress\n");
2268
2269 }
2270
2271
2272 static int
2273 oce_handle_passthrough(struct ifnet *ifp, caddr_t data)
2274 {
2275         POCE_SOFTC sc = ifp->if_softc;
2276         struct ifreq *ifr = (struct ifreq *)data;
2277         int rc = ENXIO;
2278         char cookie[32] = {0};
2279         void *priv_data = ifr_data_get_ptr(ifr);
2280         void *ioctl_ptr;
2281         uint32_t req_size;
2282         struct mbx_hdr req;
2283         OCE_DMA_MEM dma_mem;
2284         struct mbx_common_get_cntl_attr *fw_cmd;
2285
2286         if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
2287                 return EFAULT;
2288
2289         if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
2290                 return EINVAL;
2291
2292         ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
2293         if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
2294                 return EFAULT;
2295
2296         req_size = le32toh(req.u0.req.request_length);
2297         if (req_size > 65536)
2298                 return EINVAL;
2299
2300         req_size += sizeof(struct mbx_hdr);
2301         rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
2302         if (rc)
2303                 return ENOMEM;
2304
2305         if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
2306                 rc = EFAULT;
2307                 goto dma_free;
2308         }
2309
2310         rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
2311         if (rc) {
2312                 rc = EIO;
2313                 goto dma_free;
2314         }
2315
2316         if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size))
2317                 rc =  EFAULT;
2318
2319         /*
2320            firmware is filling all the attributes for this ioctl except
2321            the driver version..so fill it
2322          */
2323         if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
2324                 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr;
2325                 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
2326                         COMPONENT_REVISION, strlen(COMPONENT_REVISION));
2327         }
2328
2329 dma_free:
2330         oce_dma_free(sc, &dma_mem);
2331         return rc;
2332
2333 }
2334
2335 static void
2336 oce_eqd_set_periodic(POCE_SOFTC sc)
2337 {
2338         struct oce_set_eqd set_eqd[OCE_MAX_EQ];
2339         struct oce_aic_obj *aic;
2340         struct oce_eq *eqo;
2341         uint64_t now = 0, delta;
2342         int eqd, i, num = 0;
2343         uint32_t tx_reqs = 0, rxpkts = 0, pps;
2344         struct oce_wq *wq;
2345         struct oce_rq *rq;
2346
2347         #define ticks_to_msecs(t)       (1000 * (t) / hz)
2348
2349         for (i = 0 ; i < sc->neqs; i++) {
2350                 eqo = sc->eq[i];
2351                 aic = &sc->aic_obj[i];
2352                 /* When setting the static eq delay from the user space */
2353                 if (!aic->enable) {
2354                         if (aic->ticks)
2355                                 aic->ticks = 0;
2356                         eqd = aic->et_eqd;
2357                         goto modify_eqd;
2358                 }
2359
2360                 rq = sc->rq[i];
2361                 rxpkts = rq->rx_stats.rx_pkts;
2362                 wq = sc->wq[i];
2363                 tx_reqs = wq->tx_stats.tx_reqs;
2364                 now = ticks;
2365
2366                 if (!aic->ticks || now < aic->ticks ||
2367                     rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) {
2368                         aic->prev_rxpkts = rxpkts;
2369                         aic->prev_txreqs = tx_reqs;
2370                         aic->ticks = now;
2371                         continue;
2372                 }
2373
2374                 delta = ticks_to_msecs(now - aic->ticks);
2375
2376                 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) +
2377                       (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta);
2378                 eqd = (pps / 15000) << 2;
2379                 if (eqd < 8)
2380                         eqd = 0;
2381
2382                 /* Make sure that the eq delay is in the known range */
2383                 eqd = min(eqd, aic->max_eqd);
2384                 eqd = max(eqd, aic->min_eqd);
2385
2386                 aic->prev_rxpkts = rxpkts;
2387                 aic->prev_txreqs = tx_reqs;
2388                 aic->ticks = now;
2389
2390 modify_eqd:
2391                 if (eqd != aic->cur_eqd) {
2392                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
2393                         set_eqd[num].eq_id = eqo->eq_id;
2394                         aic->cur_eqd = eqd;
2395                         num++;
2396                 }
2397         }
2398
2399         /* Is there atleast one eq that needs to be modified? */
2400         for(i = 0; i < num; i += 8) {
2401                 if((num - i) >=8 )
2402                         oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8);
2403                 else
2404                         oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i));
2405         }
2406
2407 }
2408
2409 static void oce_detect_hw_error(POCE_SOFTC sc)
2410 {
2411
2412         uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
2413         uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2414         uint32_t i;
2415
2416         if (sc->hw_error)
2417                 return;
2418
2419         if (IS_XE201(sc)) {
2420                 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
2421                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2422                         sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
2423                         sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
2424                 }
2425         } else {
2426                 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
2427                 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
2428                 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
2429                 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
2430
2431                 ue_low = (ue_low & ~ue_low_mask);
2432                 ue_high = (ue_high & ~ue_high_mask);
2433         }
2434
2435         /* On certain platforms BE hardware can indicate spurious UEs.
2436          * Allow the h/w to stop working completely in case of a real UE.
2437          * Hence not setting the hw_error for UE detection.
2438          */
2439         if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2440                 sc->hw_error = TRUE;
2441                 device_printf(sc->dev, "Error detected in the card\n");
2442         }
2443
2444         if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2445                 device_printf(sc->dev,
2446                                 "ERR: sliport status 0x%x\n", sliport_status);
2447                 device_printf(sc->dev,
2448                                 "ERR: sliport error1 0x%x\n", sliport_err1);
2449                 device_printf(sc->dev,
2450                                 "ERR: sliport error2 0x%x\n", sliport_err2);
2451         }
2452
2453         if (ue_low) {
2454                 for (i = 0; ue_low; ue_low >>= 1, i++) {
2455                         if (ue_low & 1)
2456                                 device_printf(sc->dev, "UE: %s bit set\n",
2457                                                         ue_status_low_desc[i]);
2458                 }
2459         }
2460
2461         if (ue_high) {
2462                 for (i = 0; ue_high; ue_high >>= 1, i++) {
2463                         if (ue_high & 1)
2464                                 device_printf(sc->dev, "UE: %s bit set\n",
2465                                                         ue_status_hi_desc[i]);
2466                 }
2467         }
2468
2469 }
2470
2471
2472 static void
2473 oce_local_timer(void *arg)
2474 {
2475         POCE_SOFTC sc = arg;
2476         int i = 0;
2477
2478         oce_detect_hw_error(sc);
2479         oce_refresh_nic_stats(sc);
2480         oce_refresh_queue_stats(sc);
2481         oce_mac_addr_set(sc);
2482
2483         /* TX Watch Dog*/
2484         for (i = 0; i < sc->nwqs; i++)
2485                 oce_tx_restart(sc, sc->wq[i]);
2486
2487         /* calculate and set the eq delay for optimal interrupt rate */
2488         if (IS_BE(sc) || IS_SH(sc))
2489                 oce_eqd_set_periodic(sc);
2490
2491         callout_reset(&sc->timer, hz, oce_local_timer, sc);
2492 }
2493
2494 static void
2495 oce_tx_compl_clean(POCE_SOFTC sc)
2496 {
2497         struct oce_wq *wq;
2498         int i = 0, timeo = 0, num_wqes = 0;
2499         int pending_txqs = sc->nwqs;
2500
2501         /* Stop polling for compls when HW has been silent for 10ms or
2502          * hw_error or no outstanding completions expected
2503          */
2504         do {
2505                 pending_txqs = sc->nwqs;
2506
2507                 for_all_wq_queues(sc, wq, i) {
2508                         num_wqes = oce_wq_handler(wq);
2509
2510                         if(num_wqes)
2511                                 timeo = 0;
2512
2513                         if(!wq->ring->num_used)
2514                                 pending_txqs--;
2515                 }
2516
2517                 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error)
2518                         break;
2519
2520                 DELAY(1000);
2521         } while (TRUE);
2522
2523         for_all_wq_queues(sc, wq, i) {
2524                 while(wq->ring->num_used) {
2525                         LOCK(&wq->tx_compl_lock);
2526                         oce_process_tx_completion(wq);
2527                         UNLOCK(&wq->tx_compl_lock);
2528                 }
2529         }
2530
2531 }
2532
2533 /* NOTE : This should only be called holding
2534  *        DEVICE_LOCK.
2535  */
2536 static void
2537 oce_if_deactivate(POCE_SOFTC sc)
2538 {
2539         int i;
2540         struct oce_rq *rq;
2541         struct oce_wq *wq;
2542         struct oce_eq *eq;
2543
2544         sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2545
2546         oce_tx_compl_clean(sc);
2547
2548         /* Stop intrs and finish any bottom halves pending */
2549         oce_hw_intr_disable(sc);
2550
2551         /* Since taskqueue_drain takes a Gaint Lock, We should not acquire
2552            any other lock. So unlock device lock and require after
2553            completing taskqueue_drain.
2554         */
2555         UNLOCK(&sc->dev_lock);
2556         for (i = 0; i < sc->intr_count; i++) {
2557                 if (sc->intrs[i].tq != NULL) {
2558                         taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
2559                 }
2560         }
2561         LOCK(&sc->dev_lock);
2562
2563         /* Delete RX queue in card with flush param */
2564         oce_stop_rx(sc);
2565
2566         /* Invalidate any pending cq and eq entries*/
2567         for_all_evnt_queues(sc, eq, i)
2568                 oce_drain_eq(eq);
2569         for_all_rq_queues(sc, rq, i)
2570                 oce_drain_rq_cq(rq);
2571         for_all_wq_queues(sc, wq, i)
2572                 oce_drain_wq_cq(wq);
2573
2574         /* But still we need to get MCC aync events.
2575            So enable intrs and also arm first EQ
2576         */
2577         oce_hw_intr_enable(sc);
2578         oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
2579
2580         DELAY(10);
2581 }
2582
2583
2584 static void
2585 oce_if_activate(POCE_SOFTC sc)
2586 {
2587         struct oce_eq *eq;
2588         struct oce_rq *rq;
2589         struct oce_wq *wq;
2590         int i, rc = 0;
2591
2592         sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
2593
2594         oce_hw_intr_disable(sc);
2595
2596         oce_start_rx(sc);
2597
2598         for_all_rq_queues(sc, rq, i) {
2599                 rc = oce_start_rq(rq);
2600                 if (rc)
2601                         device_printf(sc->dev, "Unable to start RX\n");
2602         }
2603
2604         for_all_wq_queues(sc, wq, i) {
2605                 rc = oce_start_wq(wq);
2606                 if (rc)
2607                         device_printf(sc->dev, "Unable to start TX\n");
2608         }
2609
2610
2611         for_all_evnt_queues(sc, eq, i)
2612                 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
2613
2614         oce_hw_intr_enable(sc);
2615
2616 }
2617
2618 static void
2619 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
2620 {
2621         /* Update Link status */
2622         if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
2623              ASYNC_EVENT_LINK_UP) {
2624                 sc->link_status = ASYNC_EVENT_LINK_UP;
2625                 if_link_state_change(sc->ifp, LINK_STATE_UP);
2626         } else {
2627                 sc->link_status = ASYNC_EVENT_LINK_DOWN;
2628                 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2629         }
2630 }
2631
2632
2633 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc,
2634                                          struct oce_async_evt_grp5_os2bmc *evt)
2635 {
2636         DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc));
2637         if (evt->u.s.mgmt_enable)
2638                 sc->flags |= OCE_FLAGS_OS2BMC;
2639         else
2640                 return;
2641
2642         sc->bmc_filt_mask = evt->u.s.arp_filter;
2643         sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1);
2644         sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2);
2645         sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3);
2646         sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4);
2647         sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5);
2648         sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6);
2649         sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7);
2650         sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8);
2651 }
2652
2653
2654 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe)
2655 {
2656         struct oce_async_event_grp5_pvid_state *gcqe;
2657         struct oce_async_evt_grp5_os2bmc *bmccqe;
2658
2659         switch (cqe->u0.s.async_type) {
2660         case ASYNC_EVENT_PVID_STATE:
2661                 /* GRP5 PVID */
2662                 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe;
2663                 if (gcqe->enabled)
2664                         sc->pvid = gcqe->tag & VLAN_VID_MASK;
2665                 else
2666                         sc->pvid = 0;
2667                 break;
2668         case ASYNC_EVENT_OS2BMC:
2669                 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe;
2670                 oce_async_grp5_osbmc_process(sc, bmccqe);
2671                 break;
2672         default:
2673                 break;
2674         }
2675 }
2676
2677 /* Handle the Completion Queue for the Mailbox/Async notifications */
2678 uint16_t
2679 oce_mq_handler(void *arg)
2680 {
2681         struct oce_mq *mq = (struct oce_mq *)arg;
2682         POCE_SOFTC sc = mq->parent;
2683         struct oce_cq *cq = mq->cq;
2684         int num_cqes = 0, evt_type = 0, optype = 0;
2685         struct oce_mq_cqe *cqe;
2686         struct oce_async_cqe_link_state *acqe;
2687         struct oce_async_event_qnq *dbgcqe;
2688
2689
2690         bus_dmamap_sync(cq->ring->dma.tag,
2691                         cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2692         cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2693
2694         while (cqe->u0.dw[3]) {
2695                 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
2696                 if (cqe->u0.s.async_event) {
2697                         evt_type = cqe->u0.s.event_type;
2698                         optype = cqe->u0.s.async_type;
2699                         if (evt_type  == ASYNC_EVENT_CODE_LINK_STATE) {
2700                                 /* Link status evt */
2701                                 acqe = (struct oce_async_cqe_link_state *)cqe;
2702                                 process_link_state(sc, acqe);
2703                         } else if (evt_type == ASYNC_EVENT_GRP5) {
2704                                 oce_process_grp5_events(sc, cqe);
2705                         } else if (evt_type == ASYNC_EVENT_CODE_DEBUG &&
2706                                         optype == ASYNC_EVENT_DEBUG_QNQ) {
2707                                 dbgcqe =  (struct oce_async_event_qnq *)cqe;
2708                                 if(dbgcqe->valid)
2709                                         sc->qnqid = dbgcqe->vlan_tag;
2710                                 sc->qnq_debug_event = TRUE;
2711                         }
2712                 }
2713                 cqe->u0.dw[3] = 0;
2714                 RING_GET(cq->ring, 1);
2715                 bus_dmamap_sync(cq->ring->dma.tag,
2716                                 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2717                 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2718                 num_cqes++;
2719         }
2720
2721         if (num_cqes)
2722                 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
2723
2724         return 0;
2725 }
2726
2727
2728 static void
2729 setup_max_queues_want(POCE_SOFTC sc)
2730 {
2731         /* Check if it is FLEX machine. Is so dont use RSS */
2732         if ((sc->function_mode & FNM_FLEX10_MODE) ||
2733             (sc->function_mode & FNM_UMC_MODE)    ||
2734             (sc->function_mode & FNM_VNIC_MODE)   ||
2735             (!is_rss_enabled(sc))                 ||
2736             IS_BE2(sc)) {
2737                 sc->nrqs = 1;
2738                 sc->nwqs = 1;
2739         } else {
2740                 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2741                 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
2742         }
2743
2744         if (IS_BE2(sc) && is_rss_enabled(sc))
2745                 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2746 }
2747
2748
2749 static void
2750 update_queues_got(POCE_SOFTC sc)
2751 {
2752         if (is_rss_enabled(sc)) {
2753                 sc->nrqs = sc->intr_count + 1;
2754                 sc->nwqs = sc->intr_count;
2755         } else {
2756                 sc->nrqs = 1;
2757                 sc->nwqs = 1;
2758         }
2759
2760         if (IS_BE2(sc))
2761                 sc->nwqs = 1;
2762 }
2763
2764 static int
2765 oce_check_ipv6_ext_hdr(struct mbuf *m)
2766 {
2767         struct ether_header *eh = mtod(m, struct ether_header *);
2768         caddr_t m_datatemp = m->m_data;
2769
2770         if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
2771                 m->m_data += sizeof(struct ether_header);
2772                 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
2773
2774                 if((ip6->ip6_nxt != IPPROTO_TCP) && \
2775                                 (ip6->ip6_nxt != IPPROTO_UDP)){
2776                         struct ip6_ext *ip6e = NULL;
2777                         m->m_data += sizeof(struct ip6_hdr);
2778
2779                         ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
2780                         if(ip6e->ip6e_len == 0xff) {
2781                                 m->m_data = m_datatemp;
2782                                 return TRUE;
2783                         }
2784                 }
2785                 m->m_data = m_datatemp;
2786         }
2787         return FALSE;
2788 }
2789
2790 static int
2791 is_be3_a1(POCE_SOFTC sc)
2792 {
2793         if((sc->flags & OCE_FLAGS_BE3)  && ((sc->asic_revision & 0xFF) < 2)) {
2794                 return TRUE;
2795         }
2796         return FALSE;
2797 }
2798
2799 static struct mbuf *
2800 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
2801 {
2802         uint16_t vlan_tag = 0;
2803
2804         if(!M_WRITABLE(m))
2805                 return NULL;
2806
2807         /* Embed vlan tag in the packet if it is not part of it */
2808         if(m->m_flags & M_VLANTAG) {
2809                 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
2810                 m->m_flags &= ~M_VLANTAG;
2811         }
2812
2813         /* if UMC, ignore vlan tag insertion and instead insert pvid */
2814         if(sc->pvid) {
2815                 if(!vlan_tag)
2816                         vlan_tag = sc->pvid;
2817                 if (complete)
2818                         *complete = FALSE;
2819         }
2820
2821         if(vlan_tag) {
2822                 m = ether_vlanencap(m, vlan_tag);
2823         }
2824
2825         if(sc->qnqid) {
2826                 m = ether_vlanencap(m, sc->qnqid);
2827
2828                 if (complete)
2829                         *complete = FALSE;
2830         }
2831         return m;
2832 }
2833
2834 static int
2835 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
2836 {
2837         if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
2838                         oce_check_ipv6_ext_hdr(m)) {
2839                 return TRUE;
2840         }
2841         return FALSE;
2842 }
2843
2844 static void
2845 oce_get_config(POCE_SOFTC sc)
2846 {
2847         int rc = 0;
2848         uint32_t max_rss = 0;
2849
2850         if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
2851                 max_rss = OCE_LEGACY_MODE_RSS;
2852         else
2853                 max_rss = OCE_MAX_RSS;
2854
2855         if (!IS_BE(sc)) {
2856                 rc = oce_get_profile_config(sc, max_rss);
2857                 if (rc) {
2858                         sc->nwqs = OCE_MAX_WQ;
2859                         sc->nrssqs = max_rss;
2860                         sc->nrqs = sc->nrssqs + 1;
2861                 }
2862         }
2863         else { /* For BE3 don't rely on fw for determining the resources */
2864                 sc->nrssqs = max_rss;
2865                 sc->nrqs = sc->nrssqs + 1;
2866                 sc->nwqs = OCE_MAX_WQ;
2867                 sc->max_vlans = MAX_VLANFILTER_SIZE;
2868         }
2869 }
2870
2871 static void
2872 oce_rdma_close(void)
2873 {
2874   if (oce_rdma_if != NULL) {
2875     oce_rdma_if = NULL;
2876   }
2877 }
2878
2879 static void
2880 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr)
2881 {
2882   memcpy(macaddr, sc->macaddr.mac_addr, 6);
2883 }
2884
2885 int
2886 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if)
2887 {
2888   POCE_SOFTC sc;
2889   struct oce_dev_info di;
2890   int i;
2891
2892   if ((rdma_info == NULL) || (rdma_if == NULL)) {
2893     return -EINVAL;
2894   }
2895
2896   if ((rdma_info->size != OCE_RDMA_INFO_SIZE) ||
2897       (rdma_if->size != OCE_RDMA_IF_SIZE)) {
2898     return -ENXIO;
2899   }
2900
2901   rdma_info->close = oce_rdma_close;
2902   rdma_info->mbox_post = oce_mbox_post;
2903   rdma_info->common_req_hdr_init = mbx_common_req_hdr_init;
2904   rdma_info->get_mac_addr = oce_get_mac_addr;
2905
2906   oce_rdma_if = rdma_if;
2907
2908   sc = softc_head;
2909   while (sc != NULL) {
2910     if (oce_rdma_if->announce != NULL) {
2911       memset(&di, 0, sizeof(di));
2912       di.dev = sc->dev;
2913       di.softc = sc;
2914       di.ifp = sc->ifp;
2915       di.db_bhandle = sc->db_bhandle;
2916       di.db_btag = sc->db_btag;
2917       di.db_page_size = 4096;
2918       if (sc->flags & OCE_FLAGS_USING_MSIX) {
2919         di.intr_mode = OCE_INTERRUPT_MODE_MSIX;
2920       } else if (sc->flags & OCE_FLAGS_USING_MSI) {
2921         di.intr_mode = OCE_INTERRUPT_MODE_MSI;
2922       } else {
2923         di.intr_mode = OCE_INTERRUPT_MODE_INTX;
2924       }
2925       di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk
2926       if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) {
2927         di.msix.num_vectors = sc->intr_count + sc->roce_intr_count;
2928         di.msix.start_vector = sc->intr_count;
2929         for (i=0; i<di.msix.num_vectors; i++) {
2930           di.msix.vector_list[i] = sc->intrs[i].vector;
2931         }
2932       } else {
2933       }
2934       memcpy(di.mac_addr, sc->macaddr.mac_addr, 6);
2935       di.vendor_id = pci_get_vendor(sc->dev);
2936       di.dev_id = pci_get_device(sc->dev);
2937
2938       if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
2939           di.flags  |= OCE_RDMA_INFO_RDMA_SUPPORTED;
2940       }
2941
2942       rdma_if->announce(&di);
2943       sc = sc->next;
2944     }
2945   }
2946
2947   return 0;
2948 }
2949
2950 static void
2951 oce_read_env_variables( POCE_SOFTC sc )
2952 {
2953         char *value = NULL;
2954         int rc = 0;
2955
2956         /* read if user wants to enable hwlro or swlro */
2957         //value = getenv("oce_enable_hwlro");
2958         if(value && IS_SH(sc)) {
2959                 sc->enable_hwlro = strtol(value, NULL, 10);
2960                 if(sc->enable_hwlro) {
2961                         rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL);
2962                         if(rc) {
2963                                 device_printf(sc->dev, "no hardware lro support\n");
2964                                 device_printf(sc->dev, "software lro enabled\n");
2965                                 sc->enable_hwlro = 0;
2966                         }else {
2967                                 device_printf(sc->dev, "hardware lro enabled\n");
2968                                 oce_max_rsp_handled = 32;
2969                         }
2970                 }else {
2971                         device_printf(sc->dev, "software lro enabled\n");
2972                 }
2973         }else {
2974                 sc->enable_hwlro = 0;
2975         }
2976
2977         /* read mbuf size */
2978         //value = getenv("oce_rq_buf_size");
2979         if(value && IS_SH(sc)) {
2980                 oce_rq_buf_size = strtol(value, NULL, 10);
2981                 switch(oce_rq_buf_size) {
2982                 case 2048:
2983                 case 4096:
2984                 case 9216:
2985                 case 16384:
2986                         break;
2987
2988                 default:
2989                         device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n");
2990                         oce_rq_buf_size = 2048;
2991                 }
2992         }
2993
2994         return;
2995 }