sys/dev/qlxgbe/ql_os.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2013-2016 Qlogic Corporation
   5  * All rights reserved.
   6  *
   7  *  Redistribution and use in source and binary forms, with or without
   8  *  modification, are permitted provided that the following conditions
   9  *  are met:
  10  *
  11  *  1. Redistributions of source code must retain the above copyright
  12  *     notice, this list of conditions and the following disclaimer.
  13  *  2. Redistributions in binary form must reproduce the above copyright
  14  *     notice, this list of conditions and the following disclaimer in the
  15  *     documentation and/or other materials provided with the distribution.
  16  *
  17  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18  *  and ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  21  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  22  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  25  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27  *  POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * File: ql_os.c
  32  * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include "ql_os.h"
  39 #include "ql_hw.h"
  40 #include "ql_def.h"
  41 #include "ql_inline.h"
  42 #include "ql_ver.h"
  43 #include "ql_glbl.h"
  44 #include "ql_dbg.h"
  45 #include <sys/smp.h>
  46
  47 /*
  48  * Some PCI Configuration Space Related Defines
  49  */
  50
  51 #ifndef PCI_VENDOR_QLOGIC
  52 #define PCI_VENDOR_QLOGIC       0x1077
  53 #endif
  54
  55 #ifndef PCI_PRODUCT_QLOGIC_ISP8030
  56 #define PCI_PRODUCT_QLOGIC_ISP8030      0x8030
  57 #endif
  58
  59 #define PCI_QLOGIC_ISP8030 \
  60         ((PCI_PRODUCT_QLOGIC_ISP8030 << 16) | PCI_VENDOR_QLOGIC)
  61
  62 /*
  63  * static functions
  64  */
  65 static int qla_alloc_parent_dma_tag(qla_host_t *ha);
  66 static void qla_free_parent_dma_tag(qla_host_t *ha);
  67 static int qla_alloc_xmt_bufs(qla_host_t *ha);
  68 static void qla_free_xmt_bufs(qla_host_t *ha);
  69 static int qla_alloc_rcv_bufs(qla_host_t *ha);
  70 static void qla_free_rcv_bufs(qla_host_t *ha);
  71 static void qla_clear_tx_buf(qla_host_t *ha, qla_tx_buf_t *txb);
  72
  73 static void qla_init_ifnet(device_t dev, qla_host_t *ha);
  74 static int qla_sysctl_get_link_status(SYSCTL_HANDLER_ARGS);
  75 static void qla_release(qla_host_t *ha);
  76 static void qla_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
  77                 int error);
  78 static void qla_stop(qla_host_t *ha);
  79 static void qla_get_peer(qla_host_t *ha);
  80 static void qla_error_recovery(void *context, int pending);
  81 static void qla_async_event(void *context, int pending);
  82 static void qla_stats(void *context, int pending);
  83 static int qla_send(qla_host_t *ha, struct mbuf **m_headp, uint32_t txr_idx,
  84                 uint32_t iscsi_pdu);
  85
  86 /*
  87  * Hooks to the Operating Systems
  88  */
  89 static int qla_pci_probe (device_t);
  90 static int qla_pci_attach (device_t);
  91 static int qla_pci_detach (device_t);
  92
  93 static void qla_init(void *arg);
  94 static int qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
  95 static int qla_media_change(struct ifnet *ifp);
  96 static void qla_media_status(struct ifnet *ifp, struct ifmediareq *ifmr);
  97
  98 static int qla_transmit(struct ifnet *ifp, struct mbuf  *mp);
  99 static void qla_qflush(struct ifnet *ifp);
 100 static int qla_alloc_tx_br(qla_host_t *ha, qla_tx_fp_t *tx_fp);
 101 static void qla_free_tx_br(qla_host_t *ha, qla_tx_fp_t *tx_fp);
 102 static int qla_create_fp_taskqueues(qla_host_t *ha);
 103 static void qla_destroy_fp_taskqueues(qla_host_t *ha);
 104 static void qla_drain_fp_taskqueues(qla_host_t *ha);
 105
 106 static device_method_t qla_pci_methods[] = {
 107         /* Device interface */
 108         DEVMETHOD(device_probe, qla_pci_probe),
 109         DEVMETHOD(device_attach, qla_pci_attach),
 110         DEVMETHOD(device_detach, qla_pci_detach),
 111         { 0, 0 }
 112 };
 113
 114 static driver_t qla_pci_driver = {
 115         "ql", qla_pci_methods, sizeof (qla_host_t),
 116 };
 117
 118 static devclass_t qla83xx_devclass;
 119
 120 DRIVER_MODULE(qla83xx, pci, qla_pci_driver, qla83xx_devclass, 0, 0);
 121
 122 MODULE_DEPEND(qla83xx, pci, 1, 1, 1);
 123 MODULE_DEPEND(qla83xx, ether, 1, 1, 1);
 124
 125 MALLOC_DEFINE(M_QLA83XXBUF, "qla83xxbuf", "Buffers for qla83xx driver");
 126
 127 #define QL_STD_REPLENISH_THRES          0
 128 #define QL_JUMBO_REPLENISH_THRES        32
 129
 130 static char dev_str[64];
 131 static char ver_str[64];
 132
 133 /*
 134  * Name:        qla_pci_probe
 135  * Function:    Validate the PCI device to be a QLA80XX device
 136  */
 137 static int
 138 qla_pci_probe(device_t dev)
 139 {
 140         switch ((pci_get_device(dev) << 16) | (pci_get_vendor(dev))) {
 141         case PCI_QLOGIC_ISP8030:
 142                 snprintf(dev_str, sizeof(dev_str), "%s v%d.%d.%d",
 143                         "Qlogic ISP 83xx PCI CNA Adapter-Ethernet Function",
 144                         QLA_VERSION_MAJOR, QLA_VERSION_MINOR,
 145                         QLA_VERSION_BUILD);
 146                 snprintf(ver_str, sizeof(ver_str), "v%d.%d.%d",
 147                         QLA_VERSION_MAJOR, QLA_VERSION_MINOR,
 148                         QLA_VERSION_BUILD);
 149                 device_set_desc(dev, dev_str);
 150                 break;
 151         default:
 152                 return (ENXIO);
 153         }
 154
 155         if (bootverbose)
 156                 printf("%s: %s\n ", __func__, dev_str);
 157
 158         return (BUS_PROBE_DEFAULT);
 159 }
 160
 161 static void
 162 qla_add_sysctls(qla_host_t *ha)
 163 {
 164         device_t dev = ha->pci_dev;
 165
 166         SYSCTL_ADD_STRING(device_get_sysctl_ctx(dev),
 167                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 168                 OID_AUTO, "version", CTLFLAG_RD,
 169                 ver_str, 0, "Driver Version");
 170
 171         SYSCTL_ADD_STRING(device_get_sysctl_ctx(dev),
 172                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 173                 OID_AUTO, "fw_version", CTLFLAG_RD,
 174                 ha->fw_ver_str, 0, "firmware version");
 175
 176         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 177             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
 178             "link_status", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 179             (void *)ha, 0, qla_sysctl_get_link_status, "I", "Link Status");
 180
 181         ha->dbg_level = 0;
 182         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
 183                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 184                 OID_AUTO, "debug", CTLFLAG_RW,
 185                 &ha->dbg_level, ha->dbg_level, "Debug Level");
 186
 187         ha->enable_minidump = 1;
 188         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
 189                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 190                 OID_AUTO, "enable_minidump", CTLFLAG_RW,
 191                 &ha->enable_minidump, ha->enable_minidump,
 192                 "Minidump retrival prior to error recovery "
 193                 "is enabled only when this is set");
 194
 195         ha->enable_driverstate_dump = 1;
 196         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
 197                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 198                 OID_AUTO, "enable_driverstate_dump", CTLFLAG_RW,
 199                 &ha->enable_driverstate_dump, ha->enable_driverstate_dump,
 200                 "Driver State retrival prior to error recovery "
 201                 "is enabled only when this is set");
 202
 203         ha->enable_error_recovery = 1;
 204         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
 205                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 206                 OID_AUTO, "enable_error_recovery", CTLFLAG_RW,
 207                 &ha->enable_error_recovery, ha->enable_error_recovery,
 208                 "when set error recovery is enabled on fatal errors "
 209                 "otherwise the port is turned offline");
 210
 211         ha->ms_delay_after_init = 1000;
 212         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
 213                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 214                 OID_AUTO, "ms_delay_after_init", CTLFLAG_RW,
 215                 &ha->ms_delay_after_init, ha->ms_delay_after_init,
 216                 "millisecond delay after hw_init");
 217
 218         ha->std_replenish = QL_STD_REPLENISH_THRES;
 219         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
 220                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 221                 OID_AUTO, "std_replenish", CTLFLAG_RW,
 222                 &ha->std_replenish, ha->std_replenish,
 223                 "Threshold for Replenishing Standard Frames");
 224
 225         SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
 226                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 227                 OID_AUTO, "ipv4_lro",
 228                 CTLFLAG_RD, &ha->ipv4_lro,
 229                 "number of ipv4 lro completions");
 230
 231         SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
 232                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 233                 OID_AUTO, "ipv6_lro",
 234                 CTLFLAG_RD, &ha->ipv6_lro,
 235                 "number of ipv6 lro completions");
 236
 237         SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
 238                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 239                 OID_AUTO, "tx_tso_frames",
 240                 CTLFLAG_RD, &ha->tx_tso_frames,
 241                 "number of Tx TSO Frames");
 242
 243         SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
 244                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 245                 OID_AUTO, "hw_vlan_tx_frames",
 246                 CTLFLAG_RD, &ha->hw_vlan_tx_frames,
 247                 "number of Tx VLAN Frames");
 248
 249         SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
 250                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 251                 OID_AUTO, "hw_lock_failed",
 252                 CTLFLAG_RD, &ha->hw_lock_failed,
 253                 "number of hw_lock failures");
 254
 255         return;
 256 }
 257
 258 static void
 259 qla_watchdog(void *arg)
 260 {
 261         qla_host_t *ha = arg;
 262         qla_hw_t *hw;
 263         struct ifnet *ifp;
 264
 265         hw = &ha->hw;
 266         ifp = ha->ifp;
 267
 268         if (ha->qla_watchdog_exit) {
 269                 ha->qla_watchdog_exited = 1;
 270                 return;
 271         }
 272         ha->qla_watchdog_exited = 0;
 273
 274         if (!ha->qla_watchdog_pause) {
 275                 if (!ha->offline &&
 276                         (ql_hw_check_health(ha) || ha->qla_initiate_recovery ||
 277                         (ha->msg_from_peer == QL_PEER_MSG_RESET))) {
 278                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 279                         ql_update_link_state(ha);
 280
 281                         if (ha->enable_error_recovery) {
 282                                 ha->qla_watchdog_paused = 1;
 283                                 ha->qla_watchdog_pause = 1;
 284                                 ha->err_inject = 0;
 285                                 device_printf(ha->pci_dev,
 286                                         "%s: taskqueue_enqueue(err_task) \n",
 287                                         __func__);
 288                                 taskqueue_enqueue(ha->err_tq, &ha->err_task);
 289                         } else {
 290                                 if (ifp != NULL)
 291                                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 292                                 ha->offline = 1;
 293                         }
 294                         return;
 295
 296                 } else {
 297                         if (ha->qla_interface_up) {
 298                                 ha->watchdog_ticks++;
 299
 300                                 if (ha->watchdog_ticks > 1000)
 301                                         ha->watchdog_ticks = 0;
 302
 303                                 if (!ha->watchdog_ticks && QL_RUNNING(ifp)) {
 304                                         taskqueue_enqueue(ha->stats_tq,
 305                                                 &ha->stats_task);
 306                                 }
 307
 308                                 if (ha->async_event) {
 309                                         taskqueue_enqueue(ha->async_event_tq,
 310                                                 &ha->async_event_task);
 311                                 }
 312                         }
 313                         ha->qla_watchdog_paused = 0;
 314                 }
 315         } else {
 316                 ha->qla_watchdog_paused = 1;
 317         }
 318
 319         callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
 320                 qla_watchdog, ha);
 321 }
 322
 323 /*
 324  * Name:        qla_pci_attach
 325  * Function:    attaches the device to the operating system
 326  */
 327 static int
 328 qla_pci_attach(device_t dev)
 329 {
 330         qla_host_t *ha = NULL;
 331         uint32_t rsrc_len;
 332         int i;
 333         uint32_t num_rcvq = 0;
 334
 335         if ((ha = device_get_softc(dev)) == NULL) {
 336                 device_printf(dev, "cannot get softc\n");
 337                 return (ENOMEM);
 338         }
 339
 340         memset(ha, 0, sizeof (qla_host_t));
 341
 342         if (pci_get_device(dev) != PCI_PRODUCT_QLOGIC_ISP8030) {
 343                 device_printf(dev, "device is not ISP8030\n");
 344                 return (ENXIO);
 345         }
 346
 347         ha->pci_func = pci_get_function(dev) & 0x1;
 348
 349         ha->pci_dev = dev;
 350
 351         pci_enable_busmaster(dev);
 352
 353         ha->reg_rid = PCIR_BAR(0);
 354         ha->pci_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->reg_rid,
 355                                 RF_ACTIVE);
 356
 357         if (ha->pci_reg == NULL) {
 358                 device_printf(dev, "unable to map any ports\n");
 359                 goto qla_pci_attach_err;
 360         }
 361
 362         rsrc_len = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY,
 363                                         ha->reg_rid);
 364
 365         mtx_init(&ha->hw_lock, "qla83xx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF);
 366         mtx_init(&ha->sp_log_lock, "qla83xx_sp_log_lock", MTX_NETWORK_LOCK, MTX_DEF);
 367         ha->flags.lock_init = 1;
 368
 369         qla_add_sysctls(ha);
 370
 371         ha->hw.num_sds_rings = MAX_SDS_RINGS;
 372         ha->hw.num_rds_rings = MAX_RDS_RINGS;
 373         ha->hw.num_tx_rings = NUM_TX_RINGS;
 374
 375         ha->reg_rid1 = PCIR_BAR(2);
 376         ha->pci_reg1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 377                         &ha->reg_rid1, RF_ACTIVE);
 378
 379         ha->msix_count = pci_msix_count(dev);
 380
 381         if (ha->msix_count < 1 ) {
 382                 device_printf(dev, "%s: msix_count[%d] not enough\n", __func__,
 383                         ha->msix_count);
 384                 goto qla_pci_attach_err;
 385         }
 386
 387         if (ha->msix_count < (ha->hw.num_sds_rings + 1)) {
 388                 ha->hw.num_sds_rings = ha->msix_count - 1;
 389         }
 390
 391         QL_DPRINT2(ha, (dev, "%s: ha %p pci_func 0x%x rsrc_count 0x%08x"
 392                 " msix_count 0x%x pci_reg %p pci_reg1 %p\n", __func__, ha,
 393                 ha->pci_func, rsrc_len, ha->msix_count, ha->pci_reg,
 394                 ha->pci_reg1));
 395
 396         /* initialize hardware */
 397         if (ql_init_hw(ha)) {
 398                 device_printf(dev, "%s: ql_init_hw failed\n", __func__);
 399                 goto qla_pci_attach_err;
 400         }
 401
 402         device_printf(dev, "%s: firmware[%d.%d.%d.%d]\n", __func__,
 403                 ha->fw_ver_major, ha->fw_ver_minor, ha->fw_ver_sub,
 404                 ha->fw_ver_build);
 405         snprintf(ha->fw_ver_str, sizeof(ha->fw_ver_str), "%d.%d.%d.%d",
 406                         ha->fw_ver_major, ha->fw_ver_minor, ha->fw_ver_sub,
 407                         ha->fw_ver_build);
 408
 409         if (qla_get_nic_partition(ha, NULL, &num_rcvq)) {
 410                 device_printf(dev, "%s: qla_get_nic_partition failed\n",
 411                         __func__);
 412                 goto qla_pci_attach_err;
 413         }
 414         device_printf(dev, "%s: ha %p pci_func 0x%x rsrc_count 0x%08x"
 415                 " msix_count 0x%x pci_reg %p pci_reg1 %p num_rcvq = %d\n",
 416                 __func__, ha, ha->pci_func, rsrc_len, ha->msix_count,
 417                 ha->pci_reg, ha->pci_reg1, num_rcvq);
 418
 419         if ((ha->msix_count  < 64) || (num_rcvq != 32)) {
 420                 if (ha->hw.num_sds_rings > 15) {
 421                         ha->hw.num_sds_rings = 15;
 422                 }
 423         }
 424
 425         ha->hw.num_rds_rings = ha->hw.num_sds_rings;
 426         ha->hw.num_tx_rings = ha->hw.num_sds_rings;
 427
 428 #ifdef QL_ENABLE_ISCSI_TLV
 429         ha->hw.num_tx_rings = ha->hw.num_sds_rings * 2;
 430 #endif /* #ifdef QL_ENABLE_ISCSI_TLV */
 431
 432         ql_hw_add_sysctls(ha);
 433
 434         ha->msix_count = ha->hw.num_sds_rings + 1;
 435
 436         if (pci_alloc_msix(dev, &ha->msix_count)) {
 437                 device_printf(dev, "%s: pci_alloc_msi[%d] failed\n", __func__,
 438                         ha->msix_count);
 439                 ha->msix_count = 0;
 440                 goto qla_pci_attach_err;
 441         }
 442
 443         ha->mbx_irq_rid = 1;
 444         ha->mbx_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 445                                 &ha->mbx_irq_rid,
 446                                 (RF_ACTIVE | RF_SHAREABLE));
 447         if (ha->mbx_irq == NULL) {
 448                 device_printf(dev, "could not allocate mbx interrupt\n");
 449                 goto qla_pci_attach_err;
 450         }
 451         if (bus_setup_intr(dev, ha->mbx_irq, (INTR_TYPE_NET | INTR_MPSAFE),
 452                 NULL, ql_mbx_isr, ha, &ha->mbx_handle)) {
 453                 device_printf(dev, "could not setup mbx interrupt\n");
 454                 goto qla_pci_attach_err;
 455         }
 456
 457         for (i = 0; i < ha->hw.num_sds_rings; i++) {
 458                 ha->irq_vec[i].sds_idx = i;
 459                 ha->irq_vec[i].ha = ha;
 460                 ha->irq_vec[i].irq_rid = 2 + i;
 461
 462                 ha->irq_vec[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 463                                 &ha->irq_vec[i].irq_rid,
 464                                 (RF_ACTIVE | RF_SHAREABLE));
 465
 466                 if (ha->irq_vec[i].irq == NULL) {
 467                         device_printf(dev, "could not allocate interrupt\n");
 468                         goto qla_pci_attach_err;
 469                 }
 470                 if (bus_setup_intr(dev, ha->irq_vec[i].irq,
 471                         (INTR_TYPE_NET | INTR_MPSAFE),
 472                         NULL, ql_isr, &ha->irq_vec[i],
 473                         &ha->irq_vec[i].handle)) {
 474                         device_printf(dev, "could not setup interrupt\n");
 475                         goto qla_pci_attach_err;
 476                 }
 477
 478                 ha->tx_fp[i].ha = ha;
 479                 ha->tx_fp[i].txr_idx = i;
 480
 481                 if (qla_alloc_tx_br(ha, &ha->tx_fp[i])) {
 482                         device_printf(dev, "%s: could not allocate tx_br[%d]\n",
 483                                 __func__, i);
 484                         goto qla_pci_attach_err;
 485                 }
 486         }
 487
 488         if (qla_create_fp_taskqueues(ha) != 0)
 489                 goto qla_pci_attach_err;
 490
 491         printf("%s: mp__ncpus %d sds %d rds %d msi-x %d\n", __func__, mp_ncpus,
 492                 ha->hw.num_sds_rings, ha->hw.num_rds_rings, ha->msix_count);
 493
 494         ql_read_mac_addr(ha);
 495
 496         /* allocate parent dma tag */
 497         if (qla_alloc_parent_dma_tag(ha)) {
 498                 device_printf(dev, "%s: qla_alloc_parent_dma_tag failed\n",
 499                         __func__);
 500                 goto qla_pci_attach_err;
 501         }
 502
 503         /* alloc all dma buffers */
 504         if (ql_alloc_dma(ha)) {
 505                 device_printf(dev, "%s: ql_alloc_dma failed\n", __func__);
 506                 goto qla_pci_attach_err;
 507         }
 508         qla_get_peer(ha);
 509
 510         if (ql_minidump_init(ha) != 0) {
 511                 device_printf(dev, "%s: ql_minidump_init failed\n", __func__);
 512                 goto qla_pci_attach_err;
 513         }
 514         ql_alloc_drvr_state_buffer(ha);
 515         ql_alloc_sp_log_buffer(ha);
 516         /* create the o.s ethernet interface */
 517         qla_init_ifnet(dev, ha);
 518
 519         ha->flags.qla_watchdog_active = 1;
 520         ha->qla_watchdog_pause = 0;
 521
 522         callout_init(&ha->tx_callout, TRUE);
 523         ha->flags.qla_callout_init = 1;
 524
 525         /* create ioctl device interface */
 526         if (ql_make_cdev(ha)) {
 527                 device_printf(dev, "%s: ql_make_cdev failed\n", __func__);
 528                 goto qla_pci_attach_err;
 529         }
 530
 531         callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
 532                 qla_watchdog, ha);
 533
 534         TASK_INIT(&ha->err_task, 0, qla_error_recovery, ha);
 535         ha->err_tq = taskqueue_create("qla_errq", M_NOWAIT,
 536                         taskqueue_thread_enqueue, &ha->err_tq);
 537         taskqueue_start_threads(&ha->err_tq, 1, PI_NET, "%s errq",
 538                 device_get_nameunit(ha->pci_dev));
 539
 540         TASK_INIT(&ha->async_event_task, 0, qla_async_event, ha);
 541         ha->async_event_tq = taskqueue_create("qla_asyncq", M_NOWAIT,
 542                         taskqueue_thread_enqueue, &ha->async_event_tq);
 543         taskqueue_start_threads(&ha->async_event_tq, 1, PI_NET, "%s asyncq",
 544                 device_get_nameunit(ha->pci_dev));
 545
 546         TASK_INIT(&ha->stats_task, 0, qla_stats, ha);
 547         ha->stats_tq = taskqueue_create("qla_statsq", M_NOWAIT,
 548                         taskqueue_thread_enqueue, &ha->stats_tq);
 549         taskqueue_start_threads(&ha->stats_tq, 1, PI_NET, "%s taskq",
 550                 device_get_nameunit(ha->pci_dev));
 551
 552         QL_DPRINT2(ha, (dev, "%s: exit 0\n", __func__));
 553         return (0);
 554
 555 qla_pci_attach_err:
 556
 557         qla_release(ha);
 558
 559         if (ha->flags.lock_init) {
 560                 mtx_destroy(&ha->hw_lock);
 561                 mtx_destroy(&ha->sp_log_lock);
 562         }
 563
 564         QL_DPRINT2(ha, (dev, "%s: exit ENXIO\n", __func__));
 565         return (ENXIO);
 566 }
 567
 568 /*
 569  * Name:        qla_pci_detach
 570  * Function:    Unhooks the device from the operating system
 571  */
 572 static int
 573 qla_pci_detach(device_t dev)
 574 {
 575         qla_host_t *ha = NULL;
 576         struct ifnet *ifp;
 577
 578         if ((ha = device_get_softc(dev)) == NULL) {
 579                 device_printf(dev, "cannot get softc\n");
 580                 return (ENOMEM);
 581         }
 582
 583         QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
 584
 585         ifp = ha->ifp;
 586
 587         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 588         QLA_LOCK(ha, __func__, -1, 0);
 589
 590         ha->qla_detach_active = 1;
 591         qla_stop(ha);
 592
 593         qla_release(ha);
 594
 595         QLA_UNLOCK(ha, __func__);
 596
 597         if (ha->flags.lock_init) {
 598                 mtx_destroy(&ha->hw_lock);
 599                 mtx_destroy(&ha->sp_log_lock);
 600         }
 601
 602         QL_DPRINT2(ha, (dev, "%s: exit\n", __func__));
 603
 604         return (0);
 605 }
 606
 607 /*
 608  * SYSCTL Related Callbacks
 609  */
 610 static int
 611 qla_sysctl_get_link_status(SYSCTL_HANDLER_ARGS)
 612 {
 613         int err, ret = 0;
 614         qla_host_t *ha;
 615
 616         err = sysctl_handle_int(oidp, &ret, 0, req);
 617
 618         if (err || !req->newptr)
 619                 return (err);
 620
 621         if (ret == 1) {
 622                 ha = (qla_host_t *)arg1;
 623                 ql_hw_link_status(ha);
 624         }
 625         return (err);
 626 }
 627
 628 /*
 629  * Name:        qla_release
 630  * Function:    Releases the resources allocated for the device
 631  */
 632 static void
 633 qla_release(qla_host_t *ha)
 634 {
 635         device_t dev;
 636         int i;
 637
 638         dev = ha->pci_dev;
 639
 640         if (ha->async_event_tq) {
 641                 taskqueue_drain_all(ha->async_event_tq);
 642                 taskqueue_free(ha->async_event_tq);
 643         }
 644
 645         if (ha->err_tq) {
 646                 taskqueue_drain_all(ha->err_tq);
 647                 taskqueue_free(ha->err_tq);
 648         }
 649
 650         if (ha->stats_tq) {
 651                 taskqueue_drain_all(ha->stats_tq);
 652                 taskqueue_free(ha->stats_tq);
 653         }
 654
 655         ql_del_cdev(ha);
 656
 657         if (ha->flags.qla_watchdog_active) {
 658                 ha->qla_watchdog_exit = 1;
 659
 660                 while (ha->qla_watchdog_exited == 0)
 661                         qla_mdelay(__func__, 1);
 662         }
 663
 664         if (ha->flags.qla_callout_init)
 665                 callout_stop(&ha->tx_callout);
 666
 667         if (ha->ifp != NULL)
 668                 ether_ifdetach(ha->ifp);
 669
 670         ql_free_drvr_state_buffer(ha);
 671         ql_free_sp_log_buffer(ha);
 672         ql_free_dma(ha);
 673         qla_free_parent_dma_tag(ha);
 674
 675         if (ha->mbx_handle)
 676                 (void)bus_teardown_intr(dev, ha->mbx_irq, ha->mbx_handle);
 677
 678         if (ha->mbx_irq)
 679                 (void) bus_release_resource(dev, SYS_RES_IRQ, ha->mbx_irq_rid,
 680                                 ha->mbx_irq);
 681
 682         for (i = 0; i < ha->hw.num_sds_rings; i++) {
 683                 if (ha->irq_vec[i].handle) {
 684                         (void)bus_teardown_intr(dev, ha->irq_vec[i].irq,
 685                                         ha->irq_vec[i].handle);
 686                 }
 687
 688                 if (ha->irq_vec[i].irq) {
 689                         (void)bus_release_resource(dev, SYS_RES_IRQ,
 690                                 ha->irq_vec[i].irq_rid,
 691                                 ha->irq_vec[i].irq);
 692                 }
 693
 694                 qla_free_tx_br(ha, &ha->tx_fp[i]);
 695         }
 696         qla_destroy_fp_taskqueues(ha);
 697
 698         if (ha->msix_count)
 699                 pci_release_msi(dev);
 700
 701         if (ha->pci_reg)
 702                 (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid,
 703                                 ha->pci_reg);
 704
 705         if (ha->pci_reg1)
 706                 (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid1,
 707                                 ha->pci_reg1);
 708
 709         return;
 710 }
 711
 712 /*
 713  * DMA Related Functions
 714  */
 715
 716 static void
 717 qla_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 718 {
 719         *((bus_addr_t *)arg) = 0;
 720
 721         if (error) {
 722                 printf("%s: bus_dmamap_load failed (%d)\n", __func__, error);
 723                 return;
 724         }
 725
 726         *((bus_addr_t *)arg) = segs[0].ds_addr;
 727
 728         return;
 729 }
 730
 731 int
 732 ql_alloc_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
 733 {
 734         int             ret = 0;
 735         device_t        dev;
 736         bus_addr_t      b_addr;
 737
 738         dev = ha->pci_dev;
 739
 740         QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
 741
 742         ret = bus_dma_tag_create(
 743                         ha->parent_tag,/* parent */
 744                         dma_buf->alignment,
 745                         ((bus_size_t)(1ULL << 32)),/* boundary */
 746                         BUS_SPACE_MAXADDR,      /* lowaddr */
 747                         BUS_SPACE_MAXADDR,      /* highaddr */
 748                         NULL, NULL,             /* filter, filterarg */
 749                         dma_buf->size,          /* maxsize */
 750                         1,                      /* nsegments */
 751                         dma_buf->size,          /* maxsegsize */
 752                         0,                      /* flags */
 753                         NULL, NULL,             /* lockfunc, lockarg */
 754                         &dma_buf->dma_tag);
 755
 756         if (ret) {
 757                 device_printf(dev, "%s: could not create dma tag\n", __func__);
 758                 goto ql_alloc_dmabuf_exit;
 759         }
 760         ret = bus_dmamem_alloc(dma_buf->dma_tag,
 761                         (void **)&dma_buf->dma_b,
 762                         (BUS_DMA_ZERO | BUS_DMA_COHERENT | BUS_DMA_NOWAIT),
 763                         &dma_buf->dma_map);
 764         if (ret) {
 765                 bus_dma_tag_destroy(dma_buf->dma_tag);
 766                 device_printf(dev, "%s: bus_dmamem_alloc failed\n", __func__);
 767                 goto ql_alloc_dmabuf_exit;
 768         }
 769
 770         ret = bus_dmamap_load(dma_buf->dma_tag,
 771                         dma_buf->dma_map,
 772                         dma_buf->dma_b,
 773                         dma_buf->size,
 774                         qla_dmamap_callback,
 775                         &b_addr, BUS_DMA_NOWAIT);
 776
 777         if (ret || !b_addr) {
 778                 bus_dma_tag_destroy(dma_buf->dma_tag);
 779                 bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b,
 780                         dma_buf->dma_map);
 781                 ret = -1;
 782                 goto ql_alloc_dmabuf_exit;
 783         }
 784
 785         dma_buf->dma_addr = b_addr;
 786
 787 ql_alloc_dmabuf_exit:
 788         QL_DPRINT2(ha, (dev, "%s: exit ret 0x%08x tag %p map %p b %p sz 0x%x\n",
 789                 __func__, ret, (void *)dma_buf->dma_tag,
 790                 (void *)dma_buf->dma_map, (void *)dma_buf->dma_b,
 791                 dma_buf->size));
 792
 793         return ret;
 794 }
 795
 796 void
 797 ql_free_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
 798 {
 799         bus_dmamap_unload(dma_buf->dma_tag, dma_buf->dma_map);
 800         bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map);
 801         bus_dma_tag_destroy(dma_buf->dma_tag);
 802 }
 803
 804 static int
 805 qla_alloc_parent_dma_tag(qla_host_t *ha)
 806 {
 807         int             ret;
 808         device_t        dev;
 809
 810         dev = ha->pci_dev;
 811
 812         /*
 813          * Allocate parent DMA Tag
 814          */
 815         ret = bus_dma_tag_create(
 816                         bus_get_dma_tag(dev),   /* parent */
 817                         1,((bus_size_t)(1ULL << 32)),/* alignment, boundary */
 818                         BUS_SPACE_MAXADDR,      /* lowaddr */
 819                         BUS_SPACE_MAXADDR,      /* highaddr */
 820                         NULL, NULL,             /* filter, filterarg */
 821                         BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
 822                         0,                      /* nsegments */
 823                         BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
 824                         0,                      /* flags */
 825                         NULL, NULL,             /* lockfunc, lockarg */
 826                         &ha->parent_tag);
 827
 828         if (ret) {
 829                 device_printf(dev, "%s: could not create parent dma tag\n",
 830                         __func__);
 831                 return (-1);
 832         }
 833
 834         ha->flags.parent_tag = 1;
 835
 836         return (0);
 837 }
 838
 839 static void
 840 qla_free_parent_dma_tag(qla_host_t *ha)
 841 {
 842         if (ha->flags.parent_tag) {
 843                 bus_dma_tag_destroy(ha->parent_tag);
 844                 ha->flags.parent_tag = 0;
 845         }
 846 }
 847
 848 /*
 849  * Name: qla_init_ifnet
 850  * Function: Creates the Network Device Interface and Registers it with the O.S
 851  */
 852
 853 static void
 854 qla_init_ifnet(device_t dev, qla_host_t *ha)
 855 {
 856         struct ifnet *ifp;
 857
 858         QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
 859
 860         ifp = ha->ifp = if_alloc(IFT_ETHER);
 861
 862         if (ifp == NULL)
 863                 panic("%s: cannot if_alloc()\n", device_get_nameunit(dev));
 864
 865         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 866
 867         ifp->if_baudrate = IF_Gbps(10);
 868         ifp->if_capabilities = IFCAP_LINKSTATE;
 869         ifp->if_mtu = ETHERMTU;
 870
 871         ifp->if_init = qla_init;
 872         ifp->if_softc = ha;
 873         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 874         ifp->if_ioctl = qla_ioctl;
 875
 876         ifp->if_transmit = qla_transmit;
 877         ifp->if_qflush = qla_qflush;
 878
 879         IFQ_SET_MAXLEN(&ifp->if_snd, qla_get_ifq_snd_maxlen(ha));
 880         ifp->if_snd.ifq_drv_maxlen = qla_get_ifq_snd_maxlen(ha);
 881         IFQ_SET_READY(&ifp->if_snd);
 882
 883         ha->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 884
 885         ether_ifattach(ifp, qla_get_mac_addr(ha));
 886
 887         ifp->if_capabilities |= IFCAP_HWCSUM |
 888                                 IFCAP_TSO4 |
 889                                 IFCAP_TSO6 |
 890                                 IFCAP_JUMBO_MTU |
 891                                 IFCAP_VLAN_HWTAGGING |
 892                                 IFCAP_VLAN_MTU |
 893                                 IFCAP_VLAN_HWTSO |
 894                                 IFCAP_LRO;
 895
 896         ifp->if_capenable = ifp->if_capabilities;
 897
 898         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
 899
 900         ifmedia_init(&ha->media, IFM_IMASK, qla_media_change, qla_media_status);
 901
 902         ifmedia_add(&ha->media, (IFM_ETHER | qla_get_optics(ha) | IFM_FDX), 0,
 903                 NULL);
 904         ifmedia_add(&ha->media, (IFM_ETHER | IFM_AUTO), 0, NULL);
 905
 906         ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO));
 907
 908         QL_DPRINT2(ha, (dev, "%s: exit\n", __func__));
 909
 910         return;
 911 }
 912
 913 static void
 914 qla_init_locked(qla_host_t *ha)
 915 {
 916         struct ifnet *ifp = ha->ifp;
 917
 918         ql_sp_log(ha, 14, 0, 0, 0, 0, 0, 0);
 919
 920         qla_stop(ha);
 921
 922         if (qla_alloc_xmt_bufs(ha) != 0)
 923                 return;
 924
 925         qla_confirm_9kb_enable(ha);
 926
 927         if (qla_alloc_rcv_bufs(ha) != 0)
 928                 return;
 929
 930         bcopy(IF_LLADDR(ha->ifp), ha->hw.mac_addr, ETHER_ADDR_LEN);
 931
 932         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO;
 933         ifp->if_hwassist |= CSUM_TCP_IPV6 | CSUM_UDP_IPV6;
 934
 935         ha->stop_rcv = 0;
 936         if (ql_init_hw_if(ha) == 0) {
 937                 ifp = ha->ifp;
 938                 ifp->if_drv_flags |= IFF_DRV_RUNNING;
 939                 ha->hw_vlan_tx_frames = 0;
 940                 ha->tx_tso_frames = 0;
 941                 ha->qla_interface_up = 1;
 942                 ql_update_link_state(ha);
 943         } else {
 944                 if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_IF_START_FAILURE)
 945                         ha->hw.sp_log_stop = -1;
 946         }
 947
 948         ha->qla_watchdog_pause = 0;
 949
 950         return;
 951 }
 952
 953 static void
 954 qla_init(void *arg)
 955 {
 956         qla_host_t *ha;
 957
 958         ha = (qla_host_t *)arg;
 959
 960         QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
 961
 962         if (QLA_LOCK(ha, __func__, -1, 0) != 0)
 963                 return;
 964
 965         qla_init_locked(ha);
 966
 967         QLA_UNLOCK(ha, __func__);
 968
 969         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
 970 }
 971
 972 static u_int
 973 qla_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int mcnt)
 974 {
 975         uint8_t *mta = arg;
 976
 977         if (mcnt == Q8_MAX_NUM_MULTICAST_ADDRS)
 978                 return (0);
 979
 980         bcopy(LLADDR(sdl), &mta[mcnt * Q8_MAC_ADDR_LEN], Q8_MAC_ADDR_LEN);
 981
 982         return (1);
 983 }
 984
 985 static int
 986 qla_set_multi(qla_host_t *ha, uint32_t add_multi)
 987 {
 988         uint8_t mta[Q8_MAX_NUM_MULTICAST_ADDRS * Q8_MAC_ADDR_LEN];
 989         int mcnt = 0;
 990         struct ifnet *ifp = ha->ifp;
 991         int ret = 0;
 992
 993         mcnt = if_foreach_llmaddr(ifp, qla_copy_maddr, mta);
 994
 995         if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT,
 996                 QLA_LOCK_NO_SLEEP) != 0)
 997                 return (-1);
 998
 999         ql_sp_log(ha, 12, 4, ifp->if_drv_flags,
1000                 (ifp->if_drv_flags & IFF_DRV_RUNNING),
1001                 add_multi, (uint32_t)mcnt, 0);
1002
1003         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1004                 if (!add_multi) {
1005                         ret = qla_hw_del_all_mcast(ha);
1006
1007                         if (ret)
1008                                 device_printf(ha->pci_dev,
1009                                         "%s: qla_hw_del_all_mcast() failed\n",
1010                                 __func__);
1011                 }
1012
1013                 if (!ret)
1014                         ret = ql_hw_set_multi(ha, mta, mcnt, 1);
1015         }
1016
1017         QLA_UNLOCK(ha, __func__);
1018
1019         return (ret);
1020 }
1021
1022 static int
1023 qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1024 {
1025         int ret = 0;
1026         struct ifreq *ifr = (struct ifreq *)data;
1027         struct ifaddr *ifa = (struct ifaddr *)data;
1028         qla_host_t *ha;
1029
1030         ha = (qla_host_t *)ifp->if_softc;
1031         if (ha->offline || ha->qla_initiate_recovery)
1032                 return (ret);
1033
1034         switch (cmd) {
1035         case SIOCSIFADDR:
1036                 QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFADDR (0x%lx)\n",
1037                         __func__, cmd));
1038
1039                 if (ifa->ifa_addr->sa_family == AF_INET) {
1040                         ret = QLA_LOCK(ha, __func__,
1041                                         QLA_LOCK_DEFAULT_MS_TIMEOUT,
1042                                         QLA_LOCK_NO_SLEEP);
1043                         if (ret)
1044                                 break;
1045
1046                         ifp->if_flags |= IFF_UP;
1047
1048                         ql_sp_log(ha, 8, 3, ifp->if_drv_flags,
1049                                 (ifp->if_drv_flags & IFF_DRV_RUNNING),
1050                                 ntohl(IA_SIN(ifa)->sin_addr.s_addr), 0, 0);
1051
1052                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1053                                 qla_init_locked(ha);
1054                         }
1055
1056                         QLA_UNLOCK(ha, __func__);
1057                         QL_DPRINT4(ha, (ha->pci_dev,
1058                                 "%s: SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n",
1059                                 __func__, cmd,
1060                                 ntohl(IA_SIN(ifa)->sin_addr.s_addr)));
1061
1062                         arp_ifinit(ifp, ifa);
1063                 } else {
1064                         ether_ioctl(ifp, cmd, data);
1065                 }
1066                 break;
1067
1068         case SIOCSIFMTU:
1069                 QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFMTU (0x%lx)\n",
1070                         __func__, cmd));
1071
1072                 if (ifr->ifr_mtu > QLA_MAX_MTU) {
1073                         ret = EINVAL;
1074                 } else {
1075                         ret = QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT,
1076                                         QLA_LOCK_NO_SLEEP);
1077
1078                         if (ret)
1079                                 break;
1080
1081                         ifp->if_mtu = ifr->ifr_mtu;
1082                         ha->max_frame_size =
1083                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1084
1085                         ql_sp_log(ha, 9, 4, ifp->if_drv_flags,
1086                                 (ifp->if_drv_flags & IFF_DRV_RUNNING),
1087                                 ha->max_frame_size, ifp->if_mtu, 0);
1088
1089                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1090                                 qla_init_locked(ha);
1091                         }
1092
1093                         if (ifp->if_mtu > ETHERMTU)
1094                                 ha->std_replenish = QL_JUMBO_REPLENISH_THRES;
1095                         else
1096                                 ha->std_replenish = QL_STD_REPLENISH_THRES;
1097
1098
1099                         QLA_UNLOCK(ha, __func__);
1100                 }
1101
1102                 break;
1103
1104         case SIOCSIFFLAGS:
1105                 QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFFLAGS (0x%lx)\n",
1106                         __func__, cmd));
1107
1108                 ret = QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT,
1109                                 QLA_LOCK_NO_SLEEP);
1110
1111                 if (ret)
1112                         break;
1113
1114                 ql_sp_log(ha, 10, 4, ifp->if_drv_flags,
1115                         (ifp->if_drv_flags & IFF_DRV_RUNNING),
1116                         ha->if_flags, ifp->if_flags, 0);
1117
1118                 if (ifp->if_flags & IFF_UP) {
1119                         ha->max_frame_size = ifp->if_mtu +
1120                                         ETHER_HDR_LEN + ETHER_CRC_LEN;
1121                         qla_init_locked(ha);
1122
1123                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1124                                 if ((ifp->if_flags ^ ha->if_flags) &
1125                                         IFF_PROMISC) {
1126                                         ret = ql_set_promisc(ha);
1127                                 } else if ((ifp->if_flags ^ ha->if_flags) &
1128                                         IFF_ALLMULTI) {
1129                                         ret = ql_set_allmulti(ha);
1130                                 }
1131                         }
1132                 } else {
1133                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1134                                 qla_stop(ha);
1135                         ha->if_flags = ifp->if_flags;
1136                 }
1137
1138                 QLA_UNLOCK(ha, __func__);
1139                 break;
1140
1141         case SIOCADDMULTI:
1142                 QL_DPRINT4(ha, (ha->pci_dev,
1143                         "%s: %s (0x%lx)\n", __func__, "SIOCADDMULTI", cmd));
1144
1145                 if (qla_set_multi(ha, 1))
1146                         ret = EINVAL;
1147                 break;
1148
1149         case SIOCDELMULTI:
1150                 QL_DPRINT4(ha, (ha->pci_dev,
1151                         "%s: %s (0x%lx)\n", __func__, "SIOCDELMULTI", cmd));
1152
1153                 if (qla_set_multi(ha, 0))
1154                         ret = EINVAL;
1155                 break;
1156
1157         case SIOCSIFMEDIA:
1158         case SIOCGIFMEDIA:
1159                 QL_DPRINT4(ha, (ha->pci_dev,
1160                         "%s: SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n",
1161                         __func__, cmd));
1162                 ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd);
1163                 break;
1164
1165         case SIOCSIFCAP:
1166         {
1167                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1168
1169                 QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFCAP (0x%lx)\n",
1170                         __func__, cmd));
1171
1172                 if (mask & IFCAP_HWCSUM)
1173                         ifp->if_capenable ^= IFCAP_HWCSUM;
1174                 if (mask & IFCAP_TSO4)
1175                         ifp->if_capenable ^= IFCAP_TSO4;
1176                 if (mask & IFCAP_TSO6)
1177                         ifp->if_capenable ^= IFCAP_TSO6;
1178                 if (mask & IFCAP_VLAN_HWTAGGING)
1179                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1180                 if (mask & IFCAP_VLAN_HWTSO)
1181                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1182                 if (mask & IFCAP_LRO)
1183                         ifp->if_capenable ^= IFCAP_LRO;
1184
1185                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1186                         ret = QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT,
1187                                 QLA_LOCK_NO_SLEEP);
1188
1189                         if (ret)
1190                                 break;
1191
1192                         ql_sp_log(ha, 11, 4, ifp->if_drv_flags,
1193                                 (ifp->if_drv_flags & IFF_DRV_RUNNING),
1194                                 mask, ifp->if_capenable, 0);
1195
1196                         qla_init_locked(ha);
1197
1198                         QLA_UNLOCK(ha, __func__);
1199                 }
1200                 VLAN_CAPABILITIES(ifp);
1201                 break;
1202         }
1203
1204         default:
1205                 QL_DPRINT4(ha, (ha->pci_dev, "%s: default (0x%lx)\n",
1206                         __func__, cmd));
1207                 ret = ether_ioctl(ifp, cmd, data);
1208                 break;
1209         }
1210
1211         return (ret);
1212 }
1213
1214 static int
1215 qla_media_change(struct ifnet *ifp)
1216 {
1217         qla_host_t *ha;
1218         struct ifmedia *ifm;
1219         int ret = 0;
1220
1221         ha = (qla_host_t *)ifp->if_softc;
1222
1223         QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
1224
1225         ifm = &ha->media;
1226
1227         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1228                 ret = EINVAL;
1229
1230         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
1231
1232         return (ret);
1233 }
1234
1235 static void
1236 qla_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1237 {
1238         qla_host_t *ha;
1239
1240         ha = (qla_host_t *)ifp->if_softc;
1241
1242         QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
1243
1244         ifmr->ifm_status = IFM_AVALID;
1245         ifmr->ifm_active = IFM_ETHER;
1246
1247         ql_update_link_state(ha);
1248         if (ha->hw.link_up) {
1249                 ifmr->ifm_status |= IFM_ACTIVE;
1250                 ifmr->ifm_active |= (IFM_FDX | qla_get_optics(ha));
1251         }
1252
1253         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit (%s)\n", __func__,\
1254                 (ha->hw.link_up ? "link_up" : "link_down")));
1255
1256         return;
1257 }
1258
1259 static int
1260 qla_send(qla_host_t *ha, struct mbuf **m_headp, uint32_t txr_idx,
1261         uint32_t iscsi_pdu)
1262 {
1263         bus_dma_segment_t       segs[QLA_MAX_SEGMENTS];
1264         bus_dmamap_t            map;
1265         int                     nsegs;
1266         int                     ret = -1;
1267         uint32_t                tx_idx;
1268         struct mbuf             *m_head = *m_headp;
1269
1270         QL_DPRINT8(ha, (ha->pci_dev, "%s: enter\n", __func__));
1271
1272         tx_idx = ha->hw.tx_cntxt[txr_idx].txr_next;
1273
1274         if ((NULL != ha->tx_ring[txr_idx].tx_buf[tx_idx].m_head) ||
1275                 (QL_ERR_INJECT(ha, INJCT_TXBUF_MBUF_NON_NULL))){
1276                 QL_ASSERT(ha, 0, ("%s [%d]: txr_idx = %d tx_idx = %d "\
1277                         "mbuf = %p\n", __func__, __LINE__, txr_idx, tx_idx,\
1278                         ha->tx_ring[txr_idx].tx_buf[tx_idx].m_head));
1279
1280                 device_printf(ha->pci_dev, "%s [%d]: txr_idx = %d tx_idx = %d "
1281                         "mbuf = %p\n", __func__, __LINE__, txr_idx, tx_idx,
1282                         ha->tx_ring[txr_idx].tx_buf[tx_idx].m_head);
1283
1284                 if (m_head)
1285                         m_freem(m_head);
1286                 *m_headp = NULL;
1287                 QL_INITIATE_RECOVERY(ha);
1288                 return (ret);
1289         }
1290
1291         map = ha->tx_ring[txr_idx].tx_buf[tx_idx].map;
1292
1293         ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs,
1294                         BUS_DMA_NOWAIT);
1295
1296         if (ret == EFBIG) {
1297                 struct mbuf *m;
1298
1299                 QL_DPRINT8(ha, (ha->pci_dev, "%s: EFBIG [%d]\n", __func__,
1300                         m_head->m_pkthdr.len));
1301
1302                 m = m_defrag(m_head, M_NOWAIT);
1303                 if (m == NULL) {
1304                         ha->err_tx_defrag++;
1305                         m_freem(m_head);
1306                         *m_headp = NULL;
1307                         device_printf(ha->pci_dev,
1308                                 "%s: m_defrag() = NULL [%d]\n",
1309                                 __func__, ret);
1310                         return (ENOBUFS);
1311                 }
1312                 m_head = m;
1313                 *m_headp = m_head;
1314
1315                 if ((ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head,
1316                                         segs, &nsegs, BUS_DMA_NOWAIT))) {
1317                         ha->err_tx_dmamap_load++;
1318
1319                         device_printf(ha->pci_dev,
1320                                 "%s: bus_dmamap_load_mbuf_sg failed0[%d, %d]\n",
1321                                 __func__, ret, m_head->m_pkthdr.len);
1322
1323                         if (ret != ENOMEM) {
1324                                 m_freem(m_head);
1325                                 *m_headp = NULL;
1326                         }
1327                         return (ret);
1328                 }
1329
1330         } else if (ret) {
1331                 ha->err_tx_dmamap_load++;
1332
1333                 device_printf(ha->pci_dev,
1334                         "%s: bus_dmamap_load_mbuf_sg failed1[%d, %d]\n",
1335                         __func__, ret, m_head->m_pkthdr.len);
1336
1337                 if (ret != ENOMEM) {
1338                         m_freem(m_head);
1339                         *m_headp = NULL;
1340                 }
1341                 return (ret);
1342         }
1343
1344         QL_ASSERT(ha, (nsegs != 0), ("qla_send: empty packet"));
1345
1346         bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_PREWRITE);
1347
1348         if (!(ret = ql_hw_send(ha, segs, nsegs, tx_idx, m_head, txr_idx,
1349                                 iscsi_pdu))) {
1350                 ha->tx_ring[txr_idx].count++;
1351                 if (iscsi_pdu)
1352                         ha->tx_ring[txr_idx].iscsi_pkt_count++;
1353                 ha->tx_ring[txr_idx].tx_buf[tx_idx].m_head = m_head;
1354         } else {
1355                 bus_dmamap_unload(ha->tx_tag, map);
1356                 if (ret == EINVAL) {
1357                         if (m_head)
1358                                 m_freem(m_head);
1359                         *m_headp = NULL;
1360                 }
1361         }
1362
1363         QL_DPRINT8(ha, (ha->pci_dev, "%s: exit\n", __func__));
1364         return (ret);
1365 }
1366
1367 static int
1368 qla_alloc_tx_br(qla_host_t *ha, qla_tx_fp_t *fp)
1369 {
1370         snprintf(fp->tx_mtx_name, sizeof(fp->tx_mtx_name),
1371                 "qla%d_fp%d_tx_mq_lock", ha->pci_func, fp->txr_idx);
1372
1373         mtx_init(&fp->tx_mtx, fp->tx_mtx_name, NULL, MTX_DEF);
1374
1375         fp->tx_br = buf_ring_alloc(NUM_TX_DESCRIPTORS, M_DEVBUF,
1376                                    M_NOWAIT, &fp->tx_mtx);
1377         if (fp->tx_br == NULL) {
1378             QL_DPRINT1(ha, (ha->pci_dev, "buf_ring_alloc failed for "
1379                 " fp[%d, %d]\n", ha->pci_func, fp->txr_idx));
1380             return (-ENOMEM);
1381         }
1382         return 0;
1383 }
1384
1385 static void
1386 qla_free_tx_br(qla_host_t *ha, qla_tx_fp_t *fp)
1387 {
1388         struct mbuf *mp;
1389         struct ifnet *ifp = ha->ifp;
1390
1391         if (mtx_initialized(&fp->tx_mtx)) {
1392                 if (fp->tx_br != NULL) {
1393                         mtx_lock(&fp->tx_mtx);
1394
1395                         while ((mp = drbr_dequeue(ifp, fp->tx_br)) != NULL) {
1396                                 m_freem(mp);
1397                         }
1398
1399                         mtx_unlock(&fp->tx_mtx);
1400
1401                         buf_ring_free(fp->tx_br, M_DEVBUF);
1402                         fp->tx_br = NULL;
1403                 }
1404                 mtx_destroy(&fp->tx_mtx);
1405         }
1406         return;
1407 }
1408
1409 static void
1410 qla_fp_taskqueue(void *context, int pending)
1411 {
1412         qla_tx_fp_t *fp;
1413         qla_host_t *ha;
1414         struct ifnet *ifp;
1415         struct mbuf  *mp = NULL;
1416         int ret = 0;
1417         uint32_t txr_idx;
1418         uint32_t iscsi_pdu = 0;
1419         uint32_t rx_pkts_left = -1;
1420
1421         fp = context;
1422
1423         if (fp == NULL)
1424                 return;
1425
1426         ha = (qla_host_t *)fp->ha;
1427
1428         ifp = ha->ifp;
1429
1430         txr_idx = fp->txr_idx;
1431
1432         mtx_lock(&fp->tx_mtx);
1433
1434         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING) || (!ha->hw.link_up)) {
1435                 mtx_unlock(&fp->tx_mtx);
1436                 goto qla_fp_taskqueue_exit;
1437         }
1438
1439         while (rx_pkts_left && !ha->stop_rcv &&
1440                 (ifp->if_drv_flags & IFF_DRV_RUNNING) && ha->hw.link_up) {
1441                 rx_pkts_left = ql_rcv_isr(ha, fp->txr_idx, 64);
1442
1443 #ifdef QL_ENABLE_ISCSI_TLV
1444                 ql_hw_tx_done_locked(ha, fp->txr_idx);
1445                 ql_hw_tx_done_locked(ha, (fp->txr_idx + (ha->hw.num_tx_rings >> 1)));
1446 #else
1447                 ql_hw_tx_done_locked(ha, fp->txr_idx);
1448 #endif /* #ifdef QL_ENABLE_ISCSI_TLV */
1449
1450                 mp = drbr_peek(ifp, fp->tx_br);
1451
1452                 while (mp != NULL) {
1453                         if (M_HASHTYPE_GET(mp) != M_HASHTYPE_NONE) {
1454 #ifdef QL_ENABLE_ISCSI_TLV
1455                                 if (ql_iscsi_pdu(ha, mp) == 0) {
1456                                         txr_idx = txr_idx +
1457                                                 (ha->hw.num_tx_rings >> 1);
1458                                         iscsi_pdu = 1;
1459                                 } else {
1460                                         iscsi_pdu = 0;
1461                                         txr_idx = fp->txr_idx;
1462                                 }
1463 #endif /* #ifdef QL_ENABLE_ISCSI_TLV */
1464                         }
1465
1466                         ret = qla_send(ha, &mp, txr_idx, iscsi_pdu);
1467
1468                         if (ret) {
1469                                 if (mp != NULL)
1470                                         drbr_putback(ifp, fp->tx_br, mp);
1471                                 else {
1472                                         drbr_advance(ifp, fp->tx_br);
1473                                 }
1474
1475                                 mtx_unlock(&fp->tx_mtx);
1476
1477                                 goto qla_fp_taskqueue_exit0;
1478                         } else {
1479                                 drbr_advance(ifp, fp->tx_br);
1480                         }
1481
1482                         /* Send a copy of the frame to the BPF listener */
1483                         ETHER_BPF_MTAP(ifp, mp);
1484
1485                         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
1486                                 (!ha->hw.link_up))
1487                                 break;
1488
1489                         mp = drbr_peek(ifp, fp->tx_br);
1490                 }
1491         }
1492         mtx_unlock(&fp->tx_mtx);
1493
1494         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1495                 goto qla_fp_taskqueue_exit;
1496
1497 qla_fp_taskqueue_exit0:
1498
1499         if (rx_pkts_left || ((mp != NULL) && ret)) {
1500                 taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task);
1501         } else {
1502                 if (!ha->stop_rcv) {
1503                         QL_ENABLE_INTERRUPTS(ha, fp->txr_idx);
1504                 }
1505         }
1506
1507 qla_fp_taskqueue_exit:
1508
1509         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit ret = %d\n", __func__, ret));
1510         return;
1511 }
1512
1513 static int
1514 qla_create_fp_taskqueues(qla_host_t *ha)
1515 {
1516         int     i;
1517         uint8_t tq_name[32];
1518
1519         for (i = 0; i < ha->hw.num_sds_rings; i++) {
1520                 qla_tx_fp_t *fp = &ha->tx_fp[i];
1521
1522                 bzero(tq_name, sizeof (tq_name));
1523                 snprintf(tq_name, sizeof (tq_name), "ql_fp_tq_%d", i);
1524
1525                 NET_TASK_INIT(&fp->fp_task, 0, qla_fp_taskqueue, fp);
1526
1527                 fp->fp_taskqueue = taskqueue_create_fast(tq_name, M_NOWAIT,
1528                                         taskqueue_thread_enqueue,
1529                                         &fp->fp_taskqueue);
1530
1531                 if (fp->fp_taskqueue == NULL)
1532                         return (-1);
1533
1534                 taskqueue_start_threads(&fp->fp_taskqueue, 1, PI_NET, "%s",
1535                         tq_name);
1536
1537                 QL_DPRINT1(ha, (ha->pci_dev, "%s: %p\n", __func__,
1538                         fp->fp_taskqueue));
1539         }
1540
1541         return (0);
1542 }
1543
1544 static void
1545 qla_destroy_fp_taskqueues(qla_host_t *ha)
1546 {
1547         int     i;
1548
1549         for (i = 0; i < ha->hw.num_sds_rings; i++) {
1550                 qla_tx_fp_t *fp = &ha->tx_fp[i];
1551
1552                 if (fp->fp_taskqueue != NULL) {
1553                         taskqueue_drain_all(fp->fp_taskqueue);
1554                         taskqueue_free(fp->fp_taskqueue);
1555                         fp->fp_taskqueue = NULL;
1556                 }
1557         }
1558         return;
1559 }
1560
1561 static void
1562 qla_drain_fp_taskqueues(qla_host_t *ha)
1563 {
1564         int     i;
1565
1566         for (i = 0; i < ha->hw.num_sds_rings; i++) {
1567                 qla_tx_fp_t *fp = &ha->tx_fp[i];
1568
1569                 if (fp->fp_taskqueue != NULL) {
1570                         taskqueue_drain_all(fp->fp_taskqueue);
1571                 }
1572         }
1573         return;
1574 }
1575
1576 static int
1577 qla_transmit(struct ifnet *ifp, struct mbuf  *mp)
1578 {
1579         qla_host_t *ha = (qla_host_t *)ifp->if_softc;
1580         qla_tx_fp_t *fp;
1581         int rss_id = 0;
1582         int ret = 0;
1583
1584         QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
1585
1586 #if __FreeBSD_version >= 1100000
1587         if (M_HASHTYPE_GET(mp) != M_HASHTYPE_NONE)
1588 #else
1589         if (mp->m_flags & M_FLOWID)
1590 #endif
1591                 rss_id = (mp->m_pkthdr.flowid & Q8_RSS_IND_TBL_MAX_IDX) %
1592                                         ha->hw.num_sds_rings;
1593         fp = &ha->tx_fp[rss_id];
1594
1595         if (fp->tx_br == NULL) {
1596                 ret = EINVAL;
1597                 goto qla_transmit_exit;
1598         }
1599
1600         if (mp != NULL) {
1601                 ret = drbr_enqueue(ifp, fp->tx_br, mp);
1602         }
1603
1604         if (fp->fp_taskqueue != NULL)
1605                 taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task);
1606
1607         ret = 0;
1608
1609 qla_transmit_exit:
1610
1611         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit ret = %d\n", __func__, ret));
1612         return ret;
1613 }
1614
1615 static void
1616 qla_qflush(struct ifnet *ifp)
1617 {
1618         int                     i;
1619         qla_tx_fp_t             *fp;
1620         struct mbuf             *mp;
1621         qla_host_t              *ha;
1622
1623         ha = (qla_host_t *)ifp->if_softc;
1624
1625         QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
1626
1627         for (i = 0; i < ha->hw.num_sds_rings; i++) {
1628                 fp = &ha->tx_fp[i];
1629
1630                 if (fp == NULL)
1631                         continue;
1632
1633                 if (fp->tx_br) {
1634                         mtx_lock(&fp->tx_mtx);
1635
1636                         while ((mp = drbr_dequeue(ifp, fp->tx_br)) != NULL) {
1637                                 m_freem(mp);
1638                         }
1639                         mtx_unlock(&fp->tx_mtx);
1640                 }
1641         }
1642         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
1643
1644         return;
1645 }
1646
1647 static void
1648 qla_stop(qla_host_t *ha)
1649 {
1650         struct ifnet *ifp = ha->ifp;
1651         device_t        dev;
1652         int i = 0;
1653
1654         ql_sp_log(ha, 13, 0, 0, 0, 0, 0, 0);
1655
1656         dev = ha->pci_dev;
1657
1658         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1659         ha->qla_watchdog_pause = 1;
1660
1661         for (i = 0; i < ha->hw.num_sds_rings; i++) {
1662                 qla_tx_fp_t *fp;
1663
1664                 fp = &ha->tx_fp[i];
1665
1666                 if (fp == NULL)
1667                         continue;
1668
1669                 if (fp->tx_br != NULL) {
1670                         mtx_lock(&fp->tx_mtx);
1671                         mtx_unlock(&fp->tx_mtx);
1672                 }
1673         }
1674
1675         while (!ha->qla_watchdog_paused)
1676                 qla_mdelay(__func__, 1);
1677
1678         ha->qla_interface_up = 0;
1679
1680         qla_drain_fp_taskqueues(ha);
1681
1682         ql_del_hw_if(ha);
1683
1684         qla_free_xmt_bufs(ha);
1685         qla_free_rcv_bufs(ha);
1686
1687         return;
1688 }
1689
1690 /*
1691  * Buffer Management Functions for Transmit and Receive Rings
1692  */
1693 static int
1694 qla_alloc_xmt_bufs(qla_host_t *ha)
1695 {
1696         int ret = 0;
1697         uint32_t i, j;
1698         qla_tx_buf_t *txb;
1699
1700         if (bus_dma_tag_create(NULL,    /* parent */
1701                 1, 0,    /* alignment, bounds */
1702                 BUS_SPACE_MAXADDR,       /* lowaddr */
1703                 BUS_SPACE_MAXADDR,       /* highaddr */
1704                 NULL, NULL,      /* filter, filterarg */
1705                 QLA_MAX_TSO_FRAME_SIZE,     /* maxsize */
1706                 QLA_MAX_SEGMENTS,        /* nsegments */
1707                 PAGE_SIZE,        /* maxsegsize */
1708                 BUS_DMA_ALLOCNOW,        /* flags */
1709                 NULL,    /* lockfunc */
1710                 NULL,    /* lockfuncarg */
1711                 &ha->tx_tag)) {
1712                 device_printf(ha->pci_dev, "%s: tx_tag alloc failed\n",
1713                         __func__);
1714                 return (ENOMEM);
1715         }
1716
1717         for (i = 0; i < ha->hw.num_tx_rings; i++) {
1718                 bzero((void *)ha->tx_ring[i].tx_buf,
1719                         (sizeof(qla_tx_buf_t) * NUM_TX_DESCRIPTORS));
1720         }
1721
1722         for (j = 0; j < ha->hw.num_tx_rings; j++) {
1723                 for (i = 0; i < NUM_TX_DESCRIPTORS; i++) {
1724                         txb = &ha->tx_ring[j].tx_buf[i];
1725
1726                         if ((ret = bus_dmamap_create(ha->tx_tag,
1727                                         BUS_DMA_NOWAIT, &txb->map))) {
1728                                 ha->err_tx_dmamap_create++;
1729                                 device_printf(ha->pci_dev,
1730                                         "%s: bus_dmamap_create failed[%d]\n",
1731                                         __func__, ret);
1732
1733                                 qla_free_xmt_bufs(ha);
1734
1735                                 return (ret);
1736                         }
1737                 }
1738         }
1739
1740         return 0;
1741 }
1742
1743 /*
1744  * Release mbuf after it sent on the wire
1745  */
1746 static void
1747 qla_clear_tx_buf(qla_host_t *ha, qla_tx_buf_t *txb)
1748 {
1749         QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
1750
1751         if (txb->m_head) {
1752                 bus_dmamap_sync(ha->tx_tag, txb->map,
1753                         BUS_DMASYNC_POSTWRITE);
1754
1755                 bus_dmamap_unload(ha->tx_tag, txb->map);
1756
1757                 m_freem(txb->m_head);
1758                 txb->m_head = NULL;
1759
1760                 bus_dmamap_destroy(ha->tx_tag, txb->map);
1761                 txb->map = NULL;
1762         }
1763
1764         if (txb->map) {
1765                 bus_dmamap_unload(ha->tx_tag, txb->map);
1766                 bus_dmamap_destroy(ha->tx_tag, txb->map);
1767                 txb->map = NULL;
1768         }
1769
1770         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
1771 }
1772
1773 static void
1774 qla_free_xmt_bufs(qla_host_t *ha)
1775 {
1776         int             i, j;
1777
1778         for (j = 0; j < ha->hw.num_tx_rings; j++) {
1779                 for (i = 0; i < NUM_TX_DESCRIPTORS; i++)
1780                         qla_clear_tx_buf(ha, &ha->tx_ring[j].tx_buf[i]);
1781         }
1782
1783         if (ha->tx_tag != NULL) {
1784                 bus_dma_tag_destroy(ha->tx_tag);
1785                 ha->tx_tag = NULL;
1786         }
1787
1788         for (i = 0; i < ha->hw.num_tx_rings; i++) {
1789                 bzero((void *)ha->tx_ring[i].tx_buf,
1790                         (sizeof(qla_tx_buf_t) * NUM_TX_DESCRIPTORS));
1791         }
1792         return;
1793 }
1794
1795 static int
1796 qla_alloc_rcv_std(qla_host_t *ha)
1797 {
1798         int             i, j, k, r, ret = 0;
1799         qla_rx_buf_t    *rxb;
1800         qla_rx_ring_t   *rx_ring;
1801
1802         for (r = 0; r < ha->hw.num_rds_rings; r++) {
1803                 rx_ring = &ha->rx_ring[r];
1804
1805                 for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
1806                         rxb = &rx_ring->rx_buf[i];
1807
1808                         ret = bus_dmamap_create(ha->rx_tag, BUS_DMA_NOWAIT,
1809                                         &rxb->map);
1810
1811                         if (ret) {
1812                                 device_printf(ha->pci_dev,
1813                                         "%s: dmamap[%d, %d] failed\n",
1814                                         __func__, r, i);
1815
1816                                 for (k = 0; k < r; k++) {
1817                                         for (j = 0; j < NUM_RX_DESCRIPTORS;
1818                                                 j++) {
1819                                                 rxb = &ha->rx_ring[k].rx_buf[j];
1820                                                 bus_dmamap_destroy(ha->rx_tag,
1821                                                         rxb->map);
1822                                         }
1823                                 }
1824
1825                                 for (j = 0; j < i; j++) {
1826                                         bus_dmamap_destroy(ha->rx_tag,
1827                                                 rx_ring->rx_buf[j].map);
1828                                 }
1829                                 goto qla_alloc_rcv_std_err;
1830                         }
1831                 }
1832         }
1833
1834         qla_init_hw_rcv_descriptors(ha);
1835
1836         for (r = 0; r < ha->hw.num_rds_rings; r++) {
1837                 rx_ring = &ha->rx_ring[r];
1838
1839                 for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
1840                         rxb = &rx_ring->rx_buf[i];
1841                         rxb->handle = i;
1842                         if (!(ret = ql_get_mbuf(ha, rxb, NULL))) {
1843                                 /*
1844                                  * set the physical address in the
1845                                  * corresponding descriptor entry in the
1846                                  * receive ring/queue for the hba
1847                                  */
1848                                 qla_set_hw_rcv_desc(ha, r, i, rxb->handle,
1849                                         rxb->paddr,
1850                                         (rxb->m_head)->m_pkthdr.len);
1851                         } else {
1852                                 device_printf(ha->pci_dev,
1853                                         "%s: ql_get_mbuf [%d, %d] failed\n",
1854                                         __func__, r, i);
1855                                 bus_dmamap_destroy(ha->rx_tag, rxb->map);
1856                                 goto qla_alloc_rcv_std_err;
1857                         }
1858                 }
1859         }
1860         return 0;
1861
1862 qla_alloc_rcv_std_err:
1863         return (-1);
1864 }
1865
1866 static void
1867 qla_free_rcv_std(qla_host_t *ha)
1868 {
1869         int             i, r;
1870         qla_rx_buf_t    *rxb;
1871
1872         for (r = 0; r < ha->hw.num_rds_rings; r++) {
1873                 for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
1874                         rxb = &ha->rx_ring[r].rx_buf[i];
1875                         if (rxb->m_head != NULL) {
1876                                 bus_dmamap_unload(ha->rx_tag, rxb->map);
1877                                 bus_dmamap_destroy(ha->rx_tag, rxb->map);
1878                                 m_freem(rxb->m_head);
1879                                 rxb->m_head = NULL;
1880                         }
1881                 }
1882         }
1883         return;
1884 }
1885
1886 static int
1887 qla_alloc_rcv_bufs(qla_host_t *ha)
1888 {
1889         int             i, ret = 0;
1890
1891         if (bus_dma_tag_create(NULL,    /* parent */
1892                         1, 0,    /* alignment, bounds */
1893                         BUS_SPACE_MAXADDR,       /* lowaddr */
1894                         BUS_SPACE_MAXADDR,       /* highaddr */
1895                         NULL, NULL,      /* filter, filterarg */
1896                         MJUM9BYTES,     /* maxsize */
1897                         1,        /* nsegments */
1898                         MJUM9BYTES,        /* maxsegsize */
1899                         BUS_DMA_ALLOCNOW,        /* flags */
1900                         NULL,    /* lockfunc */
1901                         NULL,    /* lockfuncarg */
1902                         &ha->rx_tag)) {
1903                 device_printf(ha->pci_dev, "%s: rx_tag alloc failed\n",
1904                         __func__);
1905
1906                 return (ENOMEM);
1907         }
1908
1909         bzero((void *)ha->rx_ring, (sizeof(qla_rx_ring_t) * MAX_RDS_RINGS));
1910
1911         for (i = 0; i < ha->hw.num_sds_rings; i++) {
1912                 ha->hw.sds[i].sdsr_next = 0;
1913                 ha->hw.sds[i].rxb_free = NULL;
1914                 ha->hw.sds[i].rx_free = 0;
1915         }
1916
1917         ret = qla_alloc_rcv_std(ha);
1918
1919         return (ret);
1920 }
1921
1922 static void
1923 qla_free_rcv_bufs(qla_host_t *ha)
1924 {
1925         int             i;
1926
1927         qla_free_rcv_std(ha);
1928
1929         if (ha->rx_tag != NULL) {
1930                 bus_dma_tag_destroy(ha->rx_tag);
1931                 ha->rx_tag = NULL;
1932         }
1933
1934         bzero((void *)ha->rx_ring, (sizeof(qla_rx_ring_t) * MAX_RDS_RINGS));
1935
1936         for (i = 0; i < ha->hw.num_sds_rings; i++) {
1937                 ha->hw.sds[i].sdsr_next = 0;
1938                 ha->hw.sds[i].rxb_free = NULL;
1939                 ha->hw.sds[i].rx_free = 0;
1940         }
1941
1942         return;
1943 }
1944
1945 int
1946 ql_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp)
1947 {
1948         register struct mbuf *mp = nmp;
1949         struct ifnet            *ifp;
1950         int                     ret = 0;
1951         uint32_t                offset;
1952         bus_dma_segment_t       segs[1];
1953         int                     nsegs, mbuf_size;
1954
1955         QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
1956
1957         ifp = ha->ifp;
1958
1959         if (ha->hw.enable_9kb)
1960                 mbuf_size = MJUM9BYTES;
1961         else
1962                 mbuf_size = MCLBYTES;
1963
1964         if (mp == NULL) {
1965                 if (QL_ERR_INJECT(ha, INJCT_M_GETCL_M_GETJCL_FAILURE))
1966                         return(-1);
1967
1968                 if (ha->hw.enable_9kb)
1969                         mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, mbuf_size);
1970                 else
1971                         mp = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1972
1973                 if (mp == NULL) {
1974                         ha->err_m_getcl++;
1975                         ret = ENOBUFS;
1976                         device_printf(ha->pci_dev,
1977                                         "%s: m_getcl failed\n", __func__);
1978                         goto exit_ql_get_mbuf;
1979                 }
1980                 mp->m_len = mp->m_pkthdr.len = mbuf_size;
1981         } else {
1982                 mp->m_len = mp->m_pkthdr.len = mbuf_size;
1983                 mp->m_data = mp->m_ext.ext_buf;
1984                 mp->m_next = NULL;
1985         }
1986
1987         offset = (uint32_t)((unsigned long long)mp->m_data & 0x7ULL);
1988         if (offset) {
1989                 offset = 8 - offset;
1990                 m_adj(mp, offset);
1991         }
1992
1993         /*
1994          * Using memory from the mbuf cluster pool, invoke the bus_dma
1995          * machinery to arrange the memory mapping.
1996          */
1997         ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, rxb->map,
1998                         mp, segs, &nsegs, BUS_DMA_NOWAIT);
1999         rxb->paddr = segs[0].ds_addr;
2000
2001         if (ret || !rxb->paddr || (nsegs != 1)) {
2002                 m_free(mp);
2003                 rxb->m_head = NULL;
2004                 device_printf(ha->pci_dev,
2005                         "%s: bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
2006                         __func__, ret, (long long unsigned int)rxb->paddr,
2007                         nsegs);
2008                 ret = -1;
2009                 goto exit_ql_get_mbuf;
2010         }
2011         rxb->m_head = mp;
2012         bus_dmamap_sync(ha->rx_tag, rxb->map, BUS_DMASYNC_PREREAD);
2013
2014 exit_ql_get_mbuf:
2015         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit ret = 0x%08x\n", __func__, ret));
2016         return (ret);
2017 }
2018
2019 static void
2020 qla_get_peer(qla_host_t *ha)
2021 {
2022         device_t *peers;
2023         int count, i, slot;
2024         int my_slot = pci_get_slot(ha->pci_dev);
2025
2026         if (device_get_children(device_get_parent(ha->pci_dev), &peers, &count))
2027                 return;
2028
2029         for (i = 0; i < count; i++) {
2030                 slot = pci_get_slot(peers[i]);
2031
2032                 if ((slot >= 0) && (slot == my_slot) &&
2033                         (pci_get_device(peers[i]) ==
2034                                 pci_get_device(ha->pci_dev))) {
2035                         if (ha->pci_dev != peers[i])
2036                                 ha->peer_dev = peers[i];
2037                 }
2038         }
2039 }
2040
2041 static void
2042 qla_send_msg_to_peer(qla_host_t *ha, uint32_t msg_to_peer)
2043 {
2044         qla_host_t *ha_peer;
2045
2046         if (ha->peer_dev) {
2047                 if ((ha_peer = device_get_softc(ha->peer_dev)) != NULL) {
2048                         ha_peer->msg_from_peer = msg_to_peer;
2049                 }
2050         }
2051 }
2052
2053 void
2054 qla_set_error_recovery(qla_host_t *ha)
2055 {
2056         struct ifnet *ifp = ha->ifp;
2057
2058         if (!cold && ha->enable_error_recovery) {
2059                 if (ifp)
2060                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2061                 ha->qla_initiate_recovery = 1;
2062         } else
2063                 ha->offline = 1;
2064         return;
2065 }
2066
2067 static void
2068 qla_error_recovery(void *context, int pending)
2069 {
2070         qla_host_t *ha = context;
2071         uint32_t msecs_100 = 400;
2072         struct ifnet *ifp = ha->ifp;
2073         int i = 0;
2074
2075         device_printf(ha->pci_dev, "%s: enter\n", __func__);
2076         ha->hw.imd_compl = 1;
2077
2078         taskqueue_drain_all(ha->stats_tq);
2079         taskqueue_drain_all(ha->async_event_tq);
2080
2081         if (QLA_LOCK(ha, __func__, -1, 0) != 0)
2082                 return;
2083
2084         device_printf(ha->pci_dev, "%s: ts_usecs = %ld start\n",
2085                 __func__, qla_get_usec_timestamp());
2086
2087         if (ha->qla_interface_up) {
2088                 qla_mdelay(__func__, 300);
2089
2090                 //ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2091
2092                 for (i = 0; i < ha->hw.num_sds_rings; i++) {
2093                         qla_tx_fp_t *fp;
2094
2095                         fp = &ha->tx_fp[i];
2096
2097                         if (fp == NULL)
2098                                 continue;
2099
2100                         if (fp->tx_br != NULL) {
2101                                 mtx_lock(&fp->tx_mtx);
2102                                 mtx_unlock(&fp->tx_mtx);
2103                         }
2104                 }
2105         }
2106
2107         qla_drain_fp_taskqueues(ha);
2108
2109         if ((ha->pci_func & 0x1) == 0) {
2110                 if (!ha->msg_from_peer) {
2111                         qla_send_msg_to_peer(ha, QL_PEER_MSG_RESET);
2112
2113                         while ((ha->msg_from_peer != QL_PEER_MSG_ACK) &&
2114                                 msecs_100--)
2115                                 qla_mdelay(__func__, 100);
2116                 }
2117
2118                 ha->msg_from_peer = 0;
2119
2120                 if (ha->enable_minidump)
2121                         ql_minidump(ha);
2122
2123                 if (ha->enable_driverstate_dump)
2124                         ql_capture_drvr_state(ha);
2125
2126                 if (ql_init_hw(ha)) {
2127                         device_printf(ha->pci_dev,
2128                                 "%s: ts_usecs = %ld exit: ql_init_hw failed\n",
2129                                 __func__, qla_get_usec_timestamp());
2130                         ha->offline = 1;
2131                         goto qla_error_recovery_exit;
2132                 }
2133
2134                 if (ha->qla_interface_up) {
2135                         qla_free_xmt_bufs(ha);
2136                         qla_free_rcv_bufs(ha);
2137                 }
2138
2139                 if (!QL_ERR_INJECT(ha, INJCT_PEER_PORT_FAILURE_ERR_RECOVERY))
2140                         qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK);
2141
2142         } else {
2143                 if (ha->msg_from_peer == QL_PEER_MSG_RESET) {
2144                         ha->msg_from_peer = 0;
2145
2146                         if (!QL_ERR_INJECT(ha, INJCT_PEER_PORT_FAILURE_ERR_RECOVERY))
2147                                 qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK);
2148                 } else {
2149                         qla_send_msg_to_peer(ha, QL_PEER_MSG_RESET);
2150                 }
2151
2152                 while ((ha->msg_from_peer != QL_PEER_MSG_ACK)  && msecs_100--)
2153                         qla_mdelay(__func__, 100);
2154                 ha->msg_from_peer = 0;
2155
2156                 if (ha->enable_driverstate_dump)
2157                         ql_capture_drvr_state(ha);
2158
2159                 if (msecs_100 == 0) {
2160                         device_printf(ha->pci_dev,
2161                                 "%s: ts_usecs = %ld exit: QL_PEER_MSG_ACK not received\n",
2162                                 __func__, qla_get_usec_timestamp());
2163                         ha->offline = 1;
2164                         goto qla_error_recovery_exit;
2165                 }
2166
2167                 if (ql_init_hw(ha)) {
2168                         device_printf(ha->pci_dev,
2169                                 "%s: ts_usecs = %ld exit: ql_init_hw failed\n",
2170                                 __func__, qla_get_usec_timestamp());
2171                         ha->offline = 1;
2172                         goto qla_error_recovery_exit;
2173                 }
2174
2175                 if (ha->qla_interface_up) {
2176                         qla_free_xmt_bufs(ha);
2177                         qla_free_rcv_bufs(ha);
2178                 }
2179         }
2180
2181         qla_mdelay(__func__, ha->ms_delay_after_init);
2182
2183         *((uint32_t *)&ha->hw.flags) = 0;
2184         ha->qla_initiate_recovery = 0;
2185
2186         if (ha->qla_interface_up) {
2187                 if (qla_alloc_xmt_bufs(ha) != 0) {
2188                         ha->offline = 1;
2189                         goto qla_error_recovery_exit;
2190                 }
2191
2192                 qla_confirm_9kb_enable(ha);
2193
2194                 if (qla_alloc_rcv_bufs(ha) != 0) {
2195                         ha->offline = 1;
2196                         goto qla_error_recovery_exit;
2197                 }
2198
2199                 ha->stop_rcv = 0;
2200
2201                 if (ql_init_hw_if(ha) == 0) {
2202                         ifp = ha->ifp;
2203                         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2204                         ha->qla_watchdog_pause = 0;
2205                         ql_update_link_state(ha);
2206                 } else {
2207                         ha->offline = 1;
2208
2209                         if (ha->hw.sp_log_stop_events &
2210                                 Q8_SP_LOG_STOP_IF_START_FAILURE)
2211                                 ha->hw.sp_log_stop = -1;
2212                 }
2213         } else {
2214                 ha->qla_watchdog_pause = 0;
2215         }
2216
2217 qla_error_recovery_exit:
2218
2219         if (ha->offline ) {
2220                 device_printf(ha->pci_dev, "%s: ts_usecs = %ld port offline\n",
2221                         __func__, qla_get_usec_timestamp());
2222                 if (ha->hw.sp_log_stop_events &
2223                         Q8_SP_LOG_STOP_ERR_RECOVERY_FAILURE)
2224                         ha->hw.sp_log_stop = -1;
2225         }
2226
2227         QLA_UNLOCK(ha, __func__);
2228
2229         if (!ha->offline)
2230                 callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
2231                         qla_watchdog, ha);
2232
2233         device_printf(ha->pci_dev,
2234                 "%s: ts_usecs = %ld exit\n",
2235                 __func__, qla_get_usec_timestamp());
2236         return;
2237 }
2238
2239 static void
2240 qla_async_event(void *context, int pending)
2241 {
2242         qla_host_t *ha = context;
2243
2244         if (QLA_LOCK(ha, __func__, -1, 0) != 0)
2245                 return;
2246
2247         if (ha->async_event) {
2248                 ha->async_event = 0;
2249                 qla_hw_async_event(ha);
2250         }
2251
2252         QLA_UNLOCK(ha, __func__);
2253
2254         return;
2255 }
2256
2257 static void
2258 qla_stats(void *context, int pending)
2259 {
2260         qla_host_t *ha;
2261
2262         ha = context;
2263
2264         ql_get_stats(ha);
2265
2266         return;
2267 }