sys/dev/liquidio/lio_main.c

   1 /*
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Cavium, Inc. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 /*$FreeBSD$*/
  34
  35 #include "lio_bsd.h"
  36 #include "lio_common.h"
  37
  38 #include "lio_droq.h"
  39 #include "lio_iq.h"
  40 #include "lio_response_manager.h"
  41 #include "lio_device.h"
  42 #include "lio_ctrl.h"
  43 #include "lio_main.h"
  44 #include "lio_network.h"
  45 #include "cn23xx_pf_device.h"
  46 #include "lio_image.h"
  47 #include "lio_ioctl.h"
  48 #include "lio_rxtx.h"
  49 #include "lio_rss.h"
  50
  51 /* Number of milliseconds to wait for DDR initialization */
  52 #define LIO_DDR_TIMEOUT 10000
  53 #define LIO_MAX_FW_TYPE_LEN     8
  54
  55 static char fw_type[LIO_MAX_FW_TYPE_LEN];
  56 TUNABLE_STR("hw.lio.fw_type", fw_type, sizeof(fw_type));
  57
  58 /*
  59  * Integers that specify number of queues per PF.
  60  * Valid range is 0 to 64.
  61  * Use 0 to derive from CPU count.
  62  */
  63 static int      num_queues_per_pf0;
  64 static int      num_queues_per_pf1;
  65 TUNABLE_INT("hw.lio.num_queues_per_pf0", &num_queues_per_pf0);
  66 TUNABLE_INT("hw.lio.num_queues_per_pf1", &num_queues_per_pf1);
  67
  68 #ifdef RSS
  69 static int      lio_rss = 1;
  70 TUNABLE_INT("hw.lio.rss", &lio_rss);
  71 #endif  /* RSS */
  72
  73 /* Hardware LRO */
  74 unsigned int    lio_hwlro = 0;
  75 TUNABLE_INT("hw.lio.hwlro", &lio_hwlro);
  76
  77 /*
  78  * Bitmask indicating which consoles have debug
  79  * output redirected to syslog.
  80  */
  81 static unsigned long    console_bitmask;
  82 TUNABLE_ULONG("hw.lio.console_bitmask", &console_bitmask);
  83
  84 /*
  85  * \brief determines if a given console has debug enabled.
  86  * @param console console to check
  87  * @returns  1 = enabled. 0 otherwise
  88  */
  89 int
  90 lio_console_debug_enabled(uint32_t console)
  91 {
  92
  93         return (console_bitmask >> (console)) & 0x1;
  94 }
  95
  96 static int      lio_detach(device_t dev);
  97
  98 static int      lio_device_init(struct octeon_device *octeon_dev);
  99 static int      lio_chip_specific_setup(struct octeon_device *oct);
 100 static void     lio_watchdog(void *param);
 101 static int      lio_load_firmware(struct octeon_device *oct);
 102 static int      lio_nic_starter(struct octeon_device *oct);
 103 static int      lio_init_nic_module(struct octeon_device *oct);
 104 static int      lio_setup_nic_devices(struct octeon_device *octeon_dev);
 105 static int      lio_link_info(struct lio_recv_info *recv_info, void *ptr);
 106 static void     lio_if_cfg_callback(struct octeon_device *oct, uint32_t status,
 107                                     void *buf);
 108 static int      lio_set_rxcsum_command(struct ifnet *ifp, int command,
 109                                        uint8_t rx_cmd);
 110 static int      lio_setup_glists(struct octeon_device *oct, struct lio *lio,
 111                                  int num_iqs);
 112 static void     lio_destroy_nic_device(struct octeon_device *oct, int ifidx);
 113 static inline void      lio_update_link_status(struct ifnet *ifp,
 114                                                union octeon_link_status *ls);
 115 static void     lio_send_rx_ctrl_cmd(struct lio *lio, int start_stop);
 116 static int      lio_stop_nic_module(struct octeon_device *oct);
 117 static void     lio_destroy_resources(struct octeon_device *oct);
 118 static int      lio_setup_rx_oom_poll_fn(struct ifnet *ifp);
 119
 120 static void     lio_vlan_rx_add_vid(void *arg, struct ifnet *ifp, uint16_t vid);
 121 static void     lio_vlan_rx_kill_vid(void *arg, struct ifnet *ifp,
 122                                      uint16_t vid);
 123 static struct octeon_device *
 124         lio_get_other_octeon_device(struct octeon_device *oct);
 125
 126 static int      lio_wait_for_oq_pkts(struct octeon_device *oct);
 127
 128 int     lio_send_rss_param(struct lio *lio);
 129 static int      lio_dbg_console_print(struct octeon_device *oct,
 130                                       uint32_t console_num, char *prefix,
 131                                       char *suffix);
 132
 133 /* Polling interval for determining when NIC application is alive */
 134 #define LIO_STARTER_POLL_INTERVAL_MS    100
 135
 136 /*
 137  * vendor_info_array.
 138  * This array contains the list of IDs on which the driver should load.
 139  */
 140 struct lio_vendor_info {
 141         uint16_t        vendor_id;
 142         uint16_t        device_id;
 143         uint16_t        subdevice_id;
 144         uint8_t         revision_id;
 145         uint8_t         index;
 146 };
 147
 148 static struct lio_vendor_info lio_pci_tbl[] = {
 149         /* CN2350 10G */
 150         {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_10G_SUBDEVICE,
 151                 0x02, 0},
 152
 153         /* CN2350 10G */
 154         {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_10G_SUBDEVICE1,
 155                 0x02, 0},
 156
 157         /* CN2360 10G */
 158         {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2360_10G_SUBDEVICE,
 159                 0x02, 1},
 160
 161         /* CN2350 25G */
 162         {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_25G_SUBDEVICE,
 163                 0x02, 2},
 164
 165         /* CN2360 25G */
 166         {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2360_25G_SUBDEVICE,
 167                 0x02, 3},
 168
 169         {0, 0, 0, 0, 0}
 170 };
 171
 172 static char *lio_strings[] = {
 173         "LiquidIO 2350 10GbE Server Adapter",
 174         "LiquidIO 2360 10GbE Server Adapter",
 175         "LiquidIO 2350 25GbE Server Adapter",
 176         "LiquidIO 2360 25GbE Server Adapter",
 177 };
 178
 179 struct lio_if_cfg_resp {
 180         uint64_t        rh;
 181         struct octeon_if_cfg_info cfg_info;
 182         uint64_t        status;
 183 };
 184
 185 struct lio_if_cfg_context {
 186         int             octeon_id;
 187         volatile int    cond;
 188 };
 189
 190 struct lio_rx_ctl_context {
 191         int             octeon_id;
 192         volatile int    cond;
 193 };
 194
 195 static int
 196 lio_probe(device_t dev)
 197 {
 198         struct lio_vendor_info  *tbl;
 199
 200         uint16_t        vendor_id;
 201         uint16_t        device_id;
 202         uint16_t        subdevice_id;
 203         uint8_t         revision_id;
 204         char            device_ver[256];
 205
 206         vendor_id = pci_get_vendor(dev);
 207         if (vendor_id != PCI_VENDOR_ID_CAVIUM)
 208                 return (ENXIO);
 209
 210         device_id = pci_get_device(dev);
 211         subdevice_id = pci_get_subdevice(dev);
 212         revision_id = pci_get_revid(dev);
 213
 214         tbl = lio_pci_tbl;
 215         while (tbl->vendor_id) {
 216                 if ((vendor_id == tbl->vendor_id) &&
 217                     (device_id == tbl->device_id) &&
 218                     (subdevice_id == tbl->subdevice_id) &&
 219                     (revision_id == tbl->revision_id)) {
 220                         sprintf(device_ver, "%s, Version - %s",
 221                                 lio_strings[tbl->index], LIO_VERSION);
 222                         device_set_desc_copy(dev, device_ver);
 223                         return (BUS_PROBE_DEFAULT);
 224                 }
 225
 226                 tbl++;
 227         }
 228
 229         return (ENXIO);
 230 }
 231
 232 static int
 233 lio_attach(device_t device)
 234 {
 235         struct octeon_device    *oct_dev = NULL;
 236         uint64_t        scratch1;
 237         uint32_t        error;
 238         int             timeout, ret = 1;
 239         uint8_t         bus, dev, function;
 240
 241         oct_dev = lio_allocate_device(device);
 242         if (oct_dev == NULL) {
 243                 device_printf(device, "Error: Unable to allocate device\n");
 244                 return (-ENOMEM);
 245         }
 246
 247         oct_dev->tx_budget = LIO_DEFAULT_TX_PKTS_PROCESS_BUDGET;
 248         oct_dev->rx_budget = LIO_DEFAULT_RX_PKTS_PROCESS_BUDGET;
 249         oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
 250
 251         oct_dev->device = device;
 252         bus = pci_get_bus(device);
 253         dev = pci_get_slot(device);
 254         function = pci_get_function(device);
 255
 256         lio_dev_info(oct_dev, "Initializing device %x:%x %02x:%02x.%01x\n",
 257                      pci_get_vendor(device), pci_get_device(device), bus, dev,
 258                      function);
 259
 260         if (lio_device_init(oct_dev)) {
 261                 lio_dev_err(oct_dev, "Failed to init device\n");
 262                 lio_detach(device);
 263                 return (-ENOMEM);
 264         }
 265
 266         scratch1 = lio_read_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1);
 267         if (!(scratch1 & 4ULL)) {
 268                 /*
 269                  * Bit 2 of SLI_SCRATCH_1 is a flag that indicates that
 270                  * the lio watchdog kernel thread is running for this
 271                  * NIC.  Each NIC gets one watchdog kernel thread.
 272                  */
 273                 scratch1 |= 4ULL;
 274                 lio_write_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1, scratch1);
 275
 276                 error = kproc_create(lio_watchdog, oct_dev,
 277                                      &oct_dev->watchdog_task, 0, 0,
 278                                      "liowd/%02hhx:%02hhx.%hhx", bus,
 279                                      dev, function);
 280                 if (!error) {
 281                         kproc_resume(oct_dev->watchdog_task);
 282                 } else {
 283                         oct_dev->watchdog_task = NULL;
 284                         lio_dev_err(oct_dev,
 285                                     "failed to create kernel_thread\n");
 286                         lio_detach(device);
 287                         return (-1);
 288                 }
 289         }
 290         oct_dev->rx_pause = 1;
 291         oct_dev->tx_pause = 1;
 292
 293         timeout = 0;
 294         while (timeout < LIO_NIC_STARTER_TIMEOUT) {
 295                 lio_mdelay(LIO_STARTER_POLL_INTERVAL_MS);
 296                 timeout += LIO_STARTER_POLL_INTERVAL_MS;
 297
 298                 /*
 299                  * During the boot process interrupts are not available.
 300                  * So polling for first control message from FW.
 301                  */
 302                 if (cold)
 303                         lio_droq_bh(oct_dev->droq[0], 0);
 304
 305                 if (atomic_load_acq_int(&oct_dev->status) == LIO_DEV_CORE_OK) {
 306                         ret = lio_nic_starter(oct_dev);
 307                         break;
 308                 }
 309         }
 310
 311         if (ret) {
 312                 lio_dev_err(oct_dev, "Firmware failed to start\n");
 313                 lio_detach(device);
 314                 return (-EIO);
 315         }
 316
 317         lio_dev_dbg(oct_dev, "Device is ready\n");
 318
 319         return (0);
 320 }
 321
 322 static int
 323 lio_detach(device_t dev)
 324 {
 325         struct octeon_device    *oct_dev = device_get_softc(dev);
 326
 327         lio_dev_dbg(oct_dev, "Stopping device\n");
 328         if (oct_dev->watchdog_task) {
 329                 uint64_t        scratch1;
 330
 331                 kproc_suspend(oct_dev->watchdog_task, 0);
 332
 333                 scratch1 = lio_read_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1);
 334                 scratch1 &= ~4ULL;
 335                 lio_write_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1, scratch1);
 336         }
 337
 338         if (oct_dev->app_mode && (oct_dev->app_mode == LIO_DRV_NIC_APP))
 339                 lio_stop_nic_module(oct_dev);
 340
 341         /*
 342          * Reset the octeon device and cleanup all memory allocated for
 343          * the octeon device by  driver.
 344          */
 345         lio_destroy_resources(oct_dev);
 346
 347         lio_dev_info(oct_dev, "Device removed\n");
 348
 349         /*
 350          * This octeon device has been removed. Update the global
 351          * data structure to reflect this. Free the device structure.
 352          */
 353         lio_free_device_mem(oct_dev);
 354         return (0);
 355 }
 356
 357 static int
 358 lio_shutdown(device_t dev)
 359 {
 360         struct octeon_device    *oct_dev = device_get_softc(dev);
 361         struct lio      *lio = if_getsoftc(oct_dev->props.ifp);
 362
 363         lio_send_rx_ctrl_cmd(lio, 0);
 364
 365         return (0);
 366 }
 367
 368 static int
 369 lio_suspend(device_t dev)
 370 {
 371
 372         return (ENXIO);
 373 }
 374
 375 static int
 376 lio_resume(device_t dev)
 377 {
 378
 379         return (ENXIO);
 380 }
 381
 382 static int
 383 lio_event(struct module *mod, int event, void *junk)
 384 {
 385
 386         switch (event) {
 387         case MOD_LOAD:
 388                 lio_init_device_list(LIO_CFG_TYPE_DEFAULT);
 389                 break;
 390         default:
 391                 break;
 392         }
 393
 394         return (0);
 395 }
 396
 397 /*********************************************************************
 398  *  FreeBSD Device Interface Entry Points
 399  * *******************************************************************/
 400 static device_method_t lio_methods[] = {
 401         /* Device interface */
 402         DEVMETHOD(device_probe, lio_probe),
 403         DEVMETHOD(device_attach, lio_attach),
 404         DEVMETHOD(device_detach, lio_detach),
 405         DEVMETHOD(device_shutdown, lio_shutdown),
 406         DEVMETHOD(device_suspend, lio_suspend),
 407         DEVMETHOD(device_resume, lio_resume),
 408         DEVMETHOD_END
 409 };
 410
 411 static driver_t lio_driver = {
 412         LIO_DRV_NAME, lio_methods, sizeof(struct octeon_device),
 413 };
 414
 415 devclass_t lio_devclass;
 416 DRIVER_MODULE(lio, pci, lio_driver, lio_devclass, lio_event, 0);
 417
 418 MODULE_DEPEND(lio, pci, 1, 1, 1);
 419 MODULE_DEPEND(lio, ether, 1, 1, 1);
 420 MODULE_DEPEND(lio, firmware, 1, 1, 1);
 421
 422 static bool
 423 fw_type_is_none(void)
 424 {
 425         return strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
 426                        sizeof(LIO_FW_NAME_TYPE_NONE)) == 0;
 427 }
 428
 429 /*
 430  * \brief Device initialization for each Octeon device that is probed
 431  * @param octeon_dev  octeon device
 432  */
 433 static int
 434 lio_device_init(struct octeon_device *octeon_dev)
 435 {
 436         unsigned long   ddr_timeout = LIO_DDR_TIMEOUT;
 437         char    *dbg_enb = NULL;
 438         int     fw_loaded = 0;
 439         int     i, j, ret;
 440         uint8_t bus, dev, function;
 441         char    bootcmd[] = "\n";
 442
 443         bus = pci_get_bus(octeon_dev->device);
 444         dev = pci_get_slot(octeon_dev->device);
 445         function = pci_get_function(octeon_dev->device);
 446
 447         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_BEGIN_STATE);
 448
 449         /* Enable access to the octeon device */
 450         if (pci_enable_busmaster(octeon_dev->device)) {
 451                 lio_dev_err(octeon_dev, "pci_enable_device failed\n");
 452                 return (1);
 453         }
 454
 455         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_PCI_ENABLE_DONE);
 456
 457         /* Identify the Octeon type and map the BAR address space. */
 458         if (lio_chip_specific_setup(octeon_dev)) {
 459                 lio_dev_err(octeon_dev, "Chip specific setup failed\n");
 460                 return (1);
 461         }
 462
 463         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_PCI_MAP_DONE);
 464
 465         /*
 466          * Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
 467          * since that is what is required for the reference to be removed
 468          * during de-initialization (see 'octeon_destroy_resources').
 469          */
 470         lio_register_device(octeon_dev, bus, dev, function, true);
 471
 472
 473         octeon_dev->app_mode = LIO_DRV_INVALID_APP;
 474
 475         if (!lio_cn23xx_pf_fw_loaded(octeon_dev) && !fw_type_is_none()) {
 476                 fw_loaded = 0;
 477                 /* Do a soft reset of the Octeon device. */
 478                 if (octeon_dev->fn_list.soft_reset(octeon_dev))
 479                         return (1);
 480
 481                 /* things might have changed */
 482                 if (!lio_cn23xx_pf_fw_loaded(octeon_dev))
 483                         fw_loaded = 0;
 484                 else
 485                         fw_loaded = 1;
 486         } else {
 487                 fw_loaded = 1;
 488         }
 489
 490         /*
 491          * Initialize the dispatch mechanism used to push packets arriving on
 492          * Octeon Output queues.
 493          */
 494         if (lio_init_dispatch_list(octeon_dev))
 495                 return (1);
 496
 497         lio_register_dispatch_fn(octeon_dev, LIO_OPCODE_NIC,
 498                                  LIO_OPCODE_NIC_CORE_DRV_ACTIVE,
 499                                  lio_core_drv_init, octeon_dev);
 500         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_DISPATCH_INIT_DONE);
 501
 502         ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
 503         if (ret) {
 504                 lio_dev_err(octeon_dev,
 505                             "Failed to configure device registers\n");
 506                 return (ret);
 507         }
 508
 509         /* Initialize soft command buffer pool */
 510         if (lio_setup_sc_buffer_pool(octeon_dev)) {
 511                 lio_dev_err(octeon_dev, "sc buffer pool allocation failed\n");
 512                 return (1);
 513         }
 514
 515         atomic_store_rel_int(&octeon_dev->status,
 516                              LIO_DEV_SC_BUFF_POOL_INIT_DONE);
 517
 518         if (lio_allocate_ioq_vector(octeon_dev)) {
 519                 lio_dev_err(octeon_dev,
 520                             "IOQ vector allocation failed\n");
 521                 return (1);
 522         }
 523
 524         atomic_store_rel_int(&octeon_dev->status,
 525                              LIO_DEV_MSIX_ALLOC_VECTOR_DONE);
 526
 527         for (i = 0; i < LIO_MAX_POSSIBLE_INSTR_QUEUES; i++) {
 528                 octeon_dev->instr_queue[i] =
 529                         malloc(sizeof(struct lio_instr_queue),
 530                                M_DEVBUF, M_NOWAIT | M_ZERO);
 531                 if (octeon_dev->instr_queue[i] == NULL)
 532                         return (1);
 533         }
 534
 535         /* Setup the data structures that manage this Octeon's Input queues. */
 536         if (lio_setup_instr_queue0(octeon_dev)) {
 537                 lio_dev_err(octeon_dev,
 538                             "Instruction queue initialization failed\n");
 539                 return (1);
 540         }
 541
 542         atomic_store_rel_int(&octeon_dev->status,
 543                              LIO_DEV_INSTR_QUEUE_INIT_DONE);
 544
 545         /*
 546          * Initialize lists to manage the requests of different types that
 547          * arrive from user & kernel applications for this octeon device.
 548          */
 549
 550         if (lio_setup_response_list(octeon_dev)) {
 551                 lio_dev_err(octeon_dev, "Response list allocation failed\n");
 552                 return (1);
 553         }
 554
 555         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_RESP_LIST_INIT_DONE);
 556
 557         for (i = 0; i < LIO_MAX_POSSIBLE_OUTPUT_QUEUES; i++) {
 558                 octeon_dev->droq[i] = malloc(sizeof(*octeon_dev->droq[i]),
 559                                              M_DEVBUF, M_NOWAIT | M_ZERO);
 560                 if (octeon_dev->droq[i] == NULL)
 561                         return (1);
 562         }
 563
 564         if (lio_setup_output_queue0(octeon_dev)) {
 565                 lio_dev_err(octeon_dev, "Output queue initialization failed\n");
 566                 return (1);
 567         }
 568
 569         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_DROQ_INIT_DONE);
 570
 571         /*
 572          * Setup the interrupt handler and record the INT SUM register address
 573          */
 574         if (lio_setup_interrupt(octeon_dev,
 575                                 octeon_dev->sriov_info.num_pf_rings))
 576                 return (1);
 577
 578         /* Enable Octeon device interrupts */
 579         octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
 580
 581         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_INTR_SET_DONE);
 582
 583         /*
 584          * Send Credit for Octeon Output queues. Credits are always sent BEFORE
 585          * the output queue is enabled.
 586          * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
 587          * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
 588          * Otherwise, it is possible that the DRV_ACTIVE message will be sent
 589          * before any credits have been issued, causing the ring to be reset
 590          * (and the f/w appear to never have started).
 591          */
 592         for (j = 0; j < octeon_dev->num_oqs; j++)
 593                 lio_write_csr32(octeon_dev,
 594                                 octeon_dev->droq[j]->pkts_credit_reg,
 595                                 octeon_dev->droq[j]->max_count);
 596
 597         /* Enable the input and output queues for this Octeon device */
 598         ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
 599         if (ret) {
 600                 lio_dev_err(octeon_dev, "Failed to enable input/output queues");
 601                 return (ret);
 602         }
 603
 604         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_IO_QUEUES_DONE);
 605
 606         if (!fw_loaded) {
 607                 lio_dev_dbg(octeon_dev, "Waiting for DDR initialization...\n");
 608                 if (!ddr_timeout) {
 609                         lio_dev_info(octeon_dev,
 610                                      "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
 611                 }
 612
 613                 lio_sleep_timeout(LIO_RESET_MSECS);
 614
 615                 /*
 616                  * Wait for the octeon to initialize DDR after the
 617                  * soft-reset.
 618                  */
 619                 while (!ddr_timeout) {
 620                         if (pause("-", lio_ms_to_ticks(100))) {
 621                                 /* user probably pressed Control-C */
 622                                 return (1);
 623                         }
 624                 }
 625
 626                 ret = lio_wait_for_ddr_init(octeon_dev, &ddr_timeout);
 627                 if (ret) {
 628                         lio_dev_err(octeon_dev,
 629                                     "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
 630                                     ret);
 631                         return (1);
 632                 }
 633
 634                 if (lio_wait_for_bootloader(octeon_dev, 1100)) {
 635                         lio_dev_err(octeon_dev, "Board not responding\n");
 636                         return (1);
 637                 }
 638
 639                 /* Divert uboot to take commands from host instead. */
 640                 ret = lio_console_send_cmd(octeon_dev, bootcmd, 50);
 641
 642                 lio_dev_dbg(octeon_dev, "Initializing consoles\n");
 643                 ret = lio_init_consoles(octeon_dev);
 644                 if (ret) {
 645                         lio_dev_err(octeon_dev, "Could not access board consoles\n");
 646                         return (1);
 647                 }
 648
 649                 /*
 650                  * If console debug enabled, specify empty string to
 651                  * use default enablement ELSE specify NULL string for
 652                  * 'disabled'.
 653                  */
 654                 dbg_enb = lio_console_debug_enabled(0) ? "" : NULL;
 655                 ret = lio_add_console(octeon_dev, 0, dbg_enb);
 656
 657                 if (ret) {
 658                         lio_dev_err(octeon_dev, "Could not access board console\n");
 659                         return (1);
 660                 } else if (lio_console_debug_enabled(0)) {
 661                         /*
 662                          * If console was added AND we're logging console output
 663                          * then set our console print function.
 664                          */
 665                         octeon_dev->console[0].print = lio_dbg_console_print;
 666                 }
 667
 668                 atomic_store_rel_int(&octeon_dev->status,
 669                                      LIO_DEV_CONSOLE_INIT_DONE);
 670
 671                 lio_dev_dbg(octeon_dev, "Loading firmware\n");
 672
 673                 ret = lio_load_firmware(octeon_dev);
 674                 if (ret) {
 675                         lio_dev_err(octeon_dev, "Could not load firmware to board\n");
 676                         return (1);
 677                 }
 678         }
 679
 680         atomic_store_rel_int(&octeon_dev->status, LIO_DEV_HOST_OK);
 681
 682         return (0);
 683 }
 684
 685 /*
 686  * \brief PCI FLR for each Octeon device.
 687  * @param oct octeon device
 688  */
 689 static void
 690 lio_pci_flr(struct octeon_device *oct)
 691 {
 692         uint32_t        exppos, status;
 693
 694         pci_find_cap(oct->device, PCIY_EXPRESS, &exppos);
 695
 696         pci_save_state(oct->device);
 697
 698         /* Quiesce the device completely */
 699         pci_write_config(oct->device, PCIR_COMMAND, PCIM_CMD_INTxDIS, 2);
 700
 701         /* Wait for Transaction Pending bit clean */
 702         lio_mdelay(100);
 703
 704         status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2);
 705         if (status & PCIEM_STA_TRANSACTION_PND) {
 706                 lio_dev_info(oct, "Function reset incomplete after 100ms, sleeping for 5 seconds\n");
 707                 lio_mdelay(5);
 708
 709                 status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2);
 710                 if (status & PCIEM_STA_TRANSACTION_PND)
 711                         lio_dev_info(oct, "Function reset still incomplete after 5s, reset anyway\n");
 712         }
 713
 714         pci_write_config(oct->device, exppos + PCIER_DEVICE_CTL, PCIEM_CTL_INITIATE_FLR, 2);
 715         lio_mdelay(100);
 716
 717         pci_restore_state(oct->device);
 718 }
 719
 720 /*
 721  * \brief Debug console print function
 722  * @param octeon_dev  octeon device
 723  * @param console_num console number
 724  * @param prefix      first portion of line to display
 725  * @param suffix      second portion of line to display
 726  *
 727  * The OCTEON debug console outputs entire lines (excluding '\n').
 728  * Normally, the line will be passed in the 'prefix' parameter.
 729  * However, due to buffering, it is possible for a line to be split into two
 730  * parts, in which case they will be passed as the 'prefix' parameter and
 731  * 'suffix' parameter.
 732  */
 733 static int
 734 lio_dbg_console_print(struct octeon_device *oct, uint32_t console_num,
 735                       char *prefix, char *suffix)
 736 {
 737
 738         if (prefix != NULL && suffix != NULL)
 739                 lio_dev_info(oct, "%u: %s%s\n", console_num, prefix, suffix);
 740         else if (prefix != NULL)
 741                 lio_dev_info(oct, "%u: %s\n", console_num, prefix);
 742         else if (suffix != NULL)
 743                 lio_dev_info(oct, "%u: %s\n", console_num, suffix);
 744
 745         return (0);
 746 }
 747
 748 static void
 749 lio_watchdog(void *param)
 750 {
 751         int             core_num;
 752         uint16_t        mask_of_crashed_or_stuck_cores = 0;
 753         struct octeon_device    *oct = param;
 754         bool            err_msg_was_printed[12];
 755
 756         bzero(err_msg_was_printed, sizeof(err_msg_was_printed));
 757
 758         while (1) {
 759                 kproc_suspend_check(oct->watchdog_task);
 760                 mask_of_crashed_or_stuck_cores =
 761                         (uint16_t)lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH2);
 762
 763                 if (mask_of_crashed_or_stuck_cores) {
 764                         struct octeon_device *other_oct;
 765
 766                         oct->cores_crashed = true;
 767                         other_oct = lio_get_other_octeon_device(oct);
 768                         if (other_oct != NULL)
 769                                 other_oct->cores_crashed = true;
 770
 771                         for (core_num = 0; core_num < LIO_MAX_CORES;
 772                              core_num++) {
 773                                 bool core_crashed_or_got_stuck;
 774
 775                                 core_crashed_or_got_stuck =
 776                                     (mask_of_crashed_or_stuck_cores >>
 777                                      core_num) & 1;
 778                                 if (core_crashed_or_got_stuck &&
 779                                     !err_msg_was_printed[core_num]) {
 780                                         lio_dev_err(oct,
 781                                                     "ERROR: Octeon core %d crashed or got stuck! See oct-fwdump for details.\n",
 782                                                     core_num);
 783                                         err_msg_was_printed[core_num] = true;
 784                                 }
 785                         }
 786
 787                 }
 788
 789                 /* sleep for two seconds */
 790                 pause("-", lio_ms_to_ticks(2000));
 791         }
 792 }
 793
 794 static int
 795 lio_chip_specific_setup(struct octeon_device *oct)
 796 {
 797         char            *s;
 798         uint32_t        dev_id, rev_id;
 799         int             ret = 1;
 800
 801         dev_id = lio_read_pci_cfg(oct, 0);
 802         rev_id = pci_get_revid(oct->device);
 803         oct->subdevice_id = pci_get_subdevice(oct->device);
 804
 805         switch (dev_id) {
 806         case LIO_CN23XX_PF_PCIID:
 807                 oct->chip_id = LIO_CN23XX_PF_VID;
 808                 if (pci_get_function(oct->device) == 0) {
 809                         if (num_queues_per_pf0 < 0) {
 810                                 lio_dev_info(oct, "Invalid num_queues_per_pf0: %d, Setting it to default\n",
 811                                              num_queues_per_pf0);
 812                                 num_queues_per_pf0 = 0;
 813                         }
 814
 815                         oct->sriov_info.num_pf_rings = num_queues_per_pf0;
 816                 } else {
 817                         if (num_queues_per_pf1 < 0) {
 818                                 lio_dev_info(oct, "Invalid num_queues_per_pf1: %d, Setting it to default\n",
 819                                              num_queues_per_pf1);
 820                                 num_queues_per_pf1 = 0;
 821                         }
 822
 823                         oct->sriov_info.num_pf_rings = num_queues_per_pf1;
 824                 }
 825
 826                 ret = lio_cn23xx_pf_setup_device(oct);
 827                 s = "CN23XX";
 828                 break;
 829
 830         default:
 831                 s = "?";
 832                 lio_dev_err(oct, "Unknown device found (dev_id: %x)\n", dev_id);
 833         }
 834
 835         if (!ret)
 836                 lio_dev_info(oct, "%s PASS%d.%d %s Version: %s\n", s,
 837                              OCTEON_MAJOR_REV(oct), OCTEON_MINOR_REV(oct),
 838                              lio_get_conf(oct)->card_name, LIO_VERSION);
 839
 840         return (ret);
 841 }
 842
 843 static struct octeon_device *
 844 lio_get_other_octeon_device(struct octeon_device *oct)
 845 {
 846         struct octeon_device    *other_oct;
 847
 848         other_oct = lio_get_device(oct->octeon_id + 1);
 849
 850         if ((other_oct != NULL) && other_oct->device) {
 851                 int     oct_busnum, other_oct_busnum;
 852
 853                 oct_busnum = pci_get_bus(oct->device);
 854                 other_oct_busnum = pci_get_bus(other_oct->device);
 855
 856                 if (oct_busnum == other_oct_busnum) {
 857                         int     oct_slot, other_oct_slot;
 858
 859                         oct_slot = pci_get_slot(oct->device);
 860                         other_oct_slot = pci_get_slot(other_oct->device);
 861
 862                         if (oct_slot == other_oct_slot)
 863                                 return (other_oct);
 864                 }
 865         }
 866         return (NULL);
 867 }
 868
 869 /*
 870  * \brief Load firmware to device
 871  * @param oct octeon device
 872  *
 873  * Maps device to firmware filename, requests firmware, and downloads it
 874  */
 875 static int
 876 lio_load_firmware(struct octeon_device *oct)
 877 {
 878         const struct firmware   *fw;
 879         char    *tmp_fw_type = NULL;
 880         int     ret = 0;
 881         char    fw_name[LIO_MAX_FW_FILENAME_LEN];
 882
 883         if (fw_type[0] == '\0')
 884                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
 885         else
 886                 tmp_fw_type = fw_type;
 887
 888         sprintf(fw_name, "%s%s_%s%s", LIO_FW_BASE_NAME,
 889                 lio_get_conf(oct)->card_name, tmp_fw_type, LIO_FW_NAME_SUFFIX);
 890
 891         fw = firmware_get(fw_name);
 892         if (fw == NULL) {
 893                 lio_dev_err(oct, "Request firmware failed. Could not find file %s.\n",
 894                             fw_name);
 895                 return (EINVAL);
 896         }
 897
 898         ret = lio_download_firmware(oct, fw->data, fw->datasize);
 899
 900         firmware_put(fw, FIRMWARE_UNLOAD);
 901
 902         return (ret);
 903 }
 904
 905 static int
 906 lio_nic_starter(struct octeon_device *oct)
 907 {
 908         int     ret = 0;
 909
 910         atomic_store_rel_int(&oct->status, LIO_DEV_RUNNING);
 911
 912         if (oct->app_mode && oct->app_mode == LIO_DRV_NIC_APP) {
 913                 if (lio_init_nic_module(oct)) {
 914                         lio_dev_err(oct, "NIC initialization failed\n");
 915                         ret = -1;
 916 #ifdef CAVIUM_ONiLY_23XX_VF
 917                 } else {
 918                         if (octeon_enable_sriov(oct) < 0)
 919                                 ret = -1;
 920 #endif
 921                 }
 922         } else {
 923                 lio_dev_err(oct,
 924                             "Unexpected application running on NIC (%d). Check firmware.\n",
 925                             oct->app_mode);
 926                 ret = -1;
 927         }
 928
 929         return (ret);
 930 }
 931
 932 static int
 933 lio_init_nic_module(struct octeon_device *oct)
 934 {
 935         int     num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG(lio_get_conf(oct));
 936         int     retval = 0;
 937
 938         lio_dev_dbg(oct, "Initializing network interfaces\n");
 939
 940         /*
 941          * only default iq and oq were initialized
 942          * initialize the rest as well
 943          */
 944
 945         /* run port_config command for each port */
 946         oct->ifcount = num_nic_ports;
 947
 948         bzero(&oct->props, sizeof(struct lio_if_props));
 949
 950         oct->props.gmxport = -1;
 951
 952         retval = lio_setup_nic_devices(oct);
 953         if (retval) {
 954                 lio_dev_err(oct, "Setup NIC devices failed\n");
 955                 goto lio_init_failure;
 956         }
 957
 958         lio_dev_dbg(oct, "Network interfaces ready\n");
 959
 960         return (retval);
 961
 962 lio_init_failure:
 963
 964         oct->ifcount = 0;
 965
 966         return (retval);
 967 }
 968
 969 static int
 970 lio_ifmedia_update(struct ifnet *ifp)
 971 {
 972         struct lio      *lio = if_getsoftc(ifp);
 973         struct ifmedia  *ifm;
 974
 975         ifm = &lio->ifmedia;
 976
 977         /* We only support Ethernet media type. */
 978         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 979                 return (EINVAL);
 980
 981         switch (IFM_SUBTYPE(ifm->ifm_media)) {
 982         case IFM_AUTO:
 983                 break;
 984         case IFM_10G_CX4:
 985         case IFM_10G_SR:
 986         case IFM_10G_T:
 987         case IFM_10G_TWINAX:
 988         default:
 989                 /* We don't support changing the media type. */
 990                 lio_dev_err(lio->oct_dev, "Invalid media type (%d)\n",
 991                             IFM_SUBTYPE(ifm->ifm_media));
 992                 return (EINVAL);
 993         }
 994
 995         return (0);
 996 }
 997
 998 static int
 999 lio_get_media_subtype(struct octeon_device *oct)
1000 {
1001
1002         switch(oct->subdevice_id) {
1003         case LIO_CN2350_10G_SUBDEVICE:
1004         case LIO_CN2350_10G_SUBDEVICE1:
1005         case LIO_CN2360_10G_SUBDEVICE:
1006                 return (IFM_10G_SR);
1007
1008         case LIO_CN2350_25G_SUBDEVICE:
1009         case LIO_CN2360_25G_SUBDEVICE:
1010                 return (IFM_25G_SR);
1011         }
1012
1013         return (IFM_10G_SR);
1014 }
1015
1016 static uint64_t
1017 lio_get_baudrate(struct octeon_device *oct)
1018 {
1019
1020         switch(oct->subdevice_id) {
1021         case LIO_CN2350_10G_SUBDEVICE:
1022         case LIO_CN2350_10G_SUBDEVICE1:
1023         case LIO_CN2360_10G_SUBDEVICE:
1024                 return (IF_Gbps(10));
1025
1026         case LIO_CN2350_25G_SUBDEVICE:
1027         case LIO_CN2360_25G_SUBDEVICE:
1028                 return (IF_Gbps(25));
1029         }
1030
1031         return (IF_Gbps(10));
1032 }
1033
1034 static void
1035 lio_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1036 {
1037         struct lio      *lio = if_getsoftc(ifp);
1038
1039         /* Report link down if the driver isn't running. */
1040         if (!lio_ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
1041                 ifmr->ifm_active |= IFM_NONE;
1042                 return;
1043         }
1044
1045         /* Setup the default interface info. */
1046         ifmr->ifm_status = IFM_AVALID;
1047         ifmr->ifm_active = IFM_ETHER;
1048
1049         if (lio->linfo.link.s.link_up) {
1050                 ifmr->ifm_status |= IFM_ACTIVE;
1051         } else {
1052                 ifmr->ifm_active |= IFM_NONE;
1053                 return;
1054         }
1055
1056         ifmr->ifm_active |= lio_get_media_subtype(lio->oct_dev);
1057
1058         if (lio->linfo.link.s.duplex)
1059                 ifmr->ifm_active |= IFM_FDX;
1060         else
1061                 ifmr->ifm_active |= IFM_HDX;
1062 }
1063
1064 static uint64_t
1065 lio_get_counter(if_t ifp, ift_counter cnt)
1066 {
1067         struct lio      *lio = if_getsoftc(ifp);
1068         struct octeon_device    *oct = lio->oct_dev;
1069         uint64_t        counter = 0;
1070         int             i, q_no;
1071
1072         switch (cnt) {
1073         case IFCOUNTER_IPACKETS:
1074                 for (i = 0; i < oct->num_oqs; i++) {
1075                         q_no = lio->linfo.rxpciq[i].s.q_no;
1076                         counter += oct->droq[q_no]->stats.rx_pkts_received;
1077                 }
1078                 break;
1079         case IFCOUNTER_OPACKETS:
1080                 for (i = 0; i < oct->num_iqs; i++) {
1081                         q_no = lio->linfo.txpciq[i].s.q_no;
1082                         counter += oct->instr_queue[q_no]->stats.tx_done;
1083                 }
1084                 break;
1085         case IFCOUNTER_IBYTES:
1086                 for (i = 0; i < oct->num_oqs; i++) {
1087                         q_no = lio->linfo.rxpciq[i].s.q_no;
1088                         counter += oct->droq[q_no]->stats.rx_bytes_received;
1089                 }
1090                 break;
1091         case IFCOUNTER_OBYTES:
1092                 for (i = 0; i < oct->num_iqs; i++) {
1093                         q_no = lio->linfo.txpciq[i].s.q_no;
1094                         counter += oct->instr_queue[q_no]->stats.tx_tot_bytes;
1095                 }
1096                 break;
1097         case IFCOUNTER_IQDROPS:
1098                 for (i = 0; i < oct->num_oqs; i++) {
1099                         q_no = lio->linfo.rxpciq[i].s.q_no;
1100                         counter += oct->droq[q_no]->stats.rx_dropped;
1101                 }
1102                 break;
1103         case IFCOUNTER_OQDROPS:
1104                 for (i = 0; i < oct->num_iqs; i++) {
1105                         q_no = lio->linfo.txpciq[i].s.q_no;
1106                         counter += oct->instr_queue[q_no]->stats.tx_dropped;
1107                 }
1108                 break;
1109         case IFCOUNTER_IMCASTS:
1110                 counter = oct->link_stats.fromwire.total_mcst;
1111                 break;
1112         case IFCOUNTER_OMCASTS:
1113                 counter = oct->link_stats.fromhost.mcast_pkts_sent;
1114                 break;
1115         case IFCOUNTER_COLLISIONS:
1116                 counter = oct->link_stats.fromhost.total_collisions;
1117                 break;
1118         case IFCOUNTER_IERRORS:
1119                 counter = oct->link_stats.fromwire.fcs_err +
1120                     oct->link_stats.fromwire.l2_err +
1121                     oct->link_stats.fromwire.frame_err;
1122                 break;
1123         default:
1124                 return (if_get_counter_default(ifp, cnt));
1125         }
1126
1127         return (counter);
1128 }
1129
1130 static int
1131 lio_init_ifnet(struct lio *lio)
1132 {
1133         struct octeon_device    *oct = lio->oct_dev;
1134         if_t ifp = lio->ifp;
1135
1136         /* ifconfig entrypoint for media type/status reporting */
1137         ifmedia_init(&lio->ifmedia, IFM_IMASK, lio_ifmedia_update,
1138                      lio_ifmedia_status);
1139
1140         /* set the default interface values */
1141         ifmedia_add(&lio->ifmedia,
1142                     (IFM_ETHER | IFM_FDX | lio_get_media_subtype(oct)),
1143                     0, NULL);
1144         ifmedia_add(&lio->ifmedia, (IFM_ETHER | IFM_AUTO), 0, NULL);
1145         ifmedia_set(&lio->ifmedia, (IFM_ETHER | IFM_AUTO));
1146
1147         lio->ifmedia.ifm_media = lio->ifmedia.ifm_cur->ifm_media;
1148         lio_dev_dbg(oct, "IFMEDIA flags : %x\n", lio->ifmedia.ifm_media);
1149
1150         if_initname(ifp, device_get_name(oct->device),
1151                     device_get_unit(oct->device));
1152         if_setflags(ifp, (IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST));
1153         if_setioctlfn(ifp, lio_ioctl);
1154         if_setgetcounterfn(ifp, lio_get_counter);
1155         if_settransmitfn(ifp, lio_mq_start);
1156         if_setqflushfn(ifp, lio_qflush);
1157         if_setinitfn(ifp, lio_open);
1158         if_setmtu(ifp, lio->linfo.link.s.mtu);
1159         lio->mtu = lio->linfo.link.s.mtu;
1160         if_sethwassist(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
1161                              CSUM_TCP_IPV6 | CSUM_UDP_IPV6));
1162
1163         if_setcapabilitiesbit(ifp, (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 |
1164                                     IFCAP_TSO | IFCAP_LRO |
1165                                     IFCAP_JUMBO_MTU | IFCAP_HWSTATS |
1166                                     IFCAP_LINKSTATE | IFCAP_VLAN_HWFILTER |
1167                                     IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTAGGING |
1168                                     IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU), 0);
1169
1170         if_setcapenable(ifp, if_getcapabilities(ifp));
1171         if_setbaudrate(ifp, lio_get_baudrate(oct));
1172
1173         return (0);
1174 }
1175
1176 static void
1177 lio_tcp_lro_free(struct octeon_device *octeon_dev, struct ifnet *ifp)
1178 {
1179         struct lio      *lio = if_getsoftc(ifp);
1180         struct lio_droq *droq;
1181         int             q_no;
1182         int             i;
1183
1184         for (i = 0; i < octeon_dev->num_oqs; i++) {
1185                 q_no = lio->linfo.rxpciq[i].s.q_no;
1186                 droq = octeon_dev->droq[q_no];
1187                 if (droq->lro.ifp) {
1188                         tcp_lro_free(&droq->lro);
1189                         droq->lro.ifp = NULL;
1190                 }
1191         }
1192 }
1193
1194 static int
1195 lio_tcp_lro_init(struct octeon_device *octeon_dev, struct ifnet *ifp)
1196 {
1197         struct lio      *lio = if_getsoftc(ifp);
1198         struct lio_droq *droq;
1199         struct lro_ctrl *lro;
1200         int             i, q_no, ret = 0;
1201
1202         for (i = 0; i < octeon_dev->num_oqs; i++) {
1203                 q_no = lio->linfo.rxpciq[i].s.q_no;
1204                 droq = octeon_dev->droq[q_no];
1205                 lro = &droq->lro;
1206                 ret = tcp_lro_init(lro);
1207                 if (ret) {
1208                         lio_dev_err(octeon_dev, "LRO Initialization failed ret %d\n",
1209                                     ret);
1210                         goto lro_init_failed;
1211                 }
1212
1213                 lro->ifp = ifp;
1214         }
1215
1216         return (ret);
1217
1218 lro_init_failed:
1219         lio_tcp_lro_free(octeon_dev, ifp);
1220
1221         return (ret);
1222 }
1223
1224 static int
1225 lio_setup_nic_devices(struct octeon_device *octeon_dev)
1226 {
1227         union           octeon_if_cfg if_cfg;
1228         struct lio      *lio = NULL;
1229         struct ifnet    *ifp = NULL;
1230         struct lio_version              *vdata;
1231         struct lio_soft_command         *sc;
1232         struct lio_if_cfg_context       *ctx;
1233         struct lio_if_cfg_resp          *resp;
1234         struct lio_if_props             *props;
1235         int             num_iqueues, num_oqueues, retval;
1236         unsigned int    base_queue;
1237         unsigned int    gmx_port_id;
1238         uint32_t        ctx_size, data_size;
1239         uint32_t        ifidx_or_pfnum, resp_size;
1240         uint8_t         mac[ETHER_HDR_LEN], i, j;
1241
1242         /* This is to handle link status changes */
1243         lio_register_dispatch_fn(octeon_dev, LIO_OPCODE_NIC,
1244                                  LIO_OPCODE_NIC_INFO,
1245                                  lio_link_info, octeon_dev);
1246
1247         for (i = 0; i < octeon_dev->ifcount; i++) {
1248                 resp_size = sizeof(struct lio_if_cfg_resp);
1249                 ctx_size = sizeof(struct lio_if_cfg_context);
1250                 data_size = sizeof(struct lio_version);
1251                 sc = lio_alloc_soft_command(octeon_dev, data_size, resp_size,
1252                                             ctx_size);
1253                 if (sc == NULL)
1254                         return (ENOMEM);
1255
1256                 resp = (struct lio_if_cfg_resp *)sc->virtrptr;
1257                 ctx = (struct lio_if_cfg_context *)sc->ctxptr;
1258                 vdata = (struct lio_version *)sc->virtdptr;
1259
1260                 *((uint64_t *)vdata) = 0;
1261                 vdata->major = htobe16(LIO_BASE_MAJOR_VERSION);
1262                 vdata->minor = htobe16(LIO_BASE_MINOR_VERSION);
1263                 vdata->micro = htobe16(LIO_BASE_MICRO_VERSION);
1264
1265                 num_iqueues = octeon_dev->sriov_info.num_pf_rings;
1266                 num_oqueues = octeon_dev->sriov_info.num_pf_rings;
1267                 base_queue = octeon_dev->sriov_info.pf_srn;
1268
1269                 gmx_port_id = octeon_dev->pf_num;
1270                 ifidx_or_pfnum = octeon_dev->pf_num;
1271
1272                 lio_dev_dbg(octeon_dev, "requesting config for interface %d, iqs %d, oqs %d\n",
1273                             ifidx_or_pfnum, num_iqueues, num_oqueues);
1274                 ctx->cond = 0;
1275                 ctx->octeon_id = lio_get_device_id(octeon_dev);
1276
1277                 if_cfg.if_cfg64 = 0;
1278                 if_cfg.s.num_iqueues = num_iqueues;
1279                 if_cfg.s.num_oqueues = num_oqueues;
1280                 if_cfg.s.base_queue = base_queue;
1281                 if_cfg.s.gmx_port_id = gmx_port_id;
1282
1283                 sc->iq_no = 0;
1284
1285                 lio_prepare_soft_command(octeon_dev, sc, LIO_OPCODE_NIC,
1286                                          LIO_OPCODE_NIC_IF_CFG, 0,
1287                                          if_cfg.if_cfg64, 0);
1288
1289                 sc->callback = lio_if_cfg_callback;
1290                 sc->callback_arg = sc;
1291                 sc->wait_time = 3000;
1292
1293                 retval = lio_send_soft_command(octeon_dev, sc);
1294                 if (retval == LIO_IQ_SEND_FAILED) {
1295                         lio_dev_err(octeon_dev, "iq/oq config failed status: %x\n",
1296                                     retval);
1297                         /* Soft instr is freed by driver in case of failure. */
1298                         goto setup_nic_dev_fail;
1299                 }
1300
1301                 /*
1302                  * Sleep on a wait queue till the cond flag indicates that the
1303                  * response arrived or timed-out.
1304                  */
1305                 lio_sleep_cond(octeon_dev, &ctx->cond);
1306
1307                 retval = resp->status;
1308                 if (retval) {
1309                         lio_dev_err(octeon_dev, "iq/oq config failed\n");
1310                         goto setup_nic_dev_fail;
1311                 }
1312
1313                 lio_swap_8B_data((uint64_t *)(&resp->cfg_info),
1314                                  (sizeof(struct octeon_if_cfg_info)) >> 3);
1315
1316                 num_iqueues = bitcount64(resp->cfg_info.iqmask);
1317                 num_oqueues = bitcount64(resp->cfg_info.oqmask);
1318
1319                 if (!(num_iqueues) || !(num_oqueues)) {
1320                         lio_dev_err(octeon_dev,
1321                                     "Got bad iqueues (%016llX) or oqueues (%016llX) from firmware.\n",
1322                                     LIO_CAST64(resp->cfg_info.iqmask),
1323                                     LIO_CAST64(resp->cfg_info.oqmask));
1324                         goto setup_nic_dev_fail;
1325                 }
1326
1327                 lio_dev_dbg(octeon_dev,
1328                             "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
1329                             i, LIO_CAST64(resp->cfg_info.iqmask),
1330                             LIO_CAST64(resp->cfg_info.oqmask),
1331                             num_iqueues, num_oqueues);
1332
1333                 ifp = if_alloc(IFT_ETHER);
1334
1335                 if (ifp == NULL) {
1336                         lio_dev_err(octeon_dev, "Device allocation failed\n");
1337                         goto setup_nic_dev_fail;
1338                 }
1339
1340                 lio = malloc(sizeof(struct lio), M_DEVBUF, M_NOWAIT | M_ZERO);
1341
1342                 if (lio == NULL) {
1343                         lio_dev_err(octeon_dev, "Lio allocation failed\n");
1344                         goto setup_nic_dev_fail;
1345                 }
1346
1347                 if_setsoftc(ifp, lio);
1348
1349                 ifp->if_hw_tsomax = LIO_MAX_FRAME_SIZE;
1350                 ifp->if_hw_tsomaxsegcount = LIO_MAX_SG;
1351                 ifp->if_hw_tsomaxsegsize = PAGE_SIZE;
1352
1353                 lio->ifidx = ifidx_or_pfnum;
1354
1355                 props = &octeon_dev->props;
1356                 props->gmxport = resp->cfg_info.linfo.gmxport;
1357                 props->ifp = ifp;
1358
1359                 lio->linfo.num_rxpciq = num_oqueues;
1360                 lio->linfo.num_txpciq = num_iqueues;
1361                 for (j = 0; j < num_oqueues; j++) {
1362                         lio->linfo.rxpciq[j].rxpciq64 =
1363                             resp->cfg_info.linfo.rxpciq[j].rxpciq64;
1364                 }
1365
1366                 for (j = 0; j < num_iqueues; j++) {
1367                         lio->linfo.txpciq[j].txpciq64 =
1368                             resp->cfg_info.linfo.txpciq[j].txpciq64;
1369                 }
1370
1371                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
1372                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
1373                 lio->linfo.link.link_status64 =
1374                     resp->cfg_info.linfo.link.link_status64;
1375
1376                 /*
1377                  * Point to the properties for octeon device to which this
1378                  * interface belongs.
1379                  */
1380                 lio->oct_dev = octeon_dev;
1381                 lio->ifp = ifp;
1382
1383                 lio_dev_dbg(octeon_dev, "if%d gmx: %d hw_addr: 0x%llx\n", i,
1384                             lio->linfo.gmxport, LIO_CAST64(lio->linfo.hw_addr));
1385                 lio_init_ifnet(lio);
1386                 /* 64-bit swap required on LE machines */
1387                 lio_swap_8B_data(&lio->linfo.hw_addr, 1);
1388                 for (j = 0; j < 6; j++)
1389                         mac[j] = *((uint8_t *)(
1390                                    ((uint8_t *)&lio->linfo.hw_addr) + 2 + j));
1391
1392                 ether_ifattach(ifp, mac);
1393
1394                 /*
1395                  * By default all interfaces on a single Octeon uses the same
1396                  * tx and rx queues
1397                  */
1398                 lio->txq = lio->linfo.txpciq[0].s.q_no;
1399                 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
1400                 if (lio_setup_io_queues(octeon_dev, i, lio->linfo.num_txpciq,
1401                                         lio->linfo.num_rxpciq)) {
1402                         lio_dev_err(octeon_dev, "I/O queues creation failed\n");
1403                         goto setup_nic_dev_fail;
1404                 }
1405
1406                 lio_ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
1407
1408                 lio->tx_qsize = lio_get_tx_qsize(octeon_dev, lio->txq);
1409                 lio->rx_qsize = lio_get_rx_qsize(octeon_dev, lio->rxq);
1410
1411                 if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
1412                         lio_dev_err(octeon_dev, "Gather list allocation failed\n");
1413                         goto setup_nic_dev_fail;
1414                 }
1415
1416                 if ((lio_hwlro == 0) && lio_tcp_lro_init(octeon_dev, ifp))
1417                         goto setup_nic_dev_fail;
1418
1419                 if (lio_hwlro &&
1420                     (if_getcapenable(ifp) & IFCAP_LRO) &&
1421                     (if_getcapenable(ifp) & IFCAP_RXCSUM) &&
1422                     (if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6))
1423                         lio_set_feature(ifp, LIO_CMD_LRO_ENABLE,
1424                                         LIO_LROIPV4 | LIO_LROIPV6);
1425
1426                 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER))
1427                         lio_set_feature(ifp, LIO_CMD_VLAN_FILTER_CTL, 1);
1428                 else
1429                         lio_set_feature(ifp, LIO_CMD_VLAN_FILTER_CTL, 0);
1430
1431                 if (lio_setup_rx_oom_poll_fn(ifp))
1432                         goto setup_nic_dev_fail;
1433
1434                 lio_dev_dbg(octeon_dev, "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
1435                             i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
1436                 lio->link_changes++;
1437
1438                 lio_ifstate_set(lio, LIO_IFSTATE_REGISTERED);
1439
1440                 /*
1441                  * Sending command to firmware to enable Rx checksum offload
1442                  * by default at the time of setup of Liquidio driver for
1443                  * this device
1444                  */
1445                 lio_set_rxcsum_command(ifp, LIO_CMD_TNL_RX_CSUM_CTL,
1446                                        LIO_CMD_RXCSUM_ENABLE);
1447                 lio_set_feature(ifp, LIO_CMD_TNL_TX_CSUM_CTL,
1448                                 LIO_CMD_TXCSUM_ENABLE);
1449
1450 #ifdef RSS
1451                 if (lio_rss) {
1452                         if (lio_send_rss_param(lio))
1453                                 goto setup_nic_dev_fail;
1454                 } else
1455 #endif  /* RSS */
1456
1457                         lio_set_feature(ifp, LIO_CMD_SET_FNV,
1458                                         LIO_CMD_FNV_ENABLE);
1459
1460                 lio_dev_dbg(octeon_dev, "NIC ifidx:%d Setup successful\n", i);
1461
1462                 lio_free_soft_command(octeon_dev, sc);
1463                 lio->vlan_attach =
1464                     EVENTHANDLER_REGISTER(vlan_config,
1465                                           lio_vlan_rx_add_vid, lio,
1466                                           EVENTHANDLER_PRI_FIRST);
1467                 lio->vlan_detach =
1468                     EVENTHANDLER_REGISTER(vlan_unconfig,
1469                                           lio_vlan_rx_kill_vid, lio,
1470                                           EVENTHANDLER_PRI_FIRST);
1471
1472                 /* Update stats periodically */
1473                 callout_init(&lio->stats_timer, 0);
1474                 lio->stats_interval = LIO_DEFAULT_STATS_INTERVAL;
1475
1476                 lio_add_hw_stats(lio);
1477         }
1478
1479         return (0);
1480
1481 setup_nic_dev_fail:
1482
1483         lio_free_soft_command(octeon_dev, sc);
1484
1485         while (i--) {
1486                 lio_dev_err(octeon_dev, "NIC ifidx:%d Setup failed\n", i);
1487                 lio_destroy_nic_device(octeon_dev, i);
1488         }
1489
1490         return (ENODEV);
1491 }
1492
1493 static int
1494 lio_link_info(struct lio_recv_info *recv_info, void *ptr)
1495 {
1496         struct octeon_device    *oct = (struct octeon_device *)ptr;
1497         struct lio_recv_pkt     *recv_pkt = recv_info->recv_pkt;
1498         union octeon_link_status *ls;
1499         int     gmxport = 0, i;
1500
1501         lio_dev_dbg(oct, "%s Called\n", __func__);
1502         if (recv_pkt->buffer_size[0] != (sizeof(*ls) + LIO_DROQ_INFO_SIZE)) {
1503                 lio_dev_err(oct, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
1504                             recv_pkt->buffer_size[0],
1505                             recv_pkt->rh.r_nic_info.gmxport);
1506                 goto nic_info_err;
1507         }
1508         gmxport = recv_pkt->rh.r_nic_info.gmxport;
1509         ls = (union octeon_link_status *)(recv_pkt->buffer_ptr[0]->m_data +
1510                                           LIO_DROQ_INFO_SIZE);
1511         lio_swap_8B_data((uint64_t *)ls,
1512                          (sizeof(union octeon_link_status)) >> 3);
1513
1514         if (oct->props.gmxport == gmxport)
1515                 lio_update_link_status(oct->props.ifp, ls);
1516
1517 nic_info_err:
1518         for (i = 0; i < recv_pkt->buffer_count; i++)
1519                 lio_recv_buffer_free(recv_pkt->buffer_ptr[i]);
1520
1521         lio_free_recv_info(recv_info);
1522         return (0);
1523 }
1524
1525 void
1526 lio_free_mbuf(struct lio_instr_queue *iq, struct lio_mbuf_free_info *finfo)
1527 {
1528
1529         bus_dmamap_sync(iq->txtag, finfo->map, BUS_DMASYNC_POSTWRITE);
1530         bus_dmamap_unload(iq->txtag, finfo->map);
1531         m_freem(finfo->mb);
1532 }
1533
1534 void
1535 lio_free_sgmbuf(struct lio_instr_queue *iq, struct lio_mbuf_free_info *finfo)
1536 {
1537         struct lio_gather       *g;
1538         struct octeon_device    *oct;
1539         struct lio              *lio;
1540         int     iq_no;
1541
1542         g = finfo->g;
1543         iq_no = iq->txpciq.s.q_no;
1544         oct = iq->oct_dev;
1545         lio = if_getsoftc(oct->props.ifp);
1546
1547         mtx_lock(&lio->glist_lock[iq_no]);
1548         STAILQ_INSERT_TAIL(&lio->ghead[iq_no], &g->node, entries);
1549         mtx_unlock(&lio->glist_lock[iq_no]);
1550
1551         bus_dmamap_sync(iq->txtag, finfo->map, BUS_DMASYNC_POSTWRITE);
1552         bus_dmamap_unload(iq->txtag, finfo->map);
1553         m_freem(finfo->mb);
1554 }
1555
1556 static void
1557 lio_if_cfg_callback(struct octeon_device *oct, uint32_t status, void *buf)
1558 {
1559         struct lio_soft_command *sc = (struct lio_soft_command *)buf;
1560         struct lio_if_cfg_resp  *resp;
1561         struct lio_if_cfg_context *ctx;
1562
1563         resp = (struct lio_if_cfg_resp *)sc->virtrptr;
1564         ctx = (struct lio_if_cfg_context *)sc->ctxptr;
1565
1566         oct = lio_get_device(ctx->octeon_id);
1567         if (resp->status)
1568                 lio_dev_err(oct, "nic if cfg instruction failed. Status: %llx (0x%08x)\n",
1569                             LIO_CAST64(resp->status), status);
1570         ctx->cond = 1;
1571
1572         snprintf(oct->fw_info.lio_firmware_version, 32, "%s",
1573                  resp->cfg_info.lio_firmware_version);
1574
1575         /*
1576          * This barrier is required to be sure that the response has been
1577          * written fully before waking up the handler
1578          */
1579         wmb();
1580 }
1581
1582 static int
1583 lio_is_mac_changed(uint8_t *new, uint8_t *old)
1584 {
1585
1586         return ((new[0] != old[0]) || (new[1] != old[1]) ||
1587                 (new[2] != old[2]) || (new[3] != old[3]) ||
1588                 (new[4] != old[4]) || (new[5] != old[5]));
1589 }
1590
1591 void
1592 lio_open(void *arg)
1593 {
1594         struct lio      *lio = arg;
1595         struct ifnet    *ifp = lio->ifp;
1596         struct octeon_device    *oct = lio->oct_dev;
1597         uint8_t *mac_new, mac_old[ETHER_HDR_LEN];
1598         int     ret = 0;
1599
1600         lio_ifstate_set(lio, LIO_IFSTATE_RUNNING);
1601
1602         /* Ready for link status updates */
1603         lio->intf_open = 1;
1604
1605         lio_dev_info(oct, "Interface Open, ready for traffic\n");
1606
1607         /* tell Octeon to start forwarding packets to host */
1608         lio_send_rx_ctrl_cmd(lio, 1);
1609
1610         mac_new = IF_LLADDR(ifp);
1611         memcpy(mac_old, ((uint8_t *)&lio->linfo.hw_addr) + 2, ETHER_HDR_LEN);
1612
1613         if (lio_is_mac_changed(mac_new, mac_old)) {
1614                 ret = lio_set_mac(ifp, mac_new);
1615                 if (ret)
1616                         lio_dev_err(oct, "MAC change failed, error: %d\n", ret);
1617         }
1618
1619         /* Now inform the stack we're ready */
1620         if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
1621
1622         lio_dev_info(oct, "Interface is opened\n");
1623 }
1624
1625 static int
1626 lio_set_rxcsum_command(struct ifnet *ifp, int command, uint8_t rx_cmd)
1627 {
1628         struct lio_ctrl_pkt     nctrl;
1629         struct lio              *lio = if_getsoftc(ifp);
1630         struct octeon_device    *oct = lio->oct_dev;
1631         int     ret = 0;
1632
1633         nctrl.ncmd.cmd64 = 0;
1634         nctrl.ncmd.s.cmd = command;
1635         nctrl.ncmd.s.param1 = rx_cmd;
1636         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1637         nctrl.wait_time = 100;
1638         nctrl.lio = lio;
1639         nctrl.cb_fn = lio_ctrl_cmd_completion;
1640
1641         ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl);
1642         if (ret < 0) {
1643                 lio_dev_err(oct, "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
1644                             ret);
1645         }
1646
1647         return (ret);
1648 }
1649
1650 static int
1651 lio_stop_nic_module(struct octeon_device *oct)
1652 {
1653         int             i, j;
1654         struct lio      *lio;
1655
1656         lio_dev_dbg(oct, "Stopping network interfaces\n");
1657         if (!oct->ifcount) {
1658                 lio_dev_err(oct, "Init for Octeon was not completed\n");
1659                 return (1);
1660         }
1661
1662         mtx_lock(&oct->cmd_resp_wqlock);
1663         oct->cmd_resp_state = LIO_DRV_OFFLINE;
1664         mtx_unlock(&oct->cmd_resp_wqlock);
1665
1666         for (i = 0; i < oct->ifcount; i++) {
1667                 lio = if_getsoftc(oct->props.ifp);
1668                 for (j = 0; j < oct->num_oqs; j++)
1669                         lio_unregister_droq_ops(oct,
1670                                                 lio->linfo.rxpciq[j].s.q_no);
1671         }
1672
1673         callout_drain(&lio->stats_timer);
1674
1675         for (i = 0; i < oct->ifcount; i++)
1676                 lio_destroy_nic_device(oct, i);
1677
1678         lio_dev_dbg(oct, "Network interface stopped\n");
1679
1680         return (0);
1681 }
1682
1683 static void
1684 lio_delete_glists(struct octeon_device *oct, struct lio *lio)
1685 {
1686         struct lio_gather       *g;
1687         int     i;
1688
1689         if (lio->glist_lock != NULL) {
1690                 free((void *)lio->glist_lock, M_DEVBUF);
1691                 lio->glist_lock = NULL;
1692         }
1693
1694         if (lio->ghead == NULL)
1695                 return;
1696
1697         for (i = 0; i < lio->linfo.num_txpciq; i++) {
1698                 do {
1699                         g = (struct lio_gather *)
1700                             lio_delete_first_node(&lio->ghead[i]);
1701                         free(g, M_DEVBUF);
1702                 } while (g);
1703
1704                 if ((lio->glists_virt_base != NULL) &&
1705                     (lio->glists_virt_base[i] != NULL)) {
1706                         lio_dma_free(lio->glist_entry_size * lio->tx_qsize,
1707                                      lio->glists_virt_base[i]);
1708                 }
1709         }
1710
1711         free(lio->glists_virt_base, M_DEVBUF);
1712         lio->glists_virt_base = NULL;
1713
1714         free(lio->glists_dma_base, M_DEVBUF);
1715         lio->glists_dma_base = NULL;
1716
1717         free(lio->ghead, M_DEVBUF);
1718         lio->ghead = NULL;
1719 }
1720
1721 static int
1722 lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
1723 {
1724         struct lio_gather       *g;
1725         int     i, j;
1726
1727         lio->glist_lock = malloc(num_iqs * sizeof(*lio->glist_lock), M_DEVBUF,
1728                                  M_NOWAIT | M_ZERO);
1729         if (lio->glist_lock == NULL)
1730                 return (1);
1731
1732         lio->ghead = malloc(num_iqs * sizeof(*lio->ghead), M_DEVBUF,
1733                             M_NOWAIT | M_ZERO);
1734         if (lio->ghead == NULL) {
1735                 free((void *)lio->glist_lock, M_DEVBUF);
1736                 lio->glist_lock = NULL;
1737                 return (1);
1738         }
1739
1740         lio->glist_entry_size = ROUNDUP8((ROUNDUP4(LIO_MAX_SG) >> 2) *
1741                                          LIO_SG_ENTRY_SIZE);
1742         /*
1743          * allocate memory to store virtual and dma base address of
1744          * per glist consistent memory
1745          */
1746         lio->glists_virt_base = malloc(num_iqs * sizeof(void *), M_DEVBUF,
1747                                        M_NOWAIT | M_ZERO);
1748         lio->glists_dma_base = malloc(num_iqs * sizeof(vm_paddr_t), M_DEVBUF,
1749                                       M_NOWAIT | M_ZERO);
1750         if ((lio->glists_virt_base == NULL) || (lio->glists_dma_base == NULL)) {
1751                 lio_delete_glists(oct, lio);
1752                 return (1);
1753         }
1754
1755         for (i = 0; i < num_iqs; i++) {
1756                 mtx_init(&lio->glist_lock[i], "glist_lock", NULL, MTX_DEF);
1757
1758                 STAILQ_INIT(&lio->ghead[i]);
1759
1760                 lio->glists_virt_base[i] =
1761                     lio_dma_alloc(lio->glist_entry_size * lio->tx_qsize,
1762                                   (vm_paddr_t *)&lio->glists_dma_base[i]);
1763                 if (lio->glists_virt_base[i] == NULL) {
1764                         lio_delete_glists(oct, lio);
1765                         return (1);
1766                 }
1767
1768                 for (j = 0; j < lio->tx_qsize; j++) {
1769                         g = malloc(sizeof(*g), M_DEVBUF, M_NOWAIT | M_ZERO);
1770                         if (g == NULL)
1771                                 break;
1772
1773                         g->sg = (struct lio_sg_entry *)(uintptr_t)
1774                             ((uint64_t)(uintptr_t)lio->glists_virt_base[i] +
1775                              (j * lio->glist_entry_size));
1776                         g->sg_dma_ptr = (uint64_t)lio->glists_dma_base[i] +
1777                                 (j * lio->glist_entry_size);
1778                         STAILQ_INSERT_TAIL(&lio->ghead[i], &g->node, entries);
1779                 }
1780
1781                 if (j != lio->tx_qsize) {
1782                         lio_delete_glists(oct, lio);
1783                         return (1);
1784                 }
1785         }
1786
1787         return (0);
1788 }
1789
1790 void
1791 lio_stop(struct ifnet *ifp)
1792 {
1793         struct lio      *lio = if_getsoftc(ifp);
1794         struct octeon_device    *oct = lio->oct_dev;
1795
1796         lio_ifstate_reset(lio, LIO_IFSTATE_RUNNING);
1797         if_link_state_change(ifp, LINK_STATE_DOWN);
1798
1799         lio->intf_open = 0;
1800         lio->linfo.link.s.link_up = 0;
1801         lio->link_changes++;
1802
1803         lio_send_rx_ctrl_cmd(lio, 0);
1804
1805         /* Tell the stack that the interface is no longer active */
1806         if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1807
1808         lio_dev_info(oct, "Interface is stopped\n");
1809 }
1810
1811 static void
1812 lio_check_rx_oom_status(struct lio *lio)
1813 {
1814         struct lio_droq *droq;
1815         struct octeon_device *oct = lio->oct_dev;
1816         int     desc_refilled;
1817         int     q, q_no = 0;
1818
1819         for (q = 0; q < oct->num_oqs; q++) {
1820                 q_no = lio->linfo.rxpciq[q].s.q_no;
1821                 droq = oct->droq[q_no];
1822                 if (droq == NULL)
1823                         continue;
1824                 if (lio_read_csr32(oct, droq->pkts_credit_reg) <= 0x40) {
1825                         mtx_lock(&droq->lock);
1826                         desc_refilled = lio_droq_refill(oct, droq);
1827                         /*
1828                          * Flush the droq descriptor data to memory to be sure
1829                          * that when we update the credits the data in memory
1830                          * is accurate.
1831                          */
1832                         wmb();
1833                         lio_write_csr32(oct, droq->pkts_credit_reg,
1834                                         desc_refilled);
1835                         /* make sure mmio write completes */
1836                         __compiler_membar();
1837                         mtx_unlock(&droq->lock);
1838                 }
1839         }
1840 }
1841
1842 static void
1843 lio_poll_check_rx_oom_status(void *arg, int pending __unused)
1844 {
1845         struct lio_tq   *rx_status_tq = arg;
1846         struct lio      *lio = rx_status_tq->ctxptr;
1847
1848         if (lio_ifstate_check(lio, LIO_IFSTATE_RUNNING))
1849                 lio_check_rx_oom_status(lio);
1850
1851         taskqueue_enqueue_timeout(rx_status_tq->tq, &rx_status_tq->work,
1852                                   lio_ms_to_ticks(50));
1853 }
1854
1855 static int
1856 lio_setup_rx_oom_poll_fn(struct ifnet *ifp)
1857 {
1858         struct lio      *lio = if_getsoftc(ifp);
1859         struct octeon_device    *oct = lio->oct_dev;
1860         struct lio_tq   *rx_status_tq;
1861
1862         rx_status_tq = &lio->rx_status_tq;
1863
1864         rx_status_tq->tq = taskqueue_create("lio_rx_oom_status", M_WAITOK,
1865                                             taskqueue_thread_enqueue,
1866                                             &rx_status_tq->tq);
1867         if (rx_status_tq->tq == NULL) {
1868                 lio_dev_err(oct, "unable to create lio rx oom status tq\n");
1869                 return (-1);
1870         }
1871
1872         TIMEOUT_TASK_INIT(rx_status_tq->tq, &rx_status_tq->work, 0,
1873                           lio_poll_check_rx_oom_status, (void *)rx_status_tq);
1874
1875         rx_status_tq->ctxptr = lio;
1876
1877         taskqueue_start_threads(&rx_status_tq->tq, 1, PI_NET,
1878                                 "lio%d_rx_oom_status",
1879                                 oct->octeon_id);
1880
1881         taskqueue_enqueue_timeout(rx_status_tq->tq, &rx_status_tq->work,
1882                                   lio_ms_to_ticks(50));
1883
1884         return (0);
1885 }
1886
1887 static void
1888 lio_cleanup_rx_oom_poll_fn(struct ifnet *ifp)
1889 {
1890         struct lio      *lio = if_getsoftc(ifp);
1891
1892         if (lio->rx_status_tq.tq != NULL) {
1893                 while (taskqueue_cancel_timeout(lio->rx_status_tq.tq,
1894                                                 &lio->rx_status_tq.work, NULL))
1895                         taskqueue_drain_timeout(lio->rx_status_tq.tq,
1896                                                 &lio->rx_status_tq.work);
1897
1898                 taskqueue_free(lio->rx_status_tq.tq);
1899
1900                 lio->rx_status_tq.tq = NULL;
1901         }
1902 }
1903
1904 static void
1905 lio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1906 {
1907         struct ifnet    *ifp = oct->props.ifp;
1908         struct lio      *lio;
1909
1910         if (ifp == NULL) {
1911                 lio_dev_err(oct, "%s No ifp ptr for index %d\n",
1912                             __func__, ifidx);
1913                 return;
1914         }
1915
1916         lio = if_getsoftc(ifp);
1917
1918         lio_ifstate_set(lio, LIO_IFSTATE_DETACH);
1919
1920         lio_dev_dbg(oct, "NIC device cleanup\n");
1921
1922         if (atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1923                 lio_stop(ifp);
1924
1925         if (lio_wait_for_pending_requests(oct))
1926                 lio_dev_err(oct, "There were pending requests\n");
1927
1928         if (lio_wait_for_instr_fetch(oct))
1929                 lio_dev_err(oct, "IQ had pending instructions\n");
1930
1931         if (lio_wait_for_oq_pkts(oct))
1932                 lio_dev_err(oct, "OQ had pending packets\n");
1933
1934         if (atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1935                 ether_ifdetach(ifp);
1936
1937         lio_tcp_lro_free(oct, ifp);
1938
1939         lio_cleanup_rx_oom_poll_fn(ifp);
1940
1941         lio_delete_glists(oct, lio);
1942
1943         EVENTHANDLER_DEREGISTER(vlan_config, lio->vlan_attach);
1944         EVENTHANDLER_DEREGISTER(vlan_unconfig, lio->vlan_detach);
1945
1946         free(lio, M_DEVBUF);
1947
1948         if_free(ifp);
1949
1950         oct->props.gmxport = -1;
1951
1952         oct->props.ifp = NULL;
1953 }
1954
1955 static void
1956 print_link_info(struct ifnet *ifp)
1957 {
1958         struct lio      *lio = if_getsoftc(ifp);
1959
1960         if (!lio_ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
1961             lio_ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
1962                 struct octeon_link_info *linfo = &lio->linfo;
1963
1964                 if (linfo->link.s.link_up) {
1965                         lio_dev_info(lio->oct_dev, "%d Mbps %s Duplex UP\n",
1966                                      linfo->link.s.speed,
1967                                      (linfo->link.s.duplex) ? "Full" : "Half");
1968                 } else {
1969                         lio_dev_info(lio->oct_dev, "Link Down\n");
1970                 }
1971         }
1972 }
1973
1974 static inline void
1975 lio_update_link_status(struct ifnet *ifp, union octeon_link_status *ls)
1976 {
1977         struct lio      *lio = if_getsoftc(ifp);
1978         int     changed = (lio->linfo.link.link_status64 != ls->link_status64);
1979
1980         lio->linfo.link.link_status64 = ls->link_status64;
1981
1982         if ((lio->intf_open) && (changed)) {
1983                 print_link_info(ifp);
1984                 lio->link_changes++;
1985                 if (lio->linfo.link.s.link_up)
1986                         if_link_state_change(ifp, LINK_STATE_UP);
1987                 else
1988                         if_link_state_change(ifp, LINK_STATE_DOWN);
1989         }
1990 }
1991
1992 /*
1993  * \brief Callback for rx ctrl
1994  * @param status status of request
1995  * @param buf pointer to resp structure
1996  */
1997 static void
1998 lio_rx_ctl_callback(struct octeon_device *oct, uint32_t status, void *buf)
1999 {
2000         struct lio_soft_command *sc = (struct lio_soft_command *)buf;
2001         struct lio_rx_ctl_context *ctx;
2002
2003         ctx = (struct lio_rx_ctl_context *)sc->ctxptr;
2004
2005         oct = lio_get_device(ctx->octeon_id);
2006         if (status)
2007                 lio_dev_err(oct, "rx ctl instruction failed. Status: %llx\n",
2008                             LIO_CAST64(status));
2009         ctx->cond = 1;
2010
2011         /*
2012          * This barrier is required to be sure that the response has been
2013          * written fully before waking up the handler
2014          */
2015         wmb();
2016 }
2017
2018 static void
2019 lio_send_rx_ctrl_cmd(struct lio *lio, int start_stop)
2020 {
2021         struct lio_soft_command *sc;
2022         struct lio_rx_ctl_context *ctx;
2023         union octeon_cmd        *ncmd;
2024         struct octeon_device    *oct = (struct octeon_device *)lio->oct_dev;
2025         int     ctx_size = sizeof(struct lio_rx_ctl_context);
2026         int     retval;
2027
2028         if (oct->props.rx_on == start_stop)
2029                 return;
2030
2031         sc = lio_alloc_soft_command(oct, OCTEON_CMD_SIZE, 16, ctx_size);
2032         if (sc == NULL)
2033                 return;
2034
2035         ncmd = (union octeon_cmd *)sc->virtdptr;
2036         ctx = (struct lio_rx_ctl_context *)sc->ctxptr;
2037
2038         ctx->cond = 0;
2039         ctx->octeon_id = lio_get_device_id(oct);
2040         ncmd->cmd64 = 0;
2041         ncmd->s.cmd = LIO_CMD_RX_CTL;
2042         ncmd->s.param1 = start_stop;
2043
2044         lio_swap_8B_data((uint64_t *)ncmd, (OCTEON_CMD_SIZE >> 3));
2045
2046         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
2047
2048         lio_prepare_soft_command(oct, sc, LIO_OPCODE_NIC, LIO_OPCODE_NIC_CMD, 0,
2049                                  0, 0);
2050
2051         sc->callback = lio_rx_ctl_callback;
2052         sc->callback_arg = sc;
2053         sc->wait_time = 5000;
2054
2055         retval = lio_send_soft_command(oct, sc);
2056         if (retval == LIO_IQ_SEND_FAILED) {
2057                 lio_dev_err(oct, "Failed to send RX Control message\n");
2058         } else {
2059                 /*
2060                  * Sleep on a wait queue till the cond flag indicates that the
2061                  * response arrived or timed-out.
2062                  */
2063                 lio_sleep_cond(oct, &ctx->cond);
2064                 oct->props.rx_on = start_stop;
2065         }
2066
2067         lio_free_soft_command(oct, sc);
2068 }
2069
2070 static void
2071 lio_vlan_rx_add_vid(void *arg, struct ifnet *ifp, uint16_t vid)
2072 {
2073         struct lio_ctrl_pkt     nctrl;
2074         struct lio              *lio = if_getsoftc(ifp);
2075         struct octeon_device    *oct = lio->oct_dev;
2076         int     ret = 0;
2077
2078         if (if_getsoftc(ifp) != arg)    /* Not our event */
2079                 return;
2080
2081         if ((vid == 0) || (vid > 4095)) /* Invalid */
2082                 return;
2083
2084         bzero(&nctrl, sizeof(struct lio_ctrl_pkt));
2085
2086         nctrl.ncmd.cmd64 = 0;
2087         nctrl.ncmd.s.cmd = LIO_CMD_ADD_VLAN_FILTER;
2088         nctrl.ncmd.s.param1 = vid;
2089         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2090         nctrl.wait_time = 100;
2091         nctrl.lio = lio;
2092         nctrl.cb_fn = lio_ctrl_cmd_completion;
2093
2094         ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl);
2095         if (ret < 0) {
2096                 lio_dev_err(oct, "Add VLAN filter failed in core (ret: 0x%x)\n",
2097                             ret);
2098         }
2099 }
2100
2101 static void
2102 lio_vlan_rx_kill_vid(void *arg, struct ifnet *ifp, uint16_t vid)
2103 {
2104         struct lio_ctrl_pkt     nctrl;
2105         struct lio              *lio = if_getsoftc(ifp);
2106         struct octeon_device    *oct = lio->oct_dev;
2107         int     ret = 0;
2108
2109         if (if_getsoftc(ifp) != arg)    /* Not our event */
2110                 return;
2111
2112         if ((vid == 0) || (vid > 4095)) /* Invalid */
2113                 return;
2114
2115         bzero(&nctrl, sizeof(struct lio_ctrl_pkt));
2116
2117         nctrl.ncmd.cmd64 = 0;
2118         nctrl.ncmd.s.cmd = LIO_CMD_DEL_VLAN_FILTER;
2119         nctrl.ncmd.s.param1 = vid;
2120         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2121         nctrl.wait_time = 100;
2122         nctrl.lio = lio;
2123         nctrl.cb_fn = lio_ctrl_cmd_completion;
2124
2125         ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl);
2126         if (ret < 0) {
2127                 lio_dev_err(oct,
2128                             "Kill VLAN filter failed in core (ret: 0x%x)\n",
2129                             ret);
2130         }
2131 }
2132
2133 static int
2134 lio_wait_for_oq_pkts(struct octeon_device *oct)
2135 {
2136         int     i, pending_pkts, pkt_cnt = 0, retry = 100;
2137
2138         do {
2139                 pending_pkts = 0;
2140
2141                 for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {
2142                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
2143                                 continue;
2144
2145                         pkt_cnt = lio_droq_check_hw_for_pkts(oct->droq[i]);
2146                         if (pkt_cnt > 0) {
2147                                 pending_pkts += pkt_cnt;
2148                                 taskqueue_enqueue(oct->droq[i]->droq_taskqueue,
2149                                                   &oct->droq[i]->droq_task);
2150                         }
2151                 }
2152
2153                 pkt_cnt = 0;
2154                 lio_sleep_timeout(1);
2155         } while (retry-- && pending_pkts);
2156
2157         return (pkt_cnt);
2158 }
2159
2160 static void
2161 lio_destroy_resources(struct octeon_device *oct)
2162 {
2163         int i, refcount;
2164
2165         switch (atomic_load_acq_int(&oct->status)) {
2166         case LIO_DEV_RUNNING:
2167         case LIO_DEV_CORE_OK:
2168                 /* No more instructions will be forwarded. */
2169                 atomic_store_rel_int(&oct->status, LIO_DEV_IN_RESET);
2170
2171                 oct->app_mode = LIO_DRV_INVALID_APP;
2172                 lio_dev_dbg(oct, "Device state is now %s\n",
2173                             lio_get_state_string(&oct->status));
2174
2175                 lio_sleep_timeout(100);
2176
2177                 /* fallthrough */
2178         case LIO_DEV_HOST_OK:
2179
2180                 /* fallthrough */
2181         case LIO_DEV_CONSOLE_INIT_DONE:
2182                 /* Remove any consoles */
2183                 lio_remove_consoles(oct);
2184
2185                 /* fallthrough */
2186         case LIO_DEV_IO_QUEUES_DONE:
2187                 if (lio_wait_for_pending_requests(oct))
2188                         lio_dev_err(oct, "There were pending requests\n");
2189
2190                 if (lio_wait_for_instr_fetch(oct))
2191                         lio_dev_err(oct, "IQ had pending instructions\n");
2192
2193                 /*
2194                  * Disable the input and output queues now. No more packets will
2195                  * arrive from Octeon, but we should wait for all packet
2196                  * processing to finish.
2197                  */
2198                 oct->fn_list.disable_io_queues(oct);
2199
2200                 if (lio_wait_for_oq_pkts(oct))
2201                         lio_dev_err(oct, "OQ had pending packets\n");
2202
2203                 /* fallthrough */
2204         case LIO_DEV_INTR_SET_DONE:
2205                 /* Disable interrupts  */
2206                 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
2207
2208                 if (oct->msix_on) {
2209                         for (i = 0; i < oct->num_msix_irqs - 1; i++) {
2210                                 if (oct->ioq_vector[i].tag != NULL) {
2211                                         bus_teardown_intr(oct->device,
2212                                                   oct->ioq_vector[i].msix_res,
2213                                                       oct->ioq_vector[i].tag);
2214                                         oct->ioq_vector[i].tag = NULL;
2215                                 }
2216                                 if (oct->ioq_vector[i].msix_res != NULL) {
2217                                         bus_release_resource(oct->device,
2218                                                 SYS_RES_IRQ,
2219                                                 oct->ioq_vector[i].vector,
2220                                                 oct->ioq_vector[i].msix_res);
2221                                         oct->ioq_vector[i].msix_res = NULL;
2222                                 }
2223                         }
2224                         /* non-iov vector's argument is oct struct */
2225                         if (oct->tag != NULL) {
2226                                 bus_teardown_intr(oct->device, oct->msix_res,
2227                                                   oct->tag);
2228                                 oct->tag = NULL;
2229                         }
2230
2231                         if (oct->msix_res != NULL) {
2232                                 bus_release_resource(oct->device, SYS_RES_IRQ,
2233                                                      oct->aux_vector,
2234                                                      oct->msix_res);
2235                                 oct->msix_res = NULL;
2236                         }
2237
2238                         pci_release_msi(oct->device);
2239                 }
2240                 /* fallthrough */
2241         case LIO_DEV_IN_RESET:
2242         case LIO_DEV_DROQ_INIT_DONE:
2243                 /* Wait for any pending operations */
2244                 lio_mdelay(100);
2245                 for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {
2246                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
2247                                 continue;
2248                         lio_delete_droq(oct, i);
2249                 }
2250
2251                 /* fallthrough */
2252         case LIO_DEV_RESP_LIST_INIT_DONE:
2253                 for (i = 0; i < LIO_MAX_POSSIBLE_OUTPUT_QUEUES; i++) {
2254                         if (oct->droq[i] != NULL) {
2255                                 free(oct->droq[i], M_DEVBUF);
2256                                 oct->droq[i] = NULL;
2257                         }
2258                 }
2259                 lio_delete_response_list(oct);
2260
2261                 /* fallthrough */
2262         case LIO_DEV_INSTR_QUEUE_INIT_DONE:
2263                 for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {
2264                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
2265                                 continue;
2266
2267                         lio_delete_instr_queue(oct, i);
2268                 }
2269
2270                 /* fallthrough */
2271         case LIO_DEV_MSIX_ALLOC_VECTOR_DONE:
2272                 for (i = 0; i < LIO_MAX_POSSIBLE_INSTR_QUEUES; i++) {
2273                         if (oct->instr_queue[i] != NULL) {
2274                                 free(oct->instr_queue[i], M_DEVBUF);
2275                                 oct->instr_queue[i] = NULL;
2276                         }
2277                 }
2278                 lio_free_ioq_vector(oct);
2279
2280                 /* fallthrough */
2281         case LIO_DEV_SC_BUFF_POOL_INIT_DONE:
2282                 lio_free_sc_buffer_pool(oct);
2283
2284                 /* fallthrough */
2285         case LIO_DEV_DISPATCH_INIT_DONE:
2286                 lio_delete_dispatch_list(oct);
2287
2288                 /* fallthrough */
2289         case LIO_DEV_PCI_MAP_DONE:
2290                 refcount = lio_deregister_device(oct);
2291
2292                 if (fw_type_is_none())
2293                         lio_pci_flr(oct);
2294
2295                 if (!refcount)
2296                         oct->fn_list.soft_reset(oct);
2297
2298                 lio_unmap_pci_barx(oct, 0);
2299                 lio_unmap_pci_barx(oct, 1);
2300
2301                 /* fallthrough */
2302         case LIO_DEV_PCI_ENABLE_DONE:
2303                 /* Disable the device, releasing the PCI INT */
2304                 pci_disable_busmaster(oct->device);
2305
2306                 /* fallthrough */
2307         case LIO_DEV_BEGIN_STATE:
2308                 break;
2309         }       /* end switch (oct->status) */
2310 }