sys/dev/liquidio/base/lio_device.c

   1 /*
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Cavium, Inc. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 /*$FreeBSD$*/
  34
  35 #include "lio_bsd.h"
  36 #include "lio_common.h"
  37 #include "lio_droq.h"
  38 #include "lio_iq.h"
  39 #include "lio_response_manager.h"
  40 #include "lio_device.h"
  41 #include "lio_main.h"
  42 #include "lio_network.h"
  43 #include "cn23xx_pf_device.h"
  44 #include "lio_image.h"
  45 #include "lio_mem_ops.h"
  46
  47 static struct lio_config default_cn23xx_conf = {
  48         .card_type                      = LIO_23XX,
  49         .card_name                      = LIO_23XX_NAME,
  50         /* IQ attributes */
  51         .iq = {
  52                 .max_iqs                = LIO_CN23XX_CFG_IO_QUEUES,
  53                 .pending_list_size      = (LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS *
  54                                            LIO_CN23XX_CFG_IO_QUEUES),
  55                 .instr_type             = LIO_64BYTE_INSTR,
  56                 .db_min                 = LIO_CN23XX_DB_MIN,
  57                 .db_timeout             = LIO_CN23XX_DB_TIMEOUT,
  58                 .iq_intr_pkt            = LIO_CN23XX_DEF_IQ_INTR_THRESHOLD,
  59         },
  60
  61         /* OQ attributes */
  62         .oq = {
  63                 .max_oqs                = LIO_CN23XX_CFG_IO_QUEUES,
  64                 .pkts_per_intr          = LIO_CN23XX_OQ_PKTS_PER_INTR,
  65                 .refill_threshold       = LIO_CN23XX_OQ_REFIL_THRESHOLD,
  66                 .oq_intr_pkt            = LIO_CN23XX_OQ_INTR_PKT,
  67                 .oq_intr_time           = LIO_CN23XX_OQ_INTR_TIME,
  68         },
  69
  70         .num_nic_ports                  = LIO_CN23XX_DEFAULT_NUM_PORTS,
  71         .num_def_rx_descs               = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS,
  72         .num_def_tx_descs               = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS,
  73         .def_rx_buf_size                = LIO_CN23XX_OQ_BUF_SIZE,
  74
  75         /* For ethernet interface 0:  Port cfg Attributes */
  76         .nic_if_cfg[0] = {
  77                 /* Max Txqs: Half for each of the two ports :max_iq/2 */
  78                 .max_txqs               = LIO_MAX_TXQS_PER_INTF,
  79
  80                 /* Actual configured value. Range could be: 1...max_txqs */
  81                 .num_txqs               = LIO_DEF_TXQS_PER_INTF,
  82
  83                 /* Max Rxqs: Half for each of the two ports :max_oq/2  */
  84                 .max_rxqs               = LIO_MAX_RXQS_PER_INTF,
  85
  86                 /* Actual configured value. Range could be: 1...max_rxqs */
  87                 .num_rxqs               = LIO_DEF_RXQS_PER_INTF,
  88
  89                 /* Num of desc for rx rings */
  90                 .num_rx_descs           = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS,
  91
  92                 /* Num of desc for tx rings */
  93                 .num_tx_descs           = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS,
  94
  95                 /*
  96                  * Mbuf size, We need not change buf size even for Jumbo frames.
  97                  * Octeon can send jumbo frames in 4 consecutive descriptors,
  98                  */
  99                 .rx_buf_size                    = LIO_CN23XX_OQ_BUF_SIZE,
 100
 101                 .base_queue                     = LIO_BASE_QUEUE_NOT_REQUESTED,
 102
 103                 .gmx_port_id                    = 0,
 104         },
 105
 106         .nic_if_cfg[1] = {
 107                 /* Max Txqs: Half for each of the two ports :max_iq/2 */
 108                 .max_txqs               = LIO_MAX_TXQS_PER_INTF,
 109
 110                 /* Actual configured value. Range could be: 1...max_txqs */
 111                 .num_txqs               = LIO_DEF_TXQS_PER_INTF,
 112
 113                 /* Max Rxqs: Half for each of the two ports :max_oq/2  */
 114                 .max_rxqs               = LIO_MAX_RXQS_PER_INTF,
 115
 116                 /* Actual configured value. Range could be: 1...max_rxqs */
 117                 .num_rxqs               = LIO_DEF_RXQS_PER_INTF,
 118
 119                 /* Num of desc for rx rings */
 120                 .num_rx_descs           = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS,
 121
 122                 /* Num of desc for tx rings */
 123                 .num_tx_descs           = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS,
 124
 125                 /*
 126                  * Mbuf size, We need not change buf size even for Jumbo frames.
 127                  * Octeon can send jumbo frames in 4 consecutive descriptors,
 128                  */
 129                 .rx_buf_size                    = LIO_CN23XX_OQ_BUF_SIZE,
 130
 131                 .base_queue                     = LIO_BASE_QUEUE_NOT_REQUESTED,
 132
 133                 .gmx_port_id                    = 1,
 134         },
 135
 136         .misc                                   = {
 137                 /* Host driver link query interval */
 138                 .oct_link_query_interval        = 100,
 139
 140                 /* Octeon link query interval */
 141                 .host_link_query_interval       = 500,
 142
 143                 .enable_sli_oq_bp               = 0,
 144
 145                 /* Control queue group */
 146                 .ctrlq_grp                      = 1,
 147         }
 148 };
 149
 150 static struct lio_config_ptr {
 151         uint32_t        conf_type;
 152 }       oct_conf_info[LIO_MAX_DEVICES] = {
 153
 154         {
 155                 LIO_CFG_TYPE_DEFAULT,
 156         }, {
 157                 LIO_CFG_TYPE_DEFAULT,
 158         }, {
 159                 LIO_CFG_TYPE_DEFAULT,
 160         }, {
 161                 LIO_CFG_TYPE_DEFAULT,
 162         },
 163 };
 164
 165 static char lio_state_str[LIO_DEV_STATES + 1][32] = {
 166         "BEGIN", "PCI-ENABLE-DONE", "PCI-MAP-DONE", "DISPATCH-INIT-DONE",
 167         "IQ-INIT-DONE", "SCBUFF-POOL-INIT-DONE", "RESPLIST-INIT-DONE",
 168         "DROQ-INIT-DONE", "MBOX-SETUP-DONE", "MSIX-ALLOC-VECTOR-DONE",
 169         "INTR-SET-DONE", "IO-QUEUES-INIT-DONE", "CONSOLE-INIT-DONE",
 170         "HOST-READY", "CORE-READY", "RUNNING", "IN-RESET",
 171         "INVALID"
 172 };
 173
 174 static char     lio_app_str[LIO_DRV_APP_COUNT + 1][32] = {"BASE", "NIC", "UNKNOWN"};
 175
 176 static struct octeon_device     *octeon_device[LIO_MAX_DEVICES];
 177 static volatile int             lio_adapter_refcounts[LIO_MAX_DEVICES];
 178
 179 static uint32_t octeon_device_count;
 180 /* locks device array (i.e. octeon_device[]) */
 181 struct mtx      octeon_devices_lock;
 182
 183 static struct lio_core_setup    core_setup[LIO_MAX_DEVICES];
 184
 185 static void
 186 oct_set_config_info(int oct_id, int conf_type)
 187 {
 188
 189         if (conf_type < 0 || conf_type > (LIO_NUM_CFGS - 1))
 190                 conf_type = LIO_CFG_TYPE_DEFAULT;
 191         oct_conf_info[oct_id].conf_type = conf_type;
 192 }
 193
 194 void
 195 lio_init_device_list(int conf_type)
 196 {
 197         int     i;
 198
 199         bzero(octeon_device, (sizeof(void *) * LIO_MAX_DEVICES));
 200         for (i = 0; i < LIO_MAX_DEVICES; i++)
 201                 oct_set_config_info(i, conf_type);
 202         mtx_init(&octeon_devices_lock, "octeon_devices_lock", NULL, MTX_DEF);
 203 }
 204
 205 static void *
 206 __lio_retrieve_config_info(struct octeon_device *oct, uint16_t card_type)
 207 {
 208         void            *ret = NULL;
 209         uint32_t        oct_id = oct->octeon_id;
 210
 211         switch (oct_conf_info[oct_id].conf_type) {
 212         case LIO_CFG_TYPE_DEFAULT:
 213                 if (oct->chip_id == LIO_CN23XX_PF_VID) {
 214                         ret = &default_cn23xx_conf;
 215                 }
 216
 217                 break;
 218         default:
 219                 break;
 220         }
 221         return (ret);
 222 }
 223
 224 void   *
 225 lio_get_config_info(struct octeon_device *oct, uint16_t card_type)
 226 {
 227         void    *conf = NULL;
 228
 229         conf = __lio_retrieve_config_info(oct, card_type);
 230         if (conf == NULL)
 231                 return (NULL);
 232
 233         return (conf);
 234 }
 235
 236 char   *
 237 lio_get_state_string(volatile int *state_ptr)
 238 {
 239         int32_t istate = (int32_t)atomic_load_acq_int(state_ptr);
 240
 241         if (istate > LIO_DEV_STATES || istate < 0)
 242                 return (lio_state_str[LIO_DEV_STATE_INVALID]);
 243
 244         return (lio_state_str[istate]);
 245 }
 246
 247 static char *
 248 lio_get_app_string(uint32_t app_mode)
 249 {
 250
 251         if (app_mode <= LIO_DRV_APP_END)
 252                 return (lio_app_str[app_mode - LIO_DRV_APP_START]);
 253
 254         return (lio_app_str[LIO_DRV_INVALID_APP - LIO_DRV_APP_START]);
 255 }
 256
 257 void
 258 lio_free_device_mem(struct octeon_device *oct)
 259 {
 260         int     i;
 261
 262         for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {
 263                 if ((oct->io_qmask.oq & BIT_ULL(i)) && (oct->droq[i]))
 264                         free(oct->droq[i], M_DEVBUF);
 265         }
 266
 267         for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {
 268                 if ((oct->io_qmask.iq & BIT_ULL(i)) && (oct->instr_queue[i]))
 269                         free(oct->instr_queue[i], M_DEVBUF);
 270         }
 271
 272         i = oct->octeon_id;
 273         free(oct->chip, M_DEVBUF);
 274
 275         octeon_device[i] = NULL;
 276         octeon_device_count--;
 277 }
 278
 279 static struct octeon_device *
 280 lio_allocate_device_mem(device_t device)
 281 {
 282         struct octeon_device    *oct;
 283         uint32_t        configsize = 0, pci_id = 0, size;
 284         uint8_t         *buf = NULL;
 285
 286         pci_id = pci_get_device(device);
 287         switch (pci_id) {
 288         case LIO_CN23XX_PF_VID:
 289                 configsize = sizeof(struct lio_cn23xx_pf);
 290                 break;
 291         default:
 292                 device_printf(device, "Error: Unknown PCI Device: 0x%x\n",
 293                               pci_id);
 294                 return (NULL);
 295         }
 296
 297         if (configsize & 0x7)
 298                 configsize += (8 - (configsize & 0x7));
 299
 300         size = configsize +
 301                 (sizeof(struct lio_dispatch) * LIO_DISPATCH_LIST_SIZE);
 302
 303         buf = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
 304         if (buf == NULL)
 305                 return (NULL);
 306
 307         oct = (struct octeon_device *)device_get_softc(device);
 308         oct->chip = (void *)(buf);
 309         oct->dispatch.dlist = (struct lio_dispatch *)(buf + configsize);
 310
 311         return (oct);
 312 }
 313
 314 struct octeon_device *
 315 lio_allocate_device(device_t device)
 316 {
 317         struct octeon_device    *oct = NULL;
 318         uint32_t        oct_idx = 0;
 319
 320         mtx_lock(&octeon_devices_lock);
 321
 322         for (oct_idx = 0; oct_idx < LIO_MAX_DEVICES; oct_idx++)
 323                 if (!octeon_device[oct_idx])
 324                         break;
 325
 326         if (oct_idx < LIO_MAX_DEVICES) {
 327                 oct = lio_allocate_device_mem(device);
 328                 if (oct != NULL) {
 329                         octeon_device_count++;
 330                         octeon_device[oct_idx] = oct;
 331                 }
 332         }
 333
 334         mtx_unlock(&octeon_devices_lock);
 335
 336         if (oct == NULL)
 337                 return (NULL);
 338
 339         mtx_init(&oct->pci_win_lock, "pci_win_lock", NULL, MTX_DEF);
 340         mtx_init(&oct->mem_access_lock, "mem_access_lock", NULL, MTX_DEF);
 341
 342         oct->octeon_id = oct_idx;
 343         snprintf(oct->device_name, sizeof(oct->device_name), "%s%d",
 344                  LIO_DRV_NAME, oct->octeon_id);
 345
 346         return (oct);
 347 }
 348
 349 /*
 350  *  Register a device's bus location at initialization time.
 351  *  @param oct        - pointer to the octeon device structure.
 352  *  @param bus        - PCIe bus #
 353  *  @param dev        - PCIe device #
 354  *  @param func       - PCIe function #
 355  *  @param is_pf      - TRUE for PF, FALSE for VF
 356  *  @return reference count of device's adapter
 357  */
 358 int
 359 lio_register_device(struct octeon_device *oct, int bus, int dev, int func,
 360                     int is_pf)
 361 {
 362         int     idx, refcount;
 363
 364         oct->loc.bus = bus;
 365         oct->loc.dev = dev;
 366         oct->loc.func = func;
 367
 368         oct->adapter_refcount = &lio_adapter_refcounts[oct->octeon_id];
 369         atomic_store_rel_int(oct->adapter_refcount, 0);
 370
 371         mtx_lock(&octeon_devices_lock);
 372         for (idx = (int)oct->octeon_id - 1; idx >= 0; idx--) {
 373                 if (octeon_device[idx] == NULL) {
 374                         lio_dev_err(oct, "%s: Internal driver error, missing dev\n",
 375                                     __func__);
 376                         mtx_unlock(&octeon_devices_lock);
 377                         atomic_add_int(oct->adapter_refcount, 1);
 378                         return (1);     /* here, refcount is guaranteed to be 1 */
 379                 }
 380
 381                 /* if another device is at same bus/dev, use its refcounter */
 382                 if ((octeon_device[idx]->loc.bus == bus) &&
 383                     (octeon_device[idx]->loc.dev == dev)) {
 384                         oct->adapter_refcount =
 385                                 octeon_device[idx]->adapter_refcount;
 386                         break;
 387                 }
 388         }
 389
 390         mtx_unlock(&octeon_devices_lock);
 391
 392         atomic_add_int(oct->adapter_refcount, 1);
 393         refcount = atomic_load_acq_int(oct->adapter_refcount);
 394
 395         lio_dev_dbg(oct, "%s: %02x:%02x:%d refcount %u\n", __func__,
 396                     oct->loc.bus, oct->loc.dev, oct->loc.func, refcount);
 397
 398         return (refcount);
 399 }
 400
 401 /*
 402  *  Deregister a device at de-initialization time.
 403  *  @param oct - pointer to the octeon device structure.
 404  *  @return reference count of device's adapter
 405  */
 406 int
 407 lio_deregister_device(struct octeon_device *oct)
 408 {
 409         int     refcount;
 410
 411         atomic_subtract_int(oct->adapter_refcount, 1);
 412         refcount = atomic_load_acq_int(oct->adapter_refcount);
 413
 414         lio_dev_dbg(oct, "%s: %04d:%02d:%d refcount %u\n", __func__,
 415                     oct->loc.bus, oct->loc.dev, oct->loc.func, refcount);
 416
 417         return (refcount);
 418 }
 419
 420 int
 421 lio_allocate_ioq_vector(struct octeon_device *oct)
 422 {
 423         struct lio_ioq_vector   *ioq_vector;
 424         int     i, cpu_num, num_ioqs = 0, size;
 425
 426         if (LIO_CN23XX_PF(oct))
 427                 num_ioqs = oct->sriov_info.num_pf_rings;
 428
 429         size = sizeof(struct lio_ioq_vector) * num_ioqs;
 430
 431         oct->ioq_vector = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
 432         if (oct->ioq_vector == NULL)
 433                 return (1);
 434
 435         for (i = 0; i < num_ioqs; i++) {
 436                 ioq_vector = &oct->ioq_vector[i];
 437                 ioq_vector->oct_dev = oct;
 438                 ioq_vector->droq_index = i;
 439                 cpu_num = i % mp_ncpus;
 440                 CPU_SETOF(cpu_num, &ioq_vector->affinity_mask);
 441
 442                 if (oct->chip_id == LIO_CN23XX_PF_VID)
 443                         ioq_vector->ioq_num = i + oct->sriov_info.pf_srn;
 444                 else
 445                         ioq_vector->ioq_num = i;
 446         }
 447         return (0);
 448 }
 449
 450 void
 451 lio_free_ioq_vector(struct octeon_device *oct)
 452 {
 453
 454         free(oct->ioq_vector, M_DEVBUF);
 455         oct->ioq_vector = NULL;
 456 }
 457
 458 /* this function is only for setting up the first queue */
 459 int
 460 lio_setup_instr_queue0(struct octeon_device *oct)
 461 {
 462         union octeon_txpciq     txpciq;
 463         uint32_t        iq_no = 0;
 464         uint32_t        num_descs = 0;
 465
 466         if (LIO_CN23XX_PF(oct))
 467                 num_descs =
 468                         LIO_GET_NUM_DEF_TX_DESCS_CFG(LIO_CHIP_CONF(oct,
 469                                                                    cn23xx_pf));
 470
 471         oct->num_iqs = 0;
 472
 473         oct->instr_queue[0]->q_index = 0;
 474         oct->instr_queue[0]->app_ctx = (void *)(size_t)0;
 475         oct->instr_queue[0]->ifidx = 0;
 476         txpciq.txpciq64 = 0;
 477         txpciq.s.q_no = iq_no;
 478         txpciq.s.pkind = oct->pfvf_hsword.pkind;
 479         txpciq.s.use_qpg = 0;
 480         txpciq.s.qpg = 0;
 481         if (lio_init_instr_queue(oct, txpciq, num_descs)) {
 482                 /* prevent memory leak */
 483                 lio_delete_instr_queue(oct, 0);
 484                 return (1);
 485         }
 486
 487         oct->num_iqs++;
 488         return (0);
 489 }
 490
 491 int
 492 lio_setup_output_queue0(struct octeon_device *oct)
 493 {
 494         uint32_t        desc_size = 0, num_descs = 0, oq_no = 0;
 495
 496         if (LIO_CN23XX_PF(oct)) {
 497                 num_descs =
 498                         LIO_GET_NUM_DEF_RX_DESCS_CFG(LIO_CHIP_CONF(oct,
 499                                                                    cn23xx_pf));
 500                 desc_size =
 501                         LIO_GET_DEF_RX_BUF_SIZE_CFG(LIO_CHIP_CONF(oct,
 502                                                                   cn23xx_pf));
 503         }
 504
 505         oct->num_oqs = 0;
 506
 507         if (lio_init_droq(oct, oq_no, num_descs, desc_size, NULL)) {
 508                 return (1);
 509         }
 510
 511         oct->num_oqs++;
 512
 513         return (0);
 514 }
 515
 516 int
 517 lio_init_dispatch_list(struct octeon_device *oct)
 518 {
 519         uint32_t        i;
 520
 521         oct->dispatch.count = 0;
 522
 523         for (i = 0; i < LIO_DISPATCH_LIST_SIZE; i++) {
 524                 oct->dispatch.dlist[i].opcode = 0;
 525                 STAILQ_INIT(&oct->dispatch.dlist[i].head);
 526         }
 527
 528         mtx_init(&oct->dispatch.lock, "dispatch_lock", NULL, MTX_DEF);
 529
 530         return (0);
 531 }
 532
 533 void
 534 lio_delete_dispatch_list(struct octeon_device *oct)
 535 {
 536         struct lio_stailq_head  freelist;
 537         struct lio_stailq_node  *temp, *tmp2;
 538         uint32_t                i;
 539
 540         STAILQ_INIT(&freelist);
 541
 542         mtx_lock(&oct->dispatch.lock);
 543
 544         for (i = 0; i < LIO_DISPATCH_LIST_SIZE; i++) {
 545                 struct lio_stailq_head *dispatch;
 546
 547                 dispatch = &oct->dispatch.dlist[i].head;
 548                 while (!STAILQ_EMPTY(dispatch)) {
 549                         temp = STAILQ_FIRST(dispatch);
 550                         STAILQ_REMOVE_HEAD(&oct->dispatch.dlist[i].head,
 551                                            entries);
 552                         STAILQ_INSERT_TAIL(&freelist, temp, entries);
 553                 }
 554
 555                 oct->dispatch.dlist[i].opcode = 0;
 556         }
 557
 558         oct->dispatch.count = 0;
 559
 560         mtx_unlock(&oct->dispatch.lock);
 561
 562         STAILQ_FOREACH_SAFE(temp, &freelist, entries, tmp2) {
 563                 STAILQ_REMOVE_HEAD(&freelist, entries);
 564                 free(temp, M_DEVBUF);
 565         }
 566 }
 567
 568 lio_dispatch_fn_t
 569 lio_get_dispatch(struct octeon_device *octeon_dev, uint16_t opcode,
 570                  uint16_t subcode)
 571 {
 572         struct lio_stailq_node  *dispatch;
 573         lio_dispatch_fn_t       fn = NULL;
 574         uint32_t                idx;
 575         uint16_t        combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode);
 576
 577         idx = combined_opcode & LIO_OPCODE_MASK;
 578
 579         mtx_lock(&octeon_dev->dispatch.lock);
 580
 581         if (octeon_dev->dispatch.count == 0) {
 582                 mtx_unlock(&octeon_dev->dispatch.lock);
 583                 return (NULL);
 584         }
 585
 586         if (!(octeon_dev->dispatch.dlist[idx].opcode)) {
 587                 mtx_unlock(&octeon_dev->dispatch.lock);
 588                 return (NULL);
 589         }
 590
 591         if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
 592                 fn = octeon_dev->dispatch.dlist[idx].dispatch_fn;
 593         } else {
 594                 STAILQ_FOREACH(dispatch, &octeon_dev->dispatch.dlist[idx].head,
 595                                entries) {
 596                         if (((struct lio_dispatch *)dispatch)->opcode ==
 597                             combined_opcode) {
 598                                 fn = ((struct lio_dispatch *)
 599                                       dispatch)->dispatch_fn;
 600                                 break;
 601                         }
 602                 }
 603         }
 604
 605         mtx_unlock(&octeon_dev->dispatch.lock);
 606         return (fn);
 607 }
 608
 609 /*
 610  * lio_register_dispatch_fn
 611  * Parameters:
 612  *   octeon_id - id of the octeon device.
 613  *   opcode    - opcode for which driver should call the registered function
 614  *   subcode   - subcode for which driver should call the registered function
 615  *   fn        - The function to call when a packet with "opcode" arrives in
 616  *               octeon output queues.
 617  *   fn_arg    - The argument to be passed when calling function "fn".
 618  * Description:
 619  *   Registers a function and its argument to be called when a packet
 620  *   arrives in Octeon output queues with "opcode".
 621  * Returns:
 622  *   Success: 0
 623  *   Failure: 1
 624  * Locks:
 625  *   No locks are held.
 626  */
 627 int
 628 lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode,
 629                          uint16_t subcode, lio_dispatch_fn_t fn, void *fn_arg)
 630 {
 631         lio_dispatch_fn_t       pfn;
 632         uint32_t        idx;
 633         uint16_t        combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode);
 634
 635         idx = combined_opcode & LIO_OPCODE_MASK;
 636
 637         mtx_lock(&oct->dispatch.lock);
 638         /* Add dispatch function to first level of lookup table */
 639         if (oct->dispatch.dlist[idx].opcode == 0) {
 640                 oct->dispatch.dlist[idx].opcode = combined_opcode;
 641                 oct->dispatch.dlist[idx].dispatch_fn = fn;
 642                 oct->dispatch.dlist[idx].arg = fn_arg;
 643                 oct->dispatch.count++;
 644                 mtx_unlock(&oct->dispatch.lock);
 645                 return (0);
 646         }
 647
 648         mtx_unlock(&oct->dispatch.lock);
 649
 650         /*
 651          * Check if there was a function already registered for this
 652          * opcode/subcode.
 653          */
 654         pfn = lio_get_dispatch(oct, opcode, subcode);
 655         if (!pfn) {
 656                 struct lio_dispatch *dispatch;
 657
 658                 lio_dev_dbg(oct,
 659                             "Adding opcode to dispatch list linked list\n");
 660                 dispatch = (struct lio_dispatch *)
 661                         malloc(sizeof(struct lio_dispatch),
 662                                M_DEVBUF, M_NOWAIT | M_ZERO);
 663                 if (dispatch == NULL) {
 664                         lio_dev_err(oct,
 665                                     "No memory to add dispatch function\n");
 666                         return (1);
 667                 }
 668
 669                 dispatch->opcode = combined_opcode;
 670                 dispatch->dispatch_fn = fn;
 671                 dispatch->arg = fn_arg;
 672
 673                 /*
 674                  * Add dispatch function to linked list of fn ptrs
 675                  * at the hashed index.
 676                  */
 677                 mtx_lock(&oct->dispatch.lock);
 678                 STAILQ_INSERT_HEAD(&oct->dispatch.dlist[idx].head,
 679                                    &dispatch->node, entries);
 680                 oct->dispatch.count++;
 681                 mtx_unlock(&oct->dispatch.lock);
 682
 683         } else {
 684                 lio_dev_err(oct, "Found previously registered dispatch fn for opcode/subcode: %x/%x\n",
 685                             opcode, subcode);
 686                 return (1);
 687         }
 688
 689         return (0);
 690 }
 691
 692 /*
 693  * lio_unregister_dispatch_fn
 694  * Parameters:
 695  *   oct       - octeon device
 696  *   opcode    - driver should unregister the function for this opcode
 697  *   subcode   - driver should unregister the function for this subcode
 698  * Description:
 699  *   Unregister the function set for this opcode+subcode.
 700  * Returns:
 701  *   Success: 0
 702  *   Failure: 1
 703  * Locks:
 704  *   No locks are held.
 705  */
 706 int
 707 lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode,
 708                            uint16_t subcode)
 709 {
 710         struct lio_stailq_head  *dispatch_head;
 711         struct lio_stailq_node  *dispatch, *dfree = NULL, *tmp2;
 712         int             retval = 0;
 713         uint32_t        idx;
 714         uint16_t        combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode);
 715
 716         idx = combined_opcode & LIO_OPCODE_MASK;
 717
 718         mtx_lock(&oct->dispatch.lock);
 719
 720         if (oct->dispatch.count == 0) {
 721                 mtx_unlock(&oct->dispatch.lock);
 722                 lio_dev_err(oct, "No dispatch functions registered for this device\n");
 723                 return (1);
 724         }
 725         if (oct->dispatch.dlist[idx].opcode == combined_opcode) {
 726                 dispatch_head = &oct->dispatch.dlist[idx].head;
 727                 if (!STAILQ_EMPTY(dispatch_head)) {
 728                         dispatch = STAILQ_FIRST(dispatch_head);
 729                         oct->dispatch.dlist[idx].opcode =
 730                             ((struct lio_dispatch *)dispatch)->opcode;
 731                         oct->dispatch.dlist[idx].dispatch_fn =
 732                             ((struct lio_dispatch *)dispatch)->dispatch_fn;
 733                         oct->dispatch.dlist[idx].arg =
 734                             ((struct lio_dispatch *)dispatch)->arg;
 735                         STAILQ_REMOVE_HEAD(dispatch_head, entries);
 736                         dfree = dispatch;
 737                 } else {
 738                         oct->dispatch.dlist[idx].opcode = 0;
 739                         oct->dispatch.dlist[idx].dispatch_fn = NULL;
 740                         oct->dispatch.dlist[idx].arg = NULL;
 741                 }
 742         } else {
 743                 retval = 1;
 744                 STAILQ_FOREACH_SAFE(dispatch,
 745                                     &oct->dispatch.dlist[idx].head,
 746                                     entries, tmp2) {
 747                         if (((struct lio_dispatch *)dispatch)->opcode ==
 748                             combined_opcode) {
 749                                 STAILQ_REMOVE(&oct->dispatch.dlist[idx].head,
 750                                               dispatch,
 751                                               lio_stailq_node, entries);
 752                                 dfree = dispatch;
 753                                 retval = 0;
 754                         }
 755                 }
 756         }
 757
 758         if (!retval)
 759                 oct->dispatch.count--;
 760
 761         mtx_unlock(&oct->dispatch.lock);
 762         free(dfree, M_DEVBUF);
 763
 764         return (retval);
 765 }
 766
 767 int
 768 lio_core_drv_init(struct lio_recv_info *recv_info, void *buf)
 769 {
 770         struct octeon_device    *oct = (struct octeon_device *)buf;
 771         struct lio_recv_pkt     *recv_pkt = recv_info->recv_pkt;
 772         struct lio_core_setup   *cs = NULL;
 773         uint32_t        i;
 774         uint32_t        num_nic_ports = 0;
 775         char            app_name[16];
 776
 777         if (LIO_CN23XX_PF(oct))
 778                 num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG(
 779                                                LIO_CHIP_CONF(oct, cn23xx_pf));
 780
 781         if (atomic_load_acq_int(&oct->status) >= LIO_DEV_RUNNING) {
 782                 lio_dev_err(oct, "Received CORE OK when device state is 0x%x\n",
 783                             atomic_load_acq_int(&oct->status));
 784                 goto core_drv_init_err;
 785         }
 786
 787         strncpy(app_name,
 788                 lio_get_app_string((uint32_t)
 789                                    recv_pkt->rh.r_core_drv_init.app_mode),
 790                 sizeof(app_name) - 1);
 791         oct->app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;
 792         if (recv_pkt->rh.r_core_drv_init.app_mode == LIO_DRV_NIC_APP) {
 793                 oct->fw_info.max_nic_ports =
 794                     (uint32_t)recv_pkt->rh.r_core_drv_init.max_nic_ports;
 795                 oct->fw_info.num_gmx_ports =
 796                     (uint32_t)recv_pkt->rh.r_core_drv_init.num_gmx_ports;
 797         }
 798
 799         if (oct->fw_info.max_nic_ports < num_nic_ports) {
 800                 lio_dev_err(oct, "Config has more ports than firmware allows (%d > %d).\n",
 801                             num_nic_ports, oct->fw_info.max_nic_ports);
 802                 goto core_drv_init_err;
 803         }
 804
 805         oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags;
 806         oct->fw_info.app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;
 807         oct->pfvf_hsword.app_mode =
 808             (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;
 809
 810         oct->pfvf_hsword.pkind = recv_pkt->rh.r_core_drv_init.pkind;
 811
 812         for (i = 0; i < oct->num_iqs; i++)
 813                 oct->instr_queue[i]->txpciq.s.pkind = oct->pfvf_hsword.pkind;
 814
 815         atomic_store_rel_int(&oct->status, LIO_DEV_CORE_OK);
 816
 817         cs = &core_setup[oct->octeon_id];
 818
 819         if (recv_pkt->buffer_size[0] != (sizeof(*cs) + LIO_DROQ_INFO_SIZE)) {
 820                 lio_dev_dbg(oct, "Core setup bytes expected %llu found %d\n",
 821                             LIO_CAST64(sizeof(*cs) + LIO_DROQ_INFO_SIZE),
 822                             recv_pkt->buffer_size[0]);
 823         }
 824
 825         memcpy(cs, recv_pkt->buffer_ptr[0]->m_data + LIO_DROQ_INFO_SIZE,
 826                sizeof(*cs));
 827         strncpy(oct->boardinfo.name, cs->boardname, LIO_BOARD_NAME);
 828         strncpy(oct->boardinfo.serial_number, cs->board_serial_number,
 829                 LIO_SERIAL_NUM_LEN);
 830
 831         lio_swap_8B_data((uint64_t *)cs, (sizeof(*cs) >> 3));
 832
 833         oct->boardinfo.major = cs->board_rev_major;
 834         oct->boardinfo.minor = cs->board_rev_minor;
 835
 836         lio_dev_info(oct, "Running %s (%llu Hz)\n", app_name,
 837                      LIO_CAST64(cs->corefreq));
 838
 839 core_drv_init_err:
 840         for (i = 0; i < recv_pkt->buffer_count; i++)
 841                 lio_recv_buffer_free(recv_pkt->buffer_ptr[i]);
 842
 843         lio_free_recv_info(recv_info);
 844         return (0);
 845 }
 846
 847 int
 848 lio_get_tx_qsize(struct octeon_device *oct, uint32_t q_no)
 849 {
 850
 851         if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_INSTR_QUEUES(oct)) &&
 852             (oct->io_qmask.iq & BIT_ULL(q_no)))
 853                 return (oct->instr_queue[q_no]->max_count);
 854
 855
 856         return (-1);
 857 }
 858
 859 int
 860 lio_get_rx_qsize(struct octeon_device *oct, uint32_t q_no)
 861 {
 862
 863         if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_OUTPUT_QUEUES(oct)) &&
 864             (oct->io_qmask.oq & BIT_ULL(q_no)))
 865                 return (oct->droq[q_no]->max_count);
 866
 867         return (-1);
 868 }
 869
 870 /* Returns the host firmware handshake OCTEON specific configuration */
 871 struct lio_config *
 872 lio_get_conf(struct octeon_device *oct)
 873 {
 874         struct lio_config       *default_oct_conf = NULL;
 875
 876         /*
 877          * check the OCTEON Device model & return the corresponding octeon
 878          * configuration
 879          */
 880         if (LIO_CN23XX_PF(oct)) {
 881                 default_oct_conf = (struct lio_config *)(
 882                                                LIO_CHIP_CONF(oct, cn23xx_pf));
 883         }
 884
 885         return (default_oct_conf);
 886 }
 887
 888 /*
 889  *  Get the octeon device pointer.
 890  *  @param octeon_id  - The id for which the octeon device pointer is required.
 891  *  @return Success: Octeon device pointer.
 892  *  @return Failure: NULL.
 893  */
 894 struct octeon_device *
 895 lio_get_device(uint32_t octeon_id)
 896 {
 897
 898         if (octeon_id >= LIO_MAX_DEVICES)
 899                 return (NULL);
 900         else
 901                 return (octeon_device[octeon_id]);
 902 }
 903
 904 uint64_t
 905 lio_pci_readq(struct octeon_device *oct, uint64_t addr)
 906 {
 907         uint64_t                val64;
 908         volatile uint32_t       val32, addrhi;
 909
 910         mtx_lock(&oct->pci_win_lock);
 911
 912         /*
 913          * The windowed read happens when the LSB of the addr is written.
 914          * So write MSB first
 915          */
 916         addrhi = (addr >> 32);
 917         if (oct->chip_id == LIO_CN23XX_PF_VID)
 918                 addrhi |= 0x00060000;
 919         lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_hi, addrhi);
 920
 921         /* Read back to preserve ordering of writes */
 922         val32 = lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_hi);
 923
 924         lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_lo,
 925                         addr & 0xffffffff);
 926         val32 = lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_lo);
 927
 928         val64 = lio_read_csr64(oct, oct->reg_list.pci_win_rd_data);
 929
 930         mtx_unlock(&oct->pci_win_lock);
 931
 932         return (val64);
 933 }
 934
 935 void
 936 lio_pci_writeq(struct octeon_device *oct, uint64_t val, uint64_t addr)
 937 {
 938         volatile uint32_t       val32;
 939
 940         mtx_lock(&oct->pci_win_lock);
 941
 942         lio_write_csr64(oct, oct->reg_list.pci_win_wr_addr, addr);
 943
 944         /* The write happens when the LSB is written. So write MSB first. */
 945         lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_hi, val >> 32);
 946         /* Read the MSB to ensure ordering of writes. */
 947         val32 = lio_read_csr32(oct, oct->reg_list.pci_win_wr_data_hi);
 948
 949         lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_lo,
 950                         val & 0xffffffff);
 951
 952         mtx_unlock(&oct->pci_win_lock);
 953 }
 954
 955 int
 956 lio_mem_access_ok(struct octeon_device *oct)
 957 {
 958         uint64_t        access_okay = 0;
 959         uint64_t        lmc0_reset_ctl;
 960
 961         /* Check to make sure a DDR interface is enabled */
 962         if (LIO_CN23XX_PF(oct)) {
 963                 lmc0_reset_ctl = lio_pci_readq(oct, LIO_CN23XX_LMC0_RESET_CTL);
 964                 access_okay =
 965                     (lmc0_reset_ctl & LIO_CN23XX_LMC0_RESET_CTL_DDR3RST_MASK);
 966         }
 967
 968         return (access_okay ? 0 : 1);
 969 }
 970
 971 int
 972 lio_wait_for_ddr_init(struct octeon_device *oct, unsigned long *timeout)
 973 {
 974         int             ret = 1;
 975         uint32_t        ms;
 976
 977         if (timeout == NULL)
 978                 return (ret);
 979
 980         for (ms = 0; ret && ((*timeout == 0) || (ms <= *timeout)); ms += 100) {
 981                 ret = lio_mem_access_ok(oct);
 982
 983                 /* wait 100 ms */
 984                 if (ret)
 985                         lio_sleep_timeout(100);
 986         }
 987
 988         return (ret);
 989 }
 990
 991 /*
 992  *  Get the octeon id assigned to the octeon device passed as argument.
 993  *  This function is exported to other modules.
 994  *  @param dev - octeon device pointer passed as a void *.
 995  *  @return octeon device id
 996  */
 997 int
 998 lio_get_device_id(void *dev)
 999 {
1000         struct octeon_device    *octeon_dev = (struct octeon_device *)dev;
1001         uint32_t                i;
1002
1003         for (i = 0; i < LIO_MAX_DEVICES; i++)
1004                 if (octeon_device[i] == octeon_dev)
1005                         return (octeon_dev->octeon_id);
1006
1007         return (-1);
1008 }
1009
1010 void
1011 lio_enable_irq(struct lio_droq *droq, struct lio_instr_queue *iq)
1012 {
1013         struct octeon_device *oct = NULL;
1014         uint64_t        instr_cnt;
1015         uint32_t        pkts_pend;
1016
1017         /* the whole thing needs to be atomic, ideally */
1018         if (droq != NULL) {
1019                 oct = droq->oct_dev;
1020                 pkts_pend = atomic_load_acq_int(&droq->pkts_pending);
1021                 mtx_lock(&droq->lock);
1022                 lio_write_csr32(oct, droq->pkts_sent_reg,
1023                                 droq->pkt_count - pkts_pend);
1024                 droq->pkt_count = pkts_pend;
1025                 /* this write needs to be flushed before we release the lock */
1026                 __compiler_membar();
1027                 mtx_unlock(&droq->lock);
1028         }
1029
1030         if (iq != NULL) {
1031                 oct = iq->oct_dev;
1032                 mtx_lock(&iq->lock);
1033                 lio_write_csr32(oct, iq->inst_cnt_reg, iq->pkt_in_done);
1034                 iq->pkt_in_done = 0;
1035                 /* this write needs to be flushed before we release the lock */
1036                 __compiler_membar();
1037                 mtx_unlock(&iq->lock);
1038         }
1039
1040         /*
1041          * Implementation note:
1042          *
1043          * SLI_PKT(x)_CNTS[RESEND] is written separately so that if an interrupt
1044          * DOES occur as a result of RESEND, the DROQ lock will NOT be held.
1045          *
1046          * Write resend. Writing RESEND in SLI_PKTX_CNTS should be enough
1047          * to trigger tx interrupts as well, if they are pending.
1048          */
1049         if ((oct != NULL) && (LIO_CN23XX_PF(oct))) {
1050                 if (droq != NULL)
1051                         lio_write_csr64(oct, droq->pkts_sent_reg,
1052                                         LIO_CN23XX_INTR_RESEND);
1053                 /* we race with firmrware here. */
1054                 /* read and write the IN_DONE_CNTS */
1055                 else if (iq != NULL) {
1056                         instr_cnt = lio_read_csr64(oct, iq->inst_cnt_reg);
1057                         lio_write_csr64(oct, iq->inst_cnt_reg,
1058                                         ((instr_cnt & 0xFFFFFFFF00000000ULL) |
1059                                          LIO_CN23XX_INTR_RESEND));
1060                 }
1061         }
1062 }