sys/dev/ocs_fc/ocs_hw_queues.c

   1 /*-
   2  * Copyright (c) 2017 Broadcom. All rights reserved.
   3  * The term "Broadcom" refers to Broadcom Limited and/or its subsidiaries.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  *
  11  * 2. Redistributions in binary form must reproduce the above copyright notice,
  12  *    this list of conditions and the following disclaimer in the documentation
  13  *    and/or other materials provided with the distribution.
  14  *
  15  * 3. Neither the name of the copyright holder nor the names of its contributors
  16  *    may be used to endorse or promote products derived from this software
  17  *    without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  23  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29  * POSSIBILITY OF SUCH DAMAGE.
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 /**
  35  * @file
  36  *
  37  */
  38
  39 #include "ocs_os.h"
  40 #include "ocs_hw.h"
  41 #include "ocs_hw_queues.h"
  42
  43 #define HW_QTOP_DEBUG           0
  44
  45 /**
  46  * @brief Initialize queues
  47  *
  48  * Given the parsed queue topology spec, the SLI queues are created and
  49  * initialized
  50  *
  51  * @param hw pointer to HW object
  52  * @param qtop pointer to queue topology
  53  *
  54  * @return returns 0 for success, an error code value for failure.
  55  */
  56 ocs_hw_rtn_e
  57 ocs_hw_init_queues(ocs_hw_t *hw, ocs_hw_qtop_t *qtop)
  58 {
  59         uint32_t i, j;
  60         uint32_t default_lengths[QTOP_LAST], len;
  61         uint32_t rqset_len = 0, rqset_ulp = 0, rqset_count = 0;
  62         uint8_t rqset_filter_mask = 0;
  63         hw_eq_t *eqs[hw->config.n_rq];
  64         hw_cq_t *cqs[hw->config.n_rq];
  65         hw_rq_t *rqs[hw->config.n_rq];
  66         ocs_hw_qtop_entry_t *qt, *next_qt;
  67         ocs_hw_mrq_t mrq;
  68         bool use_mrq = FALSE;
  69
  70         hw_eq_t *eq = NULL;
  71         hw_cq_t *cq = NULL;
  72         hw_wq_t *wq = NULL;
  73         hw_rq_t *rq = NULL;
  74         hw_mq_t *mq = NULL;
  75
  76         mrq.num_pairs = 0;
  77         default_lengths[QTOP_EQ] = 1024;
  78         default_lengths[QTOP_CQ] = hw->num_qentries[SLI_QTYPE_CQ];
  79         default_lengths[QTOP_WQ] = hw->num_qentries[SLI_QTYPE_WQ];
  80         default_lengths[QTOP_RQ] = hw->num_qentries[SLI_QTYPE_RQ];
  81         default_lengths[QTOP_MQ] = OCS_HW_MQ_DEPTH;
  82
  83         ocs_hw_verify(hw != NULL, OCS_HW_RTN_INVALID_ARG);
  84
  85         hw->eq_count = 0;
  86         hw->cq_count = 0;
  87         hw->mq_count = 0;
  88         hw->wq_count = 0;
  89         hw->rq_count = 0;
  90         hw->hw_rq_count = 0;
  91         ocs_list_init(&hw->eq_list, hw_eq_t, link);
  92
  93         /* If MRQ is requested, Check if it is supported by SLI. */
  94         if ((hw->config.n_rq > 1 ) && !hw->sli.config.features.flag.mrqp) {
  95                 ocs_log_err(hw->os, "MRQ topology not supported by SLI4.\n");
  96                 return OCS_HW_RTN_ERROR;
  97         }
  98
  99         if (hw->config.n_rq > 1)
 100                 use_mrq = TRUE;
 101
 102         /* Allocate class WQ pools */
 103         for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) {
 104                 hw->wq_class_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ);
 105                 if (hw->wq_class_array[i] == NULL) {
 106                         ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n");
 107                         return OCS_HW_RTN_NO_MEMORY;
 108                 }
 109         }
 110
 111         /* Allocate per CPU WQ pools */
 112         for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) {
 113                 hw->wq_cpu_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ);
 114                 if (hw->wq_cpu_array[i] == NULL) {
 115                         ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n");
 116                         return OCS_HW_RTN_NO_MEMORY;
 117                 }
 118         }
 119
 120         ocs_hw_assert(qtop != NULL);
 121
 122         for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) {
 123                 if (i == qtop->inuse_count - 1)
 124                         next_qt = NULL;
 125                 else
 126                         next_qt = qt + 1;
 127
 128                 switch(qt->entry) {
 129                 case QTOP_EQ:
 130                         len = (qt->len) ? qt->len : default_lengths[QTOP_EQ];
 131
 132                         if (qt->set_default) {
 133                                 default_lengths[QTOP_EQ] = len;
 134                                 break;
 135                         }
 136
 137                         eq = hw_new_eq(hw, len);
 138                         if (eq == NULL) {
 139                                 hw_queue_teardown(hw);
 140                                 return OCS_HW_RTN_NO_MEMORY;
 141                         }
 142                         break;
 143
 144                 case QTOP_CQ:
 145                         len = (qt->len) ? qt->len : default_lengths[QTOP_CQ];
 146
 147                         if (qt->set_default) {
 148                                 default_lengths[QTOP_CQ] = len;
 149                                 break;
 150                         }
 151
 152                         if (!eq || !next_qt) {
 153                                 goto fail;
 154                         }
 155
 156                         /* If this CQ is for MRQ, then delay the creation */
 157                         if (!use_mrq || next_qt->entry != QTOP_RQ) {
 158                                 cq = hw_new_cq(eq, len);
 159                                 if (cq == NULL) {
 160                                         goto fail;
 161                                 }
 162                         }
 163                         break;
 164
 165                 case QTOP_WQ: {
 166                         len = (qt->len) ? qt->len : default_lengths[QTOP_WQ];
 167                         if (qt->set_default) {
 168                                 default_lengths[QTOP_WQ] = len;
 169                                 break;
 170                         }
 171
 172                         if ((hw->ulp_start + qt->ulp) > hw->ulp_max) {
 173                                 ocs_log_err(hw->os, "invalid ULP %d for WQ\n", qt->ulp);
 174                                 hw_queue_teardown(hw);
 175                                 return OCS_HW_RTN_NO_MEMORY;
 176                         }
 177
 178                         if (cq == NULL)
 179                                 goto fail;
 180
 181                         wq = hw_new_wq(cq, len, qt->class, hw->ulp_start + qt->ulp);
 182                         if (wq == NULL) {
 183                                 goto fail;
 184                         }
 185
 186                         /* Place this WQ on the EQ WQ array */
 187                         if (ocs_varray_add(eq->wq_array, wq)) {
 188                                 ocs_log_err(hw->os, "QTOP_WQ: EQ ocs_varray_add failed\n");
 189                                 hw_queue_teardown(hw);
 190                                 return OCS_HW_RTN_ERROR;
 191                         }
 192
 193                         /* Place this WQ on the HW class array */
 194                         if (qt->class < ARRAY_SIZE(hw->wq_class_array)) {
 195                                 if (ocs_varray_add(hw->wq_class_array[qt->class], wq)) {
 196                                         ocs_log_err(hw->os, "HW wq_class_array ocs_varray_add failed\n");
 197                                         hw_queue_teardown(hw);
 198                                         return OCS_HW_RTN_ERROR;
 199                                 }
 200                         } else {
 201                                 ocs_log_err(hw->os, "Invalid class value: %d\n", qt->class);
 202                                 hw_queue_teardown(hw);
 203                                 return OCS_HW_RTN_ERROR;
 204                         }
 205
 206                         /*
 207                          * Place this WQ on the per CPU list, asumming that EQs are mapped to cpu given
 208                          * by the EQ instance modulo number of CPUs
 209                          */
 210                         if (ocs_varray_add(hw->wq_cpu_array[eq->instance % ocs_get_num_cpus()], wq)) {
 211                                 ocs_log_err(hw->os, "HW wq_cpu_array ocs_varray_add failed\n");
 212                                 hw_queue_teardown(hw);
 213                                 return OCS_HW_RTN_ERROR;
 214                         }
 215
 216                         break;
 217                 }
 218                 case QTOP_RQ: {
 219                         len = (qt->len) ? qt->len : default_lengths[QTOP_RQ];
 220                         if (qt->set_default) {
 221                                 default_lengths[QTOP_RQ] = len;
 222                                 break;
 223                         }
 224
 225                         if ((hw->ulp_start + qt->ulp) > hw->ulp_max) {
 226                                 ocs_log_err(hw->os, "invalid ULP %d for RQ\n", qt->ulp);
 227                                 hw_queue_teardown(hw);
 228                                 return OCS_HW_RTN_NO_MEMORY;
 229                         }
 230
 231                         if (use_mrq) {
 232                                 mrq.rq_cfg[mrq.num_pairs].len = len;
 233                                 mrq.rq_cfg[mrq.num_pairs].ulp = hw->ulp_start + qt->ulp;
 234                                 mrq.rq_cfg[mrq.num_pairs].filter_mask = qt->filter_mask;
 235                                 mrq.rq_cfg[mrq.num_pairs].eq = eq;
 236                                 mrq.num_pairs ++;
 237                         } else {
 238                                 rq = hw_new_rq(cq, len, hw->ulp_start + qt->ulp);
 239                                 if (rq == NULL) {
 240                                         hw_queue_teardown(hw);
 241                                         return OCS_HW_RTN_NO_MEMORY;
 242                                 }
 243                                 rq->filter_mask = qt->filter_mask;
 244                         }
 245                         break;
 246                 }
 247
 248                 case QTOP_MQ:
 249                         len = (qt->len) ? qt->len : default_lengths[QTOP_MQ];
 250                         if (qt->set_default) {
 251                                 default_lengths[QTOP_MQ] = len;
 252                                 break;
 253                         }
 254
 255                         if (cq == NULL)
 256                                 goto fail;
 257
 258                         mq = hw_new_mq(cq, len);
 259                         if (mq == NULL) {
 260                                 goto fail;
 261                         }
 262                         break;
 263
 264                 default:
 265                         ocs_hw_assert(0);
 266                         break;
 267                 }
 268         }
 269
 270         if (mrq.num_pairs) {
 271                 /* First create normal RQs. */
 272                 for (i = 0; i < mrq.num_pairs; i++) {
 273                         for (j = 0; j < mrq.num_pairs; j++) {
 274                                 if ((i != j) && (mrq.rq_cfg[i].filter_mask == mrq.rq_cfg[j].filter_mask)) {
 275                                         /* This should be created using set */
 276                                         if (rqset_filter_mask && (rqset_filter_mask != mrq.rq_cfg[i].filter_mask)) {
 277                                                 ocs_log_crit(hw->os, "Cant create morethan one RQ Set\n");
 278                                                 hw_queue_teardown(hw);
 279                                                 return OCS_HW_RTN_ERROR;
 280                                         } else if (!rqset_filter_mask){
 281                                                 rqset_filter_mask = mrq.rq_cfg[i].filter_mask;
 282                                                 rqset_len = mrq.rq_cfg[i].len;
 283                                                 rqset_ulp = mrq.rq_cfg[i].ulp;
 284                                         }
 285                                         eqs[rqset_count] = mrq.rq_cfg[i].eq;
 286                                         rqset_count++;
 287                                         break;
 288                                 }
 289                         }
 290                         if (j == mrq.num_pairs) {
 291                                 /* Normal RQ */
 292                                 cq = hw_new_cq(mrq.rq_cfg[i].eq, default_lengths[QTOP_CQ]);
 293                                 if (cq == NULL) {
 294                                         hw_queue_teardown(hw);
 295                                         return OCS_HW_RTN_NO_MEMORY;
 296                                 }
 297
 298                                 rq = hw_new_rq(cq, mrq.rq_cfg[i].len, mrq.rq_cfg[i].ulp);
 299                                 if (rq == NULL) {
 300                                         hw_queue_teardown(hw);
 301                                         return OCS_HW_RTN_NO_MEMORY;
 302                                 }
 303                                 rq->filter_mask = mrq.rq_cfg[i].filter_mask;
 304                         }
 305                 }
 306
 307                 /* Now create RQ Set */
 308                 if (rqset_count) {
 309                         if (rqset_count > OCE_HW_MAX_NUM_MRQ_PAIRS) {
 310                                 ocs_log_crit(hw->os,
 311                                              "Max Supported MRQ pairs = %d\n",
 312                                              OCE_HW_MAX_NUM_MRQ_PAIRS);
 313                                 hw_queue_teardown(hw);
 314                                 return OCS_HW_RTN_ERROR;
 315                         }
 316
 317                         /* Create CQ set */
 318                         if (hw_new_cq_set(eqs, cqs, rqset_count, default_lengths[QTOP_CQ])) {
 319                                 hw_queue_teardown(hw);
 320                                 return OCS_HW_RTN_ERROR;
 321                         }
 322
 323                         /* Create RQ set */
 324                         if (hw_new_rq_set(cqs, rqs, rqset_count, rqset_len, rqset_ulp)) {
 325                                 hw_queue_teardown(hw);
 326                                 return OCS_HW_RTN_ERROR;
 327                         }
 328
 329                         for (i = 0; i < rqset_count ; i++) {
 330                                 rqs[i]->filter_mask = rqset_filter_mask;
 331                                 rqs[i]->is_mrq = TRUE;
 332                                 rqs[i]->base_mrq_id = rqs[0]->hdr->id;
 333                         }
 334
 335                         hw->hw_mrq_count = rqset_count;
 336                 }
 337         }
 338
 339         return OCS_HW_RTN_SUCCESS;
 340 fail:
 341         hw_queue_teardown(hw);
 342         return OCS_HW_RTN_NO_MEMORY;
 343
 344 }
 345
 346 /**
 347  * @brief Allocate a new EQ object
 348  *
 349  * A new EQ object is instantiated
 350  *
 351  * @param hw pointer to HW object
 352  * @param entry_count number of entries in the EQ
 353  *
 354  * @return pointer to allocated EQ object
 355  */
 356 hw_eq_t*
 357 hw_new_eq(ocs_hw_t *hw, uint32_t entry_count)
 358 {
 359         hw_eq_t *eq = ocs_malloc(hw->os, sizeof(*eq), OCS_M_ZERO | OCS_M_NOWAIT);
 360
 361         if (eq != NULL) {
 362                 eq->type = SLI_QTYPE_EQ;
 363                 eq->hw = hw;
 364                 eq->entry_count = entry_count;
 365                 eq->instance = hw->eq_count++;
 366                 eq->queue = &hw->eq[eq->instance];
 367                 ocs_list_init(&eq->cq_list, hw_cq_t, link);
 368
 369                 eq->wq_array = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ);
 370                 if (eq->wq_array == NULL) {
 371                         ocs_free(hw->os, eq, sizeof(*eq));
 372                         eq = NULL;
 373                 } else {
 374                         if (sli_queue_alloc(&hw->sli, SLI_QTYPE_EQ, eq->queue, entry_count, NULL, 0)) {
 375                                 ocs_log_err(hw->os, "EQ[%d] allocation failure\n", eq->instance);
 376                                 ocs_free(hw->os, eq, sizeof(*eq));
 377                                 eq = NULL;
 378                         } else {
 379                                 sli_eq_modify_delay(&hw->sli, eq->queue, 1, 0, 8);
 380                                 hw->hw_eq[eq->instance] = eq;
 381                                 ocs_list_add_tail(&hw->eq_list, eq);
 382                                 ocs_log_debug(hw->os, "create eq[%2d] id %3d len %4d\n", eq->instance, eq->queue->id,
 383                                         eq->entry_count);
 384                         }
 385                 }
 386         }
 387         return eq;
 388 }
 389
 390 /**
 391  * @brief Allocate a new CQ object
 392  *
 393  * A new CQ object is instantiated
 394  *
 395  * @param eq pointer to parent EQ object
 396  * @param entry_count number of entries in the CQ
 397  *
 398  * @return pointer to allocated CQ object
 399  */
 400 hw_cq_t*
 401 hw_new_cq(hw_eq_t *eq, uint32_t entry_count)
 402 {
 403         ocs_hw_t *hw = eq->hw;
 404         hw_cq_t *cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT);
 405
 406         if (cq != NULL) {
 407                 cq->eq = eq;
 408                 cq->type = SLI_QTYPE_CQ;
 409                 cq->instance = eq->hw->cq_count++;
 410                 cq->entry_count = entry_count;
 411                 cq->queue = &hw->cq[cq->instance];
 412
 413                 ocs_list_init(&cq->q_list, hw_q_t, link);
 414
 415                 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_CQ, cq->queue, cq->entry_count, eq->queue, 0)) {
 416                         ocs_log_err(hw->os, "CQ[%d] allocation failure len=%d\n",
 417                                 eq->instance,
 418                                 eq->entry_count);
 419                         ocs_free(hw->os, cq, sizeof(*cq));
 420                         cq = NULL;
 421                 } else {
 422                         hw->hw_cq[cq->instance] = cq;
 423                         ocs_list_add_tail(&eq->cq_list, cq);
 424                         ocs_log_debug(hw->os, "create cq[%2d] id %3d len %4d\n", cq->instance, cq->queue->id,
 425                                 cq->entry_count);
 426                 }
 427         }
 428         return cq;
 429 }
 430
 431 /**
 432  * @brief Allocate a new CQ Set of objects.
 433  *
 434  * @param eqs pointer to a set of EQ objects.
 435  * @param cqs pointer to a set of CQ objects to be returned.
 436  * @param num_cqs number of CQ queues in the set.
 437  * @param entry_count number of entries in the CQ.
 438  *
 439  * @return 0 on success and -1 on failure.
 440  */
 441 uint32_t
 442 hw_new_cq_set(hw_eq_t *eqs[], hw_cq_t *cqs[], uint32_t num_cqs, uint32_t entry_count)
 443 {
 444         uint32_t i;
 445         ocs_hw_t *hw = eqs[0]->hw;
 446         sli4_t *sli4 = &hw->sli;
 447         hw_cq_t *cq = NULL;
 448         sli4_queue_t *qs[SLI_MAX_CQ_SET_COUNT], *assocs[SLI_MAX_CQ_SET_COUNT];
 449
 450         /* Initialise CQS pointers to NULL */
 451         for (i = 0; i < num_cqs; i++) {
 452                 cqs[i] = NULL;
 453         }
 454
 455         for (i = 0; i < num_cqs; i++) {
 456                 cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT);
 457                 if (cq == NULL)
 458                         goto error;
 459
 460                 cqs[i]          = cq;
 461                 cq->eq          = eqs[i];
 462                 cq->type        = SLI_QTYPE_CQ;
 463                 cq->instance    = hw->cq_count++;
 464                 cq->entry_count = entry_count;
 465                 cq->queue       = &hw->cq[cq->instance];
 466                 qs[i]           = cq->queue;
 467                 assocs[i]       = eqs[i]->queue;
 468                 ocs_list_init(&cq->q_list, hw_q_t, link);
 469         }
 470
 471         if (sli_cq_alloc_set(sli4, qs, num_cqs, entry_count, assocs)) {
 472                 ocs_log_err(NULL, "Failed to create CQ Set. \n");
 473                 goto error;
 474         }
 475
 476         for (i = 0; i < num_cqs; i++) {
 477                 hw->hw_cq[cqs[i]->instance] = cqs[i];
 478                 ocs_list_add_tail(&cqs[i]->eq->cq_list, cqs[i]);
 479         }
 480
 481         return 0;
 482
 483 error:
 484         for (i = 0; i < num_cqs; i++) {
 485                 if (cqs[i]) {
 486                         ocs_free(hw->os, cqs[i], sizeof(*cqs[i]));
 487                         cqs[i] = NULL;
 488                 }
 489         }
 490         return -1;
 491 }
 492
 493 /**
 494  * @brief Allocate a new MQ object
 495  *
 496  * A new MQ object is instantiated
 497  *
 498  * @param cq pointer to parent CQ object
 499  * @param entry_count number of entries in the MQ
 500  *
 501  * @return pointer to allocated MQ object
 502  */
 503 hw_mq_t*
 504 hw_new_mq(hw_cq_t *cq, uint32_t entry_count)
 505 {
 506         ocs_hw_t *hw = cq->eq->hw;
 507         hw_mq_t *mq = ocs_malloc(hw->os, sizeof(*mq), OCS_M_ZERO | OCS_M_NOWAIT);
 508
 509         if (mq != NULL) {
 510                 mq->cq = cq;
 511                 mq->type = SLI_QTYPE_MQ;
 512                 mq->instance = cq->eq->hw->mq_count++;
 513                 mq->entry_count = entry_count;
 514                 mq->entry_size = OCS_HW_MQ_DEPTH;
 515                 mq->queue = &hw->mq[mq->instance];
 516
 517                 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_MQ,
 518                                     mq->queue,
 519                                     mq->entry_size,
 520                                     cq->queue, 0)) {
 521                         ocs_log_err(hw->os, "MQ allocation failure\n");
 522                         ocs_free(hw->os, mq, sizeof(*mq));
 523                         mq = NULL;
 524                 } else {
 525                         hw->hw_mq[mq->instance] = mq;
 526                         ocs_list_add_tail(&cq->q_list, mq);
 527                         ocs_log_debug(hw->os, "create mq[%2d] id %3d len %4d\n", mq->instance, mq->queue->id,
 528                                 mq->entry_count);
 529                 }
 530         }
 531         return mq;
 532 }
 533
 534 /**
 535  * @brief Allocate a new WQ object
 536  *
 537  * A new WQ object is instantiated
 538  *
 539  * @param cq pointer to parent CQ object
 540  * @param entry_count number of entries in the WQ
 541  * @param class WQ class
 542  * @param ulp index of chute
 543  *
 544  * @return pointer to allocated WQ object
 545  */
 546 hw_wq_t*
 547 hw_new_wq(hw_cq_t *cq, uint32_t entry_count, uint32_t class, uint32_t ulp)
 548 {
 549         ocs_hw_t *hw = cq->eq->hw;
 550         hw_wq_t *wq = ocs_malloc(hw->os, sizeof(*wq), OCS_M_ZERO | OCS_M_NOWAIT);
 551
 552         if (wq != NULL) {
 553                 wq->hw = cq->eq->hw;
 554                 wq->cq = cq;
 555                 wq->type = SLI_QTYPE_WQ;
 556                 wq->instance = cq->eq->hw->wq_count++;
 557                 wq->entry_count = entry_count;
 558                 wq->queue = &hw->wq[wq->instance];
 559                 wq->ulp = ulp;
 560                 wq->wqec_set_count = OCS_HW_WQEC_SET_COUNT;
 561                 wq->wqec_count = wq->wqec_set_count;
 562                 wq->free_count = wq->entry_count - 1;
 563                 wq->class = class;
 564                 ocs_list_init(&wq->pending_list, ocs_hw_wqe_t, link);
 565
 566                 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_WQ, wq->queue, wq->entry_count, cq->queue, ulp)) {
 567                         ocs_log_err(hw->os, "WQ allocation failure\n");
 568                         ocs_free(hw->os, wq, sizeof(*wq));
 569                         wq = NULL;
 570                 } else {
 571                         hw->hw_wq[wq->instance] = wq;
 572                         ocs_list_add_tail(&cq->q_list, wq);
 573                         ocs_log_debug(hw->os, "create wq[%2d] id %3d len %4d cls %d ulp %d\n", wq->instance, wq->queue->id,
 574                                 wq->entry_count, wq->class, wq->ulp);
 575                 }
 576         }
 577         return wq;
 578 }
 579
 580 /**
 581  * @brief Allocate a hw_rq_t object
 582  *
 583  * Allocate an RQ object, which encapsulates 2 SLI queues (for rq pair)
 584  *
 585  * @param cq pointer to parent CQ object
 586  * @param entry_count number of entries in the RQs
 587  * @param ulp ULP index for this RQ
 588  *
 589  * @return pointer to newly allocated hw_rq_t
 590  */
 591 hw_rq_t*
 592 hw_new_rq(hw_cq_t *cq, uint32_t entry_count, uint32_t ulp)
 593 {
 594         ocs_hw_t *hw = cq->eq->hw;
 595         hw_rq_t *rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT);
 596         uint32_t max_hw_rq;
 597
 598         ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq);
 599
 600         if (rq != NULL) {
 601                 rq->instance = hw->hw_rq_count++;
 602                 rq->cq = cq;
 603                 rq->type = SLI_QTYPE_RQ;
 604                 rq->ulp = ulp;
 605
 606                 rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR));
 607
 608                 /* Create the header RQ */
 609                 ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq));
 610                 rq->hdr = &hw->rq[hw->rq_count];
 611                 rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE;
 612
 613                 if (sli_fc_rq_alloc(&hw->sli, rq->hdr,
 614                                     rq->entry_count,
 615                                     rq->hdr_entry_size,
 616                                     cq->queue,
 617                                     ulp, TRUE)) {
 618                         ocs_log_err(hw->os, "RQ allocation failure - header\n");
 619                         ocs_free(hw->os, rq, sizeof(*rq));
 620                         return NULL;
 621                 }
 622                 hw->hw_rq_lookup[hw->rq_count] = rq->instance;  /* Update hw_rq_lookup[] */
 623                 hw->rq_count++;
 624                 ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d hdr  size %4d ulp %d\n",
 625                         rq->instance, rq->hdr->id, rq->entry_count, rq->hdr_entry_size, rq->ulp);
 626
 627                 /* Create the default data RQ */
 628                 ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq));
 629                 rq->data = &hw->rq[hw->rq_count];
 630                 rq->data_entry_size = hw->config.rq_default_buffer_size;
 631
 632                 if (sli_fc_rq_alloc(&hw->sli, rq->data,
 633                                     rq->entry_count,
 634                                     rq->data_entry_size,
 635                                     cq->queue,
 636                                     ulp, FALSE)) {
 637                         ocs_log_err(hw->os, "RQ allocation failure - first burst\n");
 638                         ocs_free(hw->os, rq, sizeof(*rq));
 639                         return NULL;
 640                 }
 641                 hw->hw_rq_lookup[hw->rq_count] = rq->instance;  /* Update hw_rq_lookup[] */
 642                 hw->rq_count++;
 643                 ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d data size %4d ulp %d\n", rq->instance,
 644                         rq->data->id, rq->entry_count, rq->data_entry_size, rq->ulp);
 645
 646                 hw->hw_rq[rq->instance] = rq;
 647                 ocs_list_add_tail(&cq->q_list, rq);
 648
 649                 rq->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) *
 650                                             rq->entry_count, OCS_M_ZERO | OCS_M_NOWAIT);
 651                 if (rq->rq_tracker == NULL) {
 652                         ocs_log_err(hw->os, "RQ tracker buf allocation failure\n");
 653                         return NULL;
 654                 }
 655         }
 656         return rq;
 657 }
 658
 659 /**
 660  * @brief Allocate a hw_rq_t object SET
 661  *
 662  * Allocate an RQ object SET, where each element in set
 663  * encapsulates 2 SLI queues (for rq pair)
 664  *
 665  * @param cqs pointers to be associated with RQs.
 666  * @param rqs RQ pointers to be returned on success.
 667  * @param num_rq_pairs number of rq pairs in the Set.
 668  * @param entry_count number of entries in the RQs
 669  * @param ulp ULP index for this RQ
 670  *
 671  * @return 0 in success and -1 on failure.
 672  */
 673 uint32_t
 674 hw_new_rq_set(hw_cq_t *cqs[], hw_rq_t *rqs[], uint32_t num_rq_pairs, uint32_t entry_count, uint32_t ulp)
 675 {
 676         ocs_hw_t *hw = cqs[0]->eq->hw;
 677         hw_rq_t *rq = NULL;
 678         sli4_queue_t *qs[SLI_MAX_RQ_SET_COUNT * 2] = { NULL };
 679         uint32_t max_hw_rq, i, q_count;
 680
 681         ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq);
 682
 683         /* Initialise RQS pointers */
 684         for (i = 0; i < num_rq_pairs; i++) {
 685                 rqs[i] = NULL;
 686         }
 687
 688         for (i = 0, q_count = 0; i < num_rq_pairs; i++, q_count += 2) {
 689                 rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT);
 690                 if (rq == NULL)
 691                         goto error;
 692
 693                 rqs[i] = rq;
 694                 rq->instance = hw->hw_rq_count++;
 695                 rq->cq = cqs[i];
 696                 rq->type = SLI_QTYPE_RQ;
 697                 rq->ulp = ulp;
 698                 rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR));
 699
 700                 /* Header RQ */
 701                 rq->hdr = &hw->rq[hw->rq_count];
 702                 rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE;
 703                 hw->hw_rq_lookup[hw->rq_count] = rq->instance;
 704                 hw->rq_count++;
 705                 qs[q_count] = rq->hdr;
 706
 707                 /* Data RQ */
 708                 rq->data = &hw->rq[hw->rq_count];
 709                 rq->data_entry_size = hw->config.rq_default_buffer_size;
 710                 hw->hw_rq_lookup[hw->rq_count] = rq->instance;
 711                 hw->rq_count++;
 712                 qs[q_count + 1] = rq->data;
 713
 714                 rq->rq_tracker = NULL;
 715         }
 716
 717         if (sli_fc_rq_set_alloc(&hw->sli, num_rq_pairs, qs,
 718                             cqs[0]->queue->id,
 719                             rqs[0]->entry_count,
 720                             rqs[0]->hdr_entry_size,
 721                             rqs[0]->data_entry_size,
 722                             ulp)) {
 723                 ocs_log_err(hw->os, "RQ Set allocation failure for base CQ=%d\n", cqs[0]->queue->id);
 724                 goto error;
 725         }
 726
 727         for (i = 0; i < num_rq_pairs; i++) {
 728                 hw->hw_rq[rqs[i]->instance] = rqs[i];
 729                 ocs_list_add_tail(&cqs[i]->q_list, rqs[i]);
 730                 rqs[i]->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) *
 731                                             rqs[i]->entry_count, OCS_M_ZERO | OCS_M_NOWAIT);
 732                 if (rqs[i]->rq_tracker == NULL) {
 733                         ocs_log_err(hw->os, "RQ tracker buf allocation failure\n");
 734                         goto error;
 735                 }
 736         }
 737
 738         return 0;
 739
 740 error:
 741         for (i = 0; i < num_rq_pairs; i++) {
 742                 if (rqs[i] != NULL) {
 743                         if (rqs[i]->rq_tracker != NULL) {
 744                                 ocs_free(hw->os, rqs[i]->rq_tracker,
 745                                          sizeof(ocs_hw_sequence_t*) *
 746                                          rqs[i]->entry_count);
 747                         }
 748                         ocs_free(hw->os, rqs[i], sizeof(*rqs[i]));
 749                 }
 750         }
 751
 752         return -1;
 753 }
 754
 755 /**
 756  * @brief Free an EQ object
 757  *
 758  * The EQ object and any child queue objects are freed
 759  *
 760  * @param eq pointer to EQ object
 761  *
 762  * @return none
 763  */
 764 void
 765 hw_del_eq(hw_eq_t *eq)
 766 {
 767         if (eq != NULL) {
 768                 hw_cq_t *cq;
 769                 hw_cq_t *cq_next;
 770
 771                 ocs_list_foreach_safe(&eq->cq_list, cq, cq_next) {
 772                         hw_del_cq(cq);
 773                 }
 774                 ocs_varray_free(eq->wq_array);
 775                 ocs_list_remove(&eq->hw->eq_list, eq);
 776                 eq->hw->hw_eq[eq->instance] = NULL;
 777                 ocs_free(eq->hw->os, eq, sizeof(*eq));
 778         }
 779 }
 780
 781 /**
 782  * @brief Free a CQ object
 783  *
 784  * The CQ object and any child queue objects are freed
 785  *
 786  * @param cq pointer to CQ object
 787  *
 788  * @return none
 789  */
 790 void
 791 hw_del_cq(hw_cq_t *cq)
 792 {
 793         if (cq != NULL) {
 794                 hw_q_t *q;
 795                 hw_q_t *q_next;
 796
 797                 ocs_list_foreach_safe(&cq->q_list, q, q_next) {
 798                         switch(q->type) {
 799                         case SLI_QTYPE_MQ:
 800                                 hw_del_mq((hw_mq_t*) q);
 801                                 break;
 802                         case SLI_QTYPE_WQ:
 803                                 hw_del_wq((hw_wq_t*) q);
 804                                 break;
 805                         case SLI_QTYPE_RQ:
 806                                 hw_del_rq((hw_rq_t*) q);
 807                                 break;
 808                         default:
 809                                 break;
 810                         }
 811                 }
 812                 ocs_list_remove(&cq->eq->cq_list, cq);
 813                 cq->eq->hw->hw_cq[cq->instance] = NULL;
 814                 ocs_free(cq->eq->hw->os, cq, sizeof(*cq));
 815         }
 816 }
 817
 818 /**
 819  * @brief Free a MQ object
 820  *
 821  * The MQ object is freed
 822  *
 823  * @param mq pointer to MQ object
 824  *
 825  * @return none
 826  */
 827 void
 828 hw_del_mq(hw_mq_t *mq)
 829 {
 830         if (mq != NULL) {
 831                 ocs_list_remove(&mq->cq->q_list, mq);
 832                 mq->cq->eq->hw->hw_mq[mq->instance] = NULL;
 833                 ocs_free(mq->cq->eq->hw->os, mq, sizeof(*mq));
 834         }
 835 }
 836
 837 /**
 838  * @brief Free a WQ object
 839  *
 840  * The WQ object is freed
 841  *
 842  * @param wq pointer to WQ object
 843  *
 844  * @return none
 845  */
 846 void
 847 hw_del_wq(hw_wq_t *wq)
 848 {
 849         if (wq != NULL) {
 850                 ocs_list_remove(&wq->cq->q_list, wq);
 851                 wq->cq->eq->hw->hw_wq[wq->instance] = NULL;
 852                 ocs_free(wq->cq->eq->hw->os, wq, sizeof(*wq));
 853         }
 854 }
 855
 856 /**
 857  * @brief Free an RQ object
 858  *
 859  * The RQ object is freed
 860  *
 861  * @param rq pointer to RQ object
 862  *
 863  * @return none
 864  */
 865 void
 866 hw_del_rq(hw_rq_t *rq)
 867 {
 868
 869         if (rq != NULL) {
 870                 ocs_hw_t *hw = rq->cq->eq->hw;
 871                 /* Free RQ tracker */
 872                 if (rq->rq_tracker != NULL) {
 873                         ocs_free(hw->os, rq->rq_tracker, sizeof(ocs_hw_sequence_t*) * rq->entry_count);
 874                         rq->rq_tracker = NULL;
 875                 }
 876                 ocs_list_remove(&rq->cq->q_list, rq);
 877                 hw->hw_rq[rq->instance] = NULL;
 878                 ocs_free(hw->os, rq, sizeof(*rq));
 879         }
 880 }
 881
 882 /**
 883  * @brief Display HW queue objects
 884  *
 885  * The HW queue objects are displayed using ocs_log
 886  *
 887  * @param hw pointer to HW object
 888  *
 889  * @return none
 890  */
 891 void
 892 hw_queue_dump(ocs_hw_t *hw)
 893 {
 894         hw_eq_t *eq;
 895         hw_cq_t *cq;
 896         hw_q_t *q;
 897         hw_mq_t *mq;
 898         hw_wq_t *wq;
 899         hw_rq_t *rq;
 900
 901         ocs_list_foreach(&hw->eq_list, eq) {
 902                 ocs_printf("eq[%d] id %2d\n", eq->instance, eq->queue->id);
 903                 ocs_list_foreach(&eq->cq_list, cq) {
 904                         ocs_printf("  cq[%d] id %2d current\n", cq->instance, cq->queue->id);
 905                         ocs_list_foreach(&cq->q_list, q) {
 906                                 switch(q->type) {
 907                                 case SLI_QTYPE_MQ:
 908                                         mq = (hw_mq_t *) q;
 909                                         ocs_printf("    mq[%d] id %2d\n", mq->instance, mq->queue->id);
 910                                         break;
 911                                 case SLI_QTYPE_WQ:
 912                                         wq = (hw_wq_t *) q;
 913                                         ocs_printf("    wq[%d] id %2d\n", wq->instance, wq->queue->id);
 914                                         break;
 915                                 case SLI_QTYPE_RQ:
 916                                         rq = (hw_rq_t *) q;
 917                                         ocs_printf("    rq[%d] hdr id %2d\n", rq->instance, rq->hdr->id);
 918                                         break;
 919                                 default:
 920                                         break;
 921                                 }
 922                         }
 923                 }
 924         }
 925 }
 926
 927 /**
 928  * @brief Teardown HW queue objects
 929  *
 930  * The HW queue objects are freed
 931  *
 932  * @param hw pointer to HW object
 933  *
 934  * @return none
 935  */
 936 void
 937 hw_queue_teardown(ocs_hw_t *hw)
 938 {
 939         uint32_t i;
 940         hw_eq_t *eq;
 941         hw_eq_t *eq_next;
 942
 943         if (ocs_list_valid(&hw->eq_list)) {
 944                 ocs_list_foreach_safe(&hw->eq_list, eq, eq_next) {
 945                         hw_del_eq(eq);
 946                 }
 947         }
 948         for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) {
 949                 ocs_varray_free(hw->wq_cpu_array[i]);
 950                 hw->wq_cpu_array[i] = NULL;
 951         }
 952         for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) {
 953                 ocs_varray_free(hw->wq_class_array[i]);
 954                 hw->wq_class_array[i] = NULL;
 955         }
 956 }
 957
 958 /**
 959  * @brief Allocate a WQ to an IO object
 960  *
 961  * The next work queue index is used to assign a WQ to an IO.
 962  *
 963  * If wq_steering is OCS_HW_WQ_STEERING_CLASS, a WQ from io->wq_class is
 964  * selected.
 965  *
 966  * If wq_steering is OCS_HW_WQ_STEERING_REQUEST, then a WQ from the EQ that
 967  * the IO request came in on is selected.
 968  *
 969  * If wq_steering is OCS_HW_WQ_STEERING_CPU, then a WQ associted with the
 970  * CPU the request is made on is selected.
 971  *
 972  * @param hw pointer to HW object
 973  * @param io pointer to IO object
 974  *
 975  * @return Return pointer to next WQ
 976  */
 977 hw_wq_t *
 978 ocs_hw_queue_next_wq(ocs_hw_t *hw, ocs_hw_io_t *io)
 979 {
 980         hw_eq_t *eq;
 981         hw_wq_t *wq = NULL;
 982
 983         switch(io->wq_steering) {
 984         case OCS_HW_WQ_STEERING_CLASS:
 985                 if (likely(io->wq_class < ARRAY_SIZE(hw->wq_class_array))) {
 986                         wq = ocs_varray_iter_next(hw->wq_class_array[io->wq_class]);
 987                 }
 988                 break;
 989         case OCS_HW_WQ_STEERING_REQUEST:
 990                 eq = io->eq;
 991                 if (likely(eq != NULL)) {
 992                         wq = ocs_varray_iter_next(eq->wq_array);
 993                 }
 994                 break;
 995         case OCS_HW_WQ_STEERING_CPU: {
 996                 uint32_t cpuidx = ocs_thread_getcpu();
 997
 998                 if (likely(cpuidx < ARRAY_SIZE(hw->wq_cpu_array))) {
 999                         wq = ocs_varray_iter_next(hw->wq_cpu_array[cpuidx]);
1000                 }
1001                 break;
1002         }
1003         }
1004
1005         if (unlikely(wq == NULL)) {
1006                 wq = hw->hw_wq[0];
1007         }
1008
1009         return wq;
1010 }
1011
1012 /**
1013  * @brief Return count of EQs for a queue topology object
1014  *
1015  * The EQ count for in the HWs queue topology (hw->qtop) object is returned
1016  *
1017  * @param hw pointer to HW object
1018  *
1019  * @return count of EQs
1020  */
1021 uint32_t
1022 ocs_hw_qtop_eq_count(ocs_hw_t *hw)
1023 {
1024         return hw->qtop->entry_counts[QTOP_EQ];
1025 }
1026
1027 #define TOKEN_LEN               32
1028
1029 /**
1030  * @brief return string given a QTOP entry
1031  *
1032  * @param entry QTOP entry
1033  *
1034  * @return returns string or "unknown"
1035  */
1036 #if HW_QTOP_DEBUG
1037 static char *
1038 qtopentry2s(ocs_hw_qtop_entry_e entry) {
1039         switch(entry) {
1040         #define P(x)    case x: return #x;
1041         P(QTOP_EQ)
1042         P(QTOP_CQ)
1043         P(QTOP_WQ)
1044         P(QTOP_RQ)
1045         P(QTOP_MQ)
1046         P(QTOP_THREAD_START)
1047         P(QTOP_THREAD_END)
1048         P(QTOP_LAST)
1049         #undef P
1050         }
1051         return "unknown";
1052 }
1053 #endif
1054
1055 /**
1056  * @brief Declare token types
1057  */
1058 typedef enum {
1059         TOK_LPAREN = 1,
1060         TOK_RPAREN,
1061         TOK_COLON,
1062         TOK_EQUALS,
1063         TOK_QUEUE,
1064         TOK_ATTR_NAME,
1065         TOK_NUMBER,
1066         TOK_NUMBER_VALUE,
1067         TOK_NUMBER_LIST,
1068 } tok_type_e;
1069
1070 /**
1071  * @brief Declare token sub-types
1072  */
1073 typedef enum {
1074         TOK_SUB_EQ = 100,
1075         TOK_SUB_CQ,
1076         TOK_SUB_RQ,
1077         TOK_SUB_MQ,
1078         TOK_SUB_WQ,
1079         TOK_SUB_LEN,
1080         TOK_SUB_CLASS,
1081         TOK_SUB_ULP,
1082         TOK_SUB_FILTER,
1083 } tok_subtype_e;
1084
1085 /**
1086  * @brief convert queue subtype to QTOP entry
1087  *
1088  * @param q queue subtype
1089  *
1090  * @return QTOP entry or 0
1091  */
1092 static ocs_hw_qtop_entry_e
1093 subtype2qtop(tok_subtype_e q)
1094 {
1095         switch(q) {
1096         case TOK_SUB_EQ:        return QTOP_EQ;
1097         case TOK_SUB_CQ:        return QTOP_CQ;
1098         case TOK_SUB_RQ:        return QTOP_RQ;
1099         case TOK_SUB_MQ:        return QTOP_MQ;
1100         case TOK_SUB_WQ:        return QTOP_WQ;
1101         default:
1102                 break;
1103         }
1104         return 0;
1105 }
1106
1107 /**
1108  * @brief Declare token object
1109  */
1110 typedef struct {
1111         tok_type_e type;
1112         tok_subtype_e subtype;
1113         char string[TOKEN_LEN];
1114 } tok_t;
1115
1116 /**
1117  * @brief Declare token array object
1118  */
1119 typedef struct {
1120         tok_t *tokens;                  /* Pointer to array of tokens */
1121         uint32_t alloc_count;           /* Number of tokens in the array */
1122         uint32_t inuse_count;           /* Number of tokens posted to array */
1123         uint32_t iter_idx;              /* Iterator index */
1124 } tokarray_t;
1125
1126 /**
1127  * @brief Declare token match structure
1128  */
1129 typedef struct {
1130         char *s;
1131         tok_type_e type;
1132         tok_subtype_e subtype;
1133 } tokmatch_t;
1134
1135 /**
1136  * @brief test if character is ID start character
1137  *
1138  * @param c character to test
1139  *
1140  * @return TRUE if character is an ID start character
1141  */
1142 static int32_t
1143 idstart(int c)
1144 {
1145         return  isalpha(c) || (c == '_') || (c == '$');
1146 }
1147
1148 /**
1149  * @brief test if character is an ID character
1150  *
1151  * @param c character to test
1152  *
1153  * @return TRUE if character is an ID character
1154  */
1155 static int32_t
1156 idchar(int c)
1157 {
1158         return idstart(c) || ocs_isdigit(c);
1159 }
1160
1161 /**
1162  * @brief Declare single character matches
1163  */
1164 static tokmatch_t cmatches[] = {
1165         {"(", TOK_LPAREN},
1166         {")", TOK_RPAREN},
1167         {":", TOK_COLON},
1168         {"=", TOK_EQUALS},
1169 };
1170
1171 /**
1172  * @brief Declare identifier match strings
1173  */
1174 static tokmatch_t smatches[] = {
1175         {"eq", TOK_QUEUE, TOK_SUB_EQ},
1176         {"cq", TOK_QUEUE, TOK_SUB_CQ},
1177         {"rq", TOK_QUEUE, TOK_SUB_RQ},
1178         {"mq", TOK_QUEUE, TOK_SUB_MQ},
1179         {"wq", TOK_QUEUE, TOK_SUB_WQ},
1180         {"len", TOK_ATTR_NAME, TOK_SUB_LEN},
1181         {"class", TOK_ATTR_NAME, TOK_SUB_CLASS},
1182         {"ulp", TOK_ATTR_NAME, TOK_SUB_ULP},
1183         {"filter", TOK_ATTR_NAME, TOK_SUB_FILTER},
1184 };
1185
1186 /**
1187  * @brief Scan string and return next token
1188  *
1189  * The string is scanned and the next token is returned
1190  *
1191  * @param s input string to scan
1192  * @param tok pointer to place scanned token
1193  *
1194  * @return pointer to input string following scanned token, or NULL
1195  */
1196 static const char *
1197 tokenize(const char *s, tok_t *tok)
1198 {
1199         uint32_t i;
1200
1201         memset(tok, 0, sizeof(*tok));
1202
1203         /* Skip over whitespace */
1204         while (*s && ocs_isspace(*s)) {
1205                 s++;
1206         }
1207
1208         /* Return if nothing left in this string */
1209         if (*s == 0) {
1210                 return NULL;
1211         }
1212
1213         /* Look for single character matches */
1214         for (i = 0; i < ARRAY_SIZE(cmatches); i++) {
1215                 if (cmatches[i].s[0] == *s) {
1216                         tok->type = cmatches[i].type;
1217                         tok->subtype = cmatches[i].subtype;
1218                         tok->string[0] = *s++;
1219                         return s;
1220                 }
1221         }
1222
1223         /* Scan for a hex number or decimal */
1224         if ((s[0] == '0') && ((s[1] == 'x') || (s[1] == 'X'))) {
1225                 char *p = tok->string;
1226
1227                 tok->type = TOK_NUMBER;
1228
1229                 *p++ = *s++;
1230                 *p++ = *s++;
1231                 while ((*s == '.') || ocs_isxdigit(*s)) {
1232                         if ((p - tok->string) < (int32_t)sizeof(tok->string)) {
1233                                 *p++ = *s;
1234                         }
1235                         if (*s == ',') {
1236                                 tok->type = TOK_NUMBER_LIST;
1237                         }
1238                         s++;
1239                 }
1240                 *p = 0;
1241                 return s;
1242         } else if (ocs_isdigit(*s)) {
1243                 char *p = tok->string;
1244
1245                 tok->type = TOK_NUMBER;
1246                 while ((*s == ',') || ocs_isdigit(*s)) {
1247                         if ((p - tok->string) < (int32_t)sizeof(tok->string)) {
1248                                 *p++ = *s;
1249                         }
1250                         if (*s == ',') {
1251                                 tok->type = TOK_NUMBER_LIST;
1252                         }
1253                         s++;
1254                 }
1255                 *p = 0;
1256                 return s;
1257         }
1258
1259         /* Scan for an ID */
1260         if (idstart(*s)) {
1261                 char *p = tok->string;
1262
1263                 for (*p++ = *s++; idchar(*s); s++) {
1264                         if ((p - tok->string) < TOKEN_LEN) {
1265                                 *p++ = *s;
1266                         }
1267                 }
1268
1269                 /* See if this is a $ number value */
1270                 if (tok->string[0] == '$') {
1271                         tok->type = TOK_NUMBER_VALUE;
1272                 } else {
1273                         /* Look for a string match */
1274                         for (i = 0; i < ARRAY_SIZE(smatches); i++) {
1275                                 if (strcmp(smatches[i].s, tok->string) == 0) {
1276                                         tok->type = smatches[i].type;
1277                                         tok->subtype = smatches[i].subtype;
1278                                         return s;
1279                                 }
1280                         }
1281                 }
1282         }
1283         return s;
1284 }
1285
1286 /**
1287  * @brief convert token type to string
1288  *
1289  * @param type token type
1290  *
1291  * @return string, or "unknown"
1292  */
1293 static const char *
1294 token_type2s(tok_type_e type)
1295 {
1296         switch(type) {
1297         #define P(x)    case x: return #x;
1298         P(TOK_LPAREN)
1299         P(TOK_RPAREN)
1300         P(TOK_COLON)
1301         P(TOK_EQUALS)
1302         P(TOK_QUEUE)
1303         P(TOK_ATTR_NAME)
1304         P(TOK_NUMBER)
1305         P(TOK_NUMBER_VALUE)
1306         P(TOK_NUMBER_LIST)
1307         #undef P
1308         }
1309         return "unknown";
1310 }
1311
1312 /**
1313  * @brief convert token sub-type to string
1314  *
1315  * @param subtype token sub-type
1316  *
1317  * @return string, or "unknown"
1318  */
1319 static const char *
1320 token_subtype2s(tok_subtype_e subtype)
1321 {
1322         switch(subtype) {
1323         #define P(x)    case x: return #x;
1324         P(TOK_SUB_EQ)
1325         P(TOK_SUB_CQ)
1326         P(TOK_SUB_RQ)
1327         P(TOK_SUB_MQ)
1328         P(TOK_SUB_WQ)
1329         P(TOK_SUB_LEN)
1330         P(TOK_SUB_CLASS)
1331         P(TOK_SUB_ULP)
1332         P(TOK_SUB_FILTER)
1333         #undef P
1334         }
1335         return "";
1336 }
1337
1338 /**
1339  * @brief Generate syntax error message
1340  *
1341  * A syntax error message is found, the input tokens are dumped up to and including
1342  * the token that failed as indicated by the current iterator index.
1343  *
1344  * @param hw pointer to HW object
1345  * @param tokarray pointer to token array object
1346  *
1347  * @return none
1348  */
1349 static void
1350 tok_syntax(ocs_hw_t *hw, tokarray_t *tokarray)
1351 {
1352         uint32_t i;
1353         tok_t *tok;
1354
1355         ocs_log_test(hw->os, "Syntax error:\n");
1356
1357         for (i = 0, tok = tokarray->tokens; (i <= tokarray->inuse_count); i++, tok++) {
1358                 ocs_log_test(hw->os, "%s [%2d]    %-16s %-16s %s\n", (i == tokarray->iter_idx) ? ">>>" : "   ", i,
1359                         token_type2s(tok->type), token_subtype2s(tok->subtype), tok->string);
1360         }
1361 }
1362
1363 /**
1364  * @brief parse a number
1365  *
1366  * Parses tokens of type TOK_NUMBER and TOK_NUMBER_VALUE, returning a numeric value
1367  *
1368  * @param hw pointer to HW object
1369  * @param qtop pointer to QTOP object
1370  * @param tok pointer to token to parse
1371  *
1372  * @return numeric value
1373  */
1374 static uint32_t
1375 tok_getnumber(ocs_hw_t *hw, ocs_hw_qtop_t *qtop, tok_t *tok)
1376 {
1377         uint32_t rval = 0;
1378         uint32_t num_cpus = ocs_get_num_cpus();
1379
1380         switch(tok->type) {
1381         case TOK_NUMBER_VALUE:
1382                 if (ocs_strcmp(tok->string, "$ncpu") == 0) {
1383                         rval = num_cpus;
1384                 } else if (ocs_strcmp(tok->string, "$ncpu1") == 0) {
1385                         rval = num_cpus - 1;
1386                 } else if (ocs_strcmp(tok->string, "$nwq") == 0) {
1387                         if (hw != NULL) {
1388                                 rval = hw->config.n_wq;
1389                         }
1390                 } else if (ocs_strcmp(tok->string, "$maxmrq") == 0) {
1391                         rval = MIN(num_cpus, OCS_HW_MAX_MRQS);
1392                 } else if (ocs_strcmp(tok->string, "$nulp") == 0) {
1393                         rval = hw->ulp_max - hw->ulp_start + 1;
1394                 } else if ((qtop->rptcount_idx > 0) && ocs_strcmp(tok->string, "$rpt0") == 0) {
1395                         rval = qtop->rptcount[qtop->rptcount_idx-1];
1396                 } else if ((qtop->rptcount_idx > 1) && ocs_strcmp(tok->string, "$rpt1") == 0) {
1397                         rval = qtop->rptcount[qtop->rptcount_idx-2];
1398                 } else if ((qtop->rptcount_idx > 2) && ocs_strcmp(tok->string, "$rpt2") == 0) {
1399                         rval = qtop->rptcount[qtop->rptcount_idx-3];
1400                 } else if ((qtop->rptcount_idx > 3) && ocs_strcmp(tok->string, "$rpt3") == 0) {
1401                         rval = qtop->rptcount[qtop->rptcount_idx-4];
1402                 } else {
1403                         rval = ocs_strtoul(tok->string, 0, 0);
1404                 }
1405                 break;
1406         case TOK_NUMBER:
1407                 rval = ocs_strtoul(tok->string, 0, 0);
1408                 break;
1409         default:
1410                 break;
1411         }
1412         return rval;
1413 }
1414
1415 /**
1416  * @brief parse an array of tokens
1417  *
1418  * The tokens are semantically parsed, to generate QTOP entries.
1419  *
1420  * @param hw pointer to HW object
1421  * @param tokarray array array of tokens
1422  * @param qtop ouptut QTOP object
1423  *
1424  * @return returns 0 for success, a negative error code value for failure.
1425  */
1426 static int32_t
1427 parse_topology(ocs_hw_t *hw, tokarray_t *tokarray, ocs_hw_qtop_t *qtop)
1428 {
1429         ocs_hw_qtop_entry_t *qt = qtop->entries + qtop->inuse_count;
1430         tok_t *tok;
1431
1432         for (; (tokarray->iter_idx < tokarray->inuse_count) &&
1433              ((tok = &tokarray->tokens[tokarray->iter_idx]) != NULL); ) {
1434                 if (qtop->inuse_count >= qtop->alloc_count) {
1435                         return -1;
1436                 }
1437
1438                 qt = qtop->entries + qtop->inuse_count;
1439
1440                 switch (tok[0].type)
1441                 {
1442                 case TOK_QUEUE:
1443                         qt->entry = subtype2qtop(tok[0].subtype);
1444                         qt->set_default = FALSE;
1445                         qt->len = 0;
1446                         qt->class = 0;
1447                         qtop->inuse_count++;
1448
1449                         tokarray->iter_idx++;           /* Advance current token index */
1450
1451                         /* Parse for queue attributes, possibly multiple instances */
1452                         while ((tokarray->iter_idx + 4) <= tokarray->inuse_count) {
1453                                 tok = &tokarray->tokens[tokarray->iter_idx];
1454                                 if(     (tok[0].type == TOK_COLON) &&
1455                                         (tok[1].type == TOK_ATTR_NAME) &&
1456                                         (tok[2].type == TOK_EQUALS) &&
1457                                         ((tok[3].type == TOK_NUMBER) ||
1458                                          (tok[3].type == TOK_NUMBER_VALUE) ||
1459                                          (tok[3].type == TOK_NUMBER_LIST))) {
1460                                         switch (tok[1].subtype) {
1461                                         case TOK_SUB_LEN:
1462                                                 qt->len = tok_getnumber(hw, qtop, &tok[3]);
1463                                                 break;
1464
1465                                         case TOK_SUB_CLASS:
1466                                                 qt->class = tok_getnumber(hw, qtop, &tok[3]);
1467                                                 break;
1468
1469                                         case TOK_SUB_ULP:
1470                                                 qt->ulp = tok_getnumber(hw, qtop, &tok[3]);
1471                                                 break;
1472
1473                                         case TOK_SUB_FILTER:
1474                                                 if (tok[3].type == TOK_NUMBER_LIST) {
1475                                                         uint32_t mask = 0;
1476                                                         char *p = tok[3].string;
1477
1478                                                         while ((p != NULL) && *p) {
1479                                                                 uint32_t v;
1480
1481                                                                 v = ocs_strtoul(p, 0, 0);
1482                                                                 if (v < 32) {
1483                                                                         mask |= (1U << v);
1484                                                                 }
1485
1486                                                                 p = ocs_strchr(p, ',');
1487                                                                 if (p != NULL) {
1488                                                                         p++;
1489                                                                 }
1490                                                         }
1491                                                         qt->filter_mask = mask;
1492                                                 } else {
1493                                                         qt->filter_mask = (1U << tok_getnumber(hw, qtop, &tok[3]));
1494                                                 }
1495                                                 break;
1496                                         default:
1497                                                 break;
1498                                         }
1499                                         /* Advance current token index */
1500                                         tokarray->iter_idx += 4;
1501                                 } else {
1502                                         break;
1503                                 }
1504                         }
1505                         qtop->entry_counts[qt->entry]++;
1506                         break;
1507
1508                 case TOK_ATTR_NAME:
1509                         if (    ((tokarray->iter_idx + 5) <= tokarray->inuse_count) &&
1510                                 (tok[1].type == TOK_COLON) &&
1511                                 (tok[2].type == TOK_QUEUE) &&
1512                                 (tok[3].type == TOK_EQUALS) &&
1513                                 ((tok[4].type == TOK_NUMBER) || (tok[4].type == TOK_NUMBER_VALUE))) {
1514                                 qt->entry = subtype2qtop(tok[2].subtype);
1515                                 qt->set_default = TRUE;
1516                                 switch(tok[0].subtype) {
1517                                 case TOK_SUB_LEN:
1518                                         qt->len = tok_getnumber(hw, qtop, &tok[4]);
1519                                         break;
1520                                 case TOK_SUB_CLASS:
1521                                         qt->class = tok_getnumber(hw, qtop, &tok[4]);
1522                                         break;
1523                                 case TOK_SUB_ULP:
1524                                         qt->ulp = tok_getnumber(hw, qtop, &tok[4]);
1525                                         break;
1526                                 default:
1527                                         break;
1528                                 }
1529                                 qtop->inuse_count++;
1530                                 tokarray->iter_idx += 5;
1531                         } else {
1532                                 tok_syntax(hw, tokarray);
1533                                 return -1;
1534                         }
1535                         break;
1536
1537                 case TOK_NUMBER:
1538                 case TOK_NUMBER_VALUE: {
1539                         uint32_t rpt_count = 1;
1540                         uint32_t i;
1541
1542                         rpt_count = tok_getnumber(hw, qtop, tok);
1543
1544                         if (tok[1].type == TOK_LPAREN) {
1545                                 uint32_t iter_idx_save;
1546
1547                                 tokarray->iter_idx += 2;
1548
1549                                 /* save token array iteration index */
1550                                 iter_idx_save = tokarray->iter_idx;
1551
1552                                 for (i = 0; i < rpt_count; i++) {
1553                                         uint32_t rptcount_idx = qtop->rptcount_idx;
1554
1555                                         if (qtop->rptcount_idx < ARRAY_SIZE(qtop->rptcount)) {
1556                                                 qtop->rptcount[qtop->rptcount_idx++] = i;
1557                                         }
1558
1559                                         /* restore token array iteration index */
1560                                         tokarray->iter_idx = iter_idx_save;
1561
1562                                         /* parse, append to qtop */
1563                                         parse_topology(hw, tokarray, qtop);
1564
1565                                         qtop->rptcount_idx = rptcount_idx;
1566                                 }
1567                         }
1568                         break;
1569                 }
1570
1571                 case TOK_RPAREN:
1572                         tokarray->iter_idx++;
1573                         return 0;
1574
1575                 default:
1576                         tok_syntax(hw, tokarray);
1577                         return -1;
1578                 }
1579         }
1580         return 0;
1581 }
1582
1583 /**
1584  * @brief Parse queue topology string
1585  *
1586  * The queue topology object is allocated, and filled with the results of parsing the
1587  * passed in queue topology string
1588  *
1589  * @param hw pointer to HW object
1590  * @param qtop_string input queue topology string
1591  *
1592  * @return pointer to allocated QTOP object, or NULL if there was an error
1593  */
1594 ocs_hw_qtop_t *
1595 ocs_hw_qtop_parse(ocs_hw_t *hw, const char *qtop_string)
1596 {
1597         ocs_hw_qtop_t *qtop;
1598         tokarray_t tokarray;
1599         const char *s;
1600 #if HW_QTOP_DEBUG
1601         uint32_t i;
1602         ocs_hw_qtop_entry_t *qt;
1603 #endif
1604
1605         ocs_log_debug(hw->os, "queue topology: %s\n", qtop_string);
1606
1607         /* Allocate a token array */
1608         tokarray.tokens = ocs_malloc(hw->os, MAX_TOKENS * sizeof(*tokarray.tokens), OCS_M_ZERO | OCS_M_NOWAIT);
1609         if (tokarray.tokens == NULL) {
1610                 return NULL;
1611         }
1612         tokarray.alloc_count = MAX_TOKENS;
1613         tokarray.inuse_count = 0;
1614         tokarray.iter_idx = 0;
1615
1616         /* Parse the tokens */
1617         for (s = qtop_string; (tokarray.inuse_count < tokarray.alloc_count) &&
1618              ((s = tokenize(s, &tokarray.tokens[tokarray.inuse_count]))) != NULL; ) {
1619                 tokarray.inuse_count++;
1620         }
1621
1622         /* Allocate a queue topology structure */
1623         qtop = ocs_malloc(hw->os, sizeof(*qtop), OCS_M_ZERO | OCS_M_NOWAIT);
1624         if (qtop == NULL) {
1625                 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens));
1626                 ocs_log_err(hw->os, "malloc qtop failed\n");
1627                 return NULL;
1628         }
1629         qtop->os = hw->os;
1630
1631         /* Allocate queue topology entries */
1632         qtop->entries = ocs_malloc(hw->os, OCS_HW_MAX_QTOP_ENTRIES*sizeof(*qtop->entries), OCS_M_ZERO | OCS_M_NOWAIT);
1633         if (qtop->entries == NULL) {
1634                 ocs_log_err(hw->os, "malloc qtop entries failed\n");
1635                 ocs_free(hw->os, qtop, sizeof(*qtop));
1636                 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens));
1637                 return NULL;
1638         }
1639         qtop->alloc_count = OCS_HW_MAX_QTOP_ENTRIES;
1640         qtop->inuse_count = 0;
1641
1642         /* Parse the tokens */
1643         parse_topology(hw, &tokarray, qtop);
1644 #if HW_QTOP_DEBUG
1645         for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) {
1646                 ocs_log_debug(hw->os, "entry %s set_df %d len %4d class %d ulp %d\n", qtopentry2s(qt->entry), qt->set_default, qt->len,
1647                        qt->class, qt->ulp);
1648         }
1649 #endif
1650
1651         /* Free the tokens array */
1652         ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens));
1653
1654         return qtop;
1655 }
1656
1657 /**
1658  * @brief free queue topology object
1659  *
1660  * @param qtop pointer to QTOP object
1661  *
1662  * @return none
1663  */
1664 void
1665 ocs_hw_qtop_free(ocs_hw_qtop_t *qtop)
1666 {
1667         if (qtop != NULL) {
1668                 if (qtop->entries != NULL) {
1669                         ocs_free(qtop->os, qtop->entries, qtop->alloc_count*sizeof(*qtop->entries));
1670                 }
1671                 ocs_free(qtop->os, qtop, sizeof(*qtop));
1672         }
1673 }
1674
1675 /* Uncomment this to turn on RQ debug */
1676 // #define ENABLE_DEBUG_RQBUF
1677
1678 static int32_t ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id);
1679 static ocs_hw_sequence_t * ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex);
1680 static int32_t ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq);
1681 static ocs_hw_rtn_e ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq);
1682
1683 /**
1684  * @brief Process receive queue completions for RQ Pair mode.
1685  *
1686  * @par Description
1687  * RQ completions are processed. In RQ pair mode, a single header and single payload
1688  * buffer are received, and passed to the function that has registered for unsolicited
1689  * callbacks.
1690  *
1691  * @param hw Hardware context.
1692  * @param cq Pointer to HW completion queue.
1693  * @param cqe Completion queue entry.
1694  *
1695  * @return Returns 0 for success, or a negative error code value for failure.
1696  */
1697
1698 int32_t
1699 ocs_hw_rqpair_process_rq(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe)
1700 {
1701         uint16_t rq_id;
1702         uint32_t index;
1703         int32_t rqindex;
1704         int32_t  rq_status;
1705         uint32_t h_len;
1706         uint32_t p_len;
1707         ocs_hw_sequence_t *seq;
1708
1709         rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index);
1710         if (0 != rq_status) {
1711                 switch (rq_status) {
1712                 case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED:
1713                 case SLI4_FC_ASYNC_RQ_DMA_FAILURE:
1714                         /* just get RQ buffer then return to chip */
1715                         rqindex = ocs_hw_rqpair_find(hw, rq_id);
1716                         if (rqindex < 0) {
1717                                 ocs_log_test(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n",
1718                                              rq_status, rq_id);
1719                                 break;
1720                         }
1721
1722                         /* get RQ buffer */
1723                         seq = ocs_hw_rqpair_get(hw, rqindex, index);
1724
1725                         /* return to chip */
1726                         if (ocs_hw_rqpair_sequence_free(hw, seq)) {
1727                                 ocs_log_test(hw->os, "status=%#x, failed to return buffers to RQ\n",
1728                                              rq_status);
1729                                 break;
1730                         }
1731                         break;
1732                 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED:
1733                 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC:
1734                         /* since RQ buffers were not consumed, cannot return them to chip */
1735                         /* fall through */
1736                         ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status);
1737                 default:
1738                         break;
1739                 }
1740                 return -1;
1741         }
1742
1743         rqindex = ocs_hw_rqpair_find(hw, rq_id);
1744         if (rqindex < 0) {
1745                 ocs_log_test(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id);
1746                 return -1;
1747         }
1748
1749         OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++;
1750                  rq->payload_use_count++;})
1751
1752         seq = ocs_hw_rqpair_get(hw, rqindex, index);
1753         ocs_hw_assert(seq != NULL);
1754
1755         seq->hw = hw;
1756         seq->auto_xrdy = 0;
1757         seq->out_of_xris = 0;
1758         seq->xri = 0;
1759         seq->hio = NULL;
1760
1761         sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len);
1762         seq->header->dma.len = h_len;
1763         seq->payload->dma.len = p_len;
1764         seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe);
1765         seq->hw_priv = cq->eq;
1766
1767         /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */
1768         if (hw->config.bounce) {
1769                 fc_header_t *hdr = seq->header->dma.virt;
1770                 uint32_t s_id = fc_be24toh(hdr->s_id);
1771                 uint32_t d_id = fc_be24toh(hdr->d_id);
1772                 uint32_t ox_id =  ocs_be16toh(hdr->ox_id);
1773                 if (hw->callback.bounce != NULL) {
1774                         (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id);
1775                 }
1776         } else {
1777                 hw->callback.unsolicited(hw->args.unsolicited, seq);
1778         }
1779
1780         return 0;
1781 }
1782
1783 /**
1784  * @brief Process receive queue completions for RQ Pair mode - Auto xfer rdy
1785  *
1786  * @par Description
1787  * RQ completions are processed. In RQ pair mode, a single header and single payload
1788  * buffer are received, and passed to the function that has registered for unsolicited
1789  * callbacks.
1790  *
1791  * @param hw Hardware context.
1792  * @param cq Pointer to HW completion queue.
1793  * @param cqe Completion queue entry.
1794  *
1795  * @return Returns 0 for success, or a negative error code value for failure.
1796  */
1797
1798 int32_t
1799 ocs_hw_rqpair_process_auto_xfr_rdy_cmd(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe)
1800 {
1801         /* Seems silly to call a SLI function to decode - use the structure directly for performance */
1802         sli4_fc_optimized_write_cmd_cqe_t *opt_wr = (sli4_fc_optimized_write_cmd_cqe_t*)cqe;
1803         uint16_t rq_id;
1804         uint32_t index;
1805         int32_t rqindex;
1806         int32_t  rq_status;
1807         uint32_t h_len;
1808         uint32_t p_len;
1809         ocs_hw_sequence_t *seq;
1810         uint8_t axr_lock_taken = 0;
1811 #if defined(OCS_DISC_SPIN_DELAY)
1812         uint32_t        delay = 0;
1813         char            prop_buf[32];
1814 #endif
1815
1816         rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index);
1817         if (0 != rq_status) {
1818                 switch (rq_status) {
1819                 case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED:
1820                 case SLI4_FC_ASYNC_RQ_DMA_FAILURE:
1821                         /* just get RQ buffer then return to chip */
1822                         rqindex = ocs_hw_rqpair_find(hw, rq_id);
1823                         if (rqindex < 0) {
1824                                 ocs_log_err(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n",
1825                                             rq_status, rq_id);
1826                                 break;
1827                         }
1828
1829                         /* get RQ buffer */
1830                         seq = ocs_hw_rqpair_get(hw, rqindex, index);
1831
1832                         /* return to chip */
1833                         if (ocs_hw_rqpair_sequence_free(hw, seq)) {
1834                                 ocs_log_err(hw->os, "status=%#x, failed to return buffers to RQ\n",
1835                                             rq_status);
1836                                 break;
1837                         }
1838                         break;
1839                 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED:
1840                 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC:
1841                         /* since RQ buffers were not consumed, cannot return them to chip */
1842                         ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status);
1843                         /* fall through */
1844                 default:
1845                         break;
1846                 }
1847                 return -1;
1848         }
1849
1850         rqindex = ocs_hw_rqpair_find(hw, rq_id);
1851         if (rqindex < 0) {
1852                 ocs_log_err(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id);
1853                 return -1;
1854         }
1855
1856         OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++;
1857                  rq->payload_use_count++;})
1858
1859         seq = ocs_hw_rqpair_get(hw, rqindex, index);
1860         ocs_hw_assert(seq != NULL);
1861
1862         seq->hw = hw;
1863         seq->auto_xrdy = opt_wr->agxr;
1864         seq->out_of_xris = opt_wr->oox;
1865         seq->xri = opt_wr->xri;
1866         seq->hio = NULL;
1867
1868         sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len);
1869         seq->header->dma.len = h_len;
1870         seq->payload->dma.len = p_len;
1871         seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe);
1872         seq->hw_priv = cq->eq;
1873
1874         if (seq->auto_xrdy) {
1875                 fc_header_t *fc_hdr = seq->header->dma.virt;
1876
1877                 seq->hio = ocs_hw_io_lookup(hw, seq->xri);
1878                 ocs_lock(&seq->hio->axr_lock);
1879                 axr_lock_taken = 1;
1880
1881                 /* save the FCFI, src_id, dest_id and ox_id because we need it for the sequence object when the data comes. */
1882                 seq->hio->axr_buf->fcfi = seq->fcfi;
1883                 seq->hio->axr_buf->hdr.ox_id = fc_hdr->ox_id;
1884                 seq->hio->axr_buf->hdr.s_id = fc_hdr->s_id;
1885                 seq->hio->axr_buf->hdr.d_id = fc_hdr->d_id;
1886                 seq->hio->axr_buf->cmd_cqe = 1;
1887
1888                 /*
1889                  * Since auto xfer rdy is used for this IO, then clear the sequence
1890                  * initiative bit in the header so that the upper layers wait for the
1891                  * data. This should flow exactly like the first burst case.
1892                  */
1893                 fc_hdr->f_ctl &= fc_htobe24(~FC_FCTL_SEQUENCE_INITIATIVE);
1894
1895                 /* If AXR CMD CQE came before previous TRSP CQE of same XRI */
1896                 if (seq->hio->type == OCS_HW_IO_TARGET_RSP) {
1897                         seq->hio->axr_buf->call_axr_cmd = 1;
1898                         seq->hio->axr_buf->cmd_seq = seq;
1899                         goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd;
1900                 }
1901         }
1902
1903         /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */
1904         if (hw->config.bounce) {
1905                 fc_header_t *hdr = seq->header->dma.virt;
1906                 uint32_t s_id = fc_be24toh(hdr->s_id);
1907                 uint32_t d_id = fc_be24toh(hdr->d_id);
1908                 uint32_t ox_id =  ocs_be16toh(hdr->ox_id);
1909                 if (hw->callback.bounce != NULL) {
1910                         (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id);
1911                 }
1912         } else {
1913                 hw->callback.unsolicited(hw->args.unsolicited, seq);
1914         }
1915
1916         if (seq->auto_xrdy) {
1917                 /* If data cqe came before cmd cqe in out of order in case of AXR */
1918                 if(seq->hio->axr_buf->data_cqe == 1) {
1919 #if defined(OCS_DISC_SPIN_DELAY)
1920                         if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) {
1921                                 delay = ocs_strtoul(prop_buf, 0, 0);
1922                                 ocs_udelay(delay);
1923                         }
1924 #endif
1925                         /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */
1926                         if (hw->config.bounce) {
1927                                 fc_header_t *hdr = seq->header->dma.virt;
1928                                 uint32_t s_id = fc_be24toh(hdr->s_id);
1929                                 uint32_t d_id = fc_be24toh(hdr->d_id);
1930                                 uint32_t ox_id =  ocs_be16toh(hdr->ox_id);
1931                                 if (hw->callback.bounce != NULL) {
1932                                         (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, &seq->hio->axr_buf->seq, s_id, d_id, ox_id);
1933                                 }
1934                         } else {
1935                                 hw->callback.unsolicited(hw->args.unsolicited, &seq->hio->axr_buf->seq);
1936                         }
1937                 }
1938         }
1939
1940 exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd:
1941         if(axr_lock_taken) {
1942                 ocs_unlock(&seq->hio->axr_lock);
1943         }
1944         return 0;
1945 }
1946
1947 /**
1948  * @brief Process CQ completions for Auto xfer rdy data phases.
1949  *
1950  * @par Description
1951  * The data is DMA'd into the data buffer posted to the SGL prior to the XRI
1952  * being assigned to an IO. When the completion is received, All of the data
1953  * is in the single buffer.
1954  *
1955  * @param hw Hardware context.
1956  * @param cq Pointer to HW completion queue.
1957  * @param cqe Completion queue entry.
1958  *
1959  * @return Returns 0 for success, or a negative error code value for failure.
1960  */
1961
1962 int32_t
1963 ocs_hw_rqpair_process_auto_xfr_rdy_data(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe)
1964 {
1965         /* Seems silly to call a SLI function to decode - use the structure directly for performance */
1966         sli4_fc_optimized_write_data_cqe_t *opt_wr = (sli4_fc_optimized_write_data_cqe_t*)cqe;
1967         ocs_hw_sequence_t *seq;
1968         ocs_hw_io_t *io;
1969         ocs_hw_auto_xfer_rdy_buffer_t *buf;
1970 #if defined(OCS_DISC_SPIN_DELAY)
1971         uint32_t        delay = 0;
1972         char            prop_buf[32];
1973 #endif
1974         /* Look up the IO */
1975         io = ocs_hw_io_lookup(hw, opt_wr->xri);
1976         ocs_lock(&io->axr_lock);
1977         buf = io->axr_buf;
1978         buf->data_cqe = 1;
1979         seq = &buf->seq;
1980         seq->hw = hw;
1981         seq->auto_xrdy = 1;
1982         seq->out_of_xris = 0;
1983         seq->xri = opt_wr->xri;
1984         seq->hio = io;
1985         seq->header = &buf->header;
1986         seq->payload = &buf->payload;
1987
1988         seq->header->dma.len = sizeof(fc_header_t);
1989         seq->payload->dma.len = opt_wr->total_data_placed;
1990         seq->fcfi = buf->fcfi;
1991         seq->hw_priv = cq->eq;
1992
1993         if (opt_wr->status == SLI4_FC_WCQE_STATUS_SUCCESS) {
1994                 seq->status = OCS_HW_UNSOL_SUCCESS;
1995         } else if (opt_wr->status == SLI4_FC_WCQE_STATUS_REMOTE_STOP) {
1996                 seq->status = OCS_HW_UNSOL_ABTS_RCVD;
1997         } else {
1998                 seq->status = OCS_HW_UNSOL_ERROR;
1999         }
2000
2001         /* If AXR CMD CQE came before previous TRSP CQE of same XRI */
2002         if(io->type == OCS_HW_IO_TARGET_RSP) {
2003                 io->axr_buf->call_axr_data = 1;
2004                 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data;
2005         }
2006
2007         if(!buf->cmd_cqe) {
2008                 /* if data cqe came before cmd cqe, return here, cmd cqe will handle */
2009                 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data;
2010         }
2011 #if defined(OCS_DISC_SPIN_DELAY)
2012         if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) {
2013                 delay = ocs_strtoul(prop_buf, 0, 0);
2014                 ocs_udelay(delay);
2015         }
2016 #endif
2017
2018         /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */
2019         if (hw->config.bounce) {
2020                 fc_header_t *hdr = seq->header->dma.virt;
2021                 uint32_t s_id = fc_be24toh(hdr->s_id);
2022                 uint32_t d_id = fc_be24toh(hdr->d_id);
2023                 uint32_t ox_id =  ocs_be16toh(hdr->ox_id);
2024                 if (hw->callback.bounce != NULL) {
2025                         (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id);
2026                 }
2027         } else {
2028                 hw->callback.unsolicited(hw->args.unsolicited, seq);
2029         }
2030
2031 exit_ocs_hw_rqpair_process_auto_xfr_rdy_data:
2032         ocs_unlock(&io->axr_lock);
2033         return 0;
2034 }
2035
2036 /**
2037  * @brief Return pointer to RQ buffer entry.
2038  *
2039  * @par Description
2040  * Returns a pointer to the RQ buffer entry given by @c rqindex and @c bufindex.
2041  *
2042  * @param hw Hardware context.
2043  * @param rqindex Index of the RQ that is being processed.
2044  * @param bufindex Index into the RQ that is being processed.
2045  *
2046  * @return Pointer to the sequence structure, or NULL otherwise.
2047  */
2048 static ocs_hw_sequence_t *
2049 ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex)
2050 {
2051         sli4_queue_t *rq_hdr = &hw->rq[rqindex];
2052         sli4_queue_t *rq_payload = &hw->rq[rqindex+1];
2053         ocs_hw_sequence_t *seq = NULL;
2054         hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]];
2055
2056 #if defined(ENABLE_DEBUG_RQBUF)
2057         uint64_t rqbuf_debug_value = 0xdead0000 | ((rq->id & 0xf) << 12) | (bufindex & 0xfff);
2058 #endif
2059
2060         if (bufindex >= rq_hdr->length) {
2061                 ocs_log_err(hw->os, "RQ index %d bufindex %d exceed ring length %d for id %d\n",
2062                             rqindex, bufindex, rq_hdr->length, rq_hdr->id);
2063                 return NULL;
2064         }
2065
2066         sli_queue_lock(rq_hdr);
2067         sli_queue_lock(rq_payload);
2068
2069 #if defined(ENABLE_DEBUG_RQBUF)
2070         /* Put a debug value into the rq, to track which entries are still valid */
2071         _sli_queue_poke(&hw->sli, rq_hdr, bufindex, (uint8_t *)&rqbuf_debug_value);
2072         _sli_queue_poke(&hw->sli, rq_payload, bufindex, (uint8_t *)&rqbuf_debug_value);
2073 #endif
2074
2075         seq = rq->rq_tracker[bufindex];
2076         rq->rq_tracker[bufindex] = NULL;
2077
2078         if (seq == NULL ) {
2079                 ocs_log_err(hw->os, "RQ buffer NULL, rqindex %d, bufindex %d, current q index = %d\n",
2080                             rqindex, bufindex, rq_hdr->index);
2081         }
2082
2083         sli_queue_unlock(rq_payload);
2084         sli_queue_unlock(rq_hdr);
2085         return seq;
2086 }
2087
2088 /**
2089  * @brief Posts an RQ buffer to a queue and update the verification structures
2090  *
2091  * @param hw            hardware context
2092  * @param seq Pointer to sequence object.
2093  *
2094  * @return Returns 0 on success, or a non-zero value otherwise.
2095  */
2096 static int32_t
2097 ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq)
2098 {
2099         sli4_queue_t *rq_hdr = &hw->rq[seq->header->rqindex];
2100         sli4_queue_t *rq_payload = &hw->rq[seq->payload->rqindex];
2101         uint32_t hw_rq_index = hw->hw_rq_lookup[seq->header->rqindex];
2102         hw_rq_t *rq = hw->hw_rq[hw_rq_index];
2103         uint32_t     phys_hdr[2];
2104         uint32_t     phys_payload[2];
2105         int32_t      qindex_hdr;
2106         int32_t      qindex_payload;
2107
2108         /* Update the RQ verification lookup tables */
2109         phys_hdr[0] = ocs_addr32_hi(seq->header->dma.phys);
2110         phys_hdr[1] = ocs_addr32_lo(seq->header->dma.phys);
2111         phys_payload[0] = ocs_addr32_hi(seq->payload->dma.phys);
2112         phys_payload[1] = ocs_addr32_lo(seq->payload->dma.phys);
2113
2114         sli_queue_lock(rq_hdr);
2115         sli_queue_lock(rq_payload);
2116
2117         /*
2118          * Note: The header must be posted last for buffer pair mode because
2119          *       posting on the header queue posts the payload queue as well.
2120          *       We do not ring the payload queue independently in RQ pair mode.
2121          */
2122         qindex_payload = _sli_queue_write(&hw->sli, rq_payload, (void *)phys_payload);
2123         qindex_hdr = _sli_queue_write(&hw->sli, rq_hdr, (void *)phys_hdr);
2124         if (qindex_hdr < 0 ||
2125             qindex_payload < 0) {
2126                 ocs_log_err(hw->os, "RQ_ID=%#x write failed\n", rq_hdr->id);
2127                 sli_queue_unlock(rq_payload);
2128                 sli_queue_unlock(rq_hdr);
2129                 return OCS_HW_RTN_ERROR;
2130         }
2131
2132         /* ensure the indexes are the same */
2133         ocs_hw_assert(qindex_hdr == qindex_payload);
2134
2135         /* Update the lookup table */
2136         if (rq->rq_tracker[qindex_hdr] == NULL) {
2137                 rq->rq_tracker[qindex_hdr] = seq;
2138         } else {
2139                 ocs_log_test(hw->os, "expected rq_tracker[%d][%d] buffer to be NULL\n",
2140                              hw_rq_index, qindex_hdr);
2141         }
2142
2143         sli_queue_unlock(rq_payload);
2144         sli_queue_unlock(rq_hdr);
2145         return OCS_HW_RTN_SUCCESS;
2146 }
2147
2148 /**
2149  * @brief Return RQ buffers (while in RQ pair mode).
2150  *
2151  * @par Description
2152  * The header and payload buffers are returned to the Receive Queue.
2153  *
2154  * @param hw Hardware context.
2155  * @param seq Header/payload sequence buffers.
2156  *
2157  * @return Returns OCS_HW_RTN_SUCCESS on success, or an error code value on failure.
2158  */
2159
2160 ocs_hw_rtn_e
2161 ocs_hw_rqpair_sequence_free(ocs_hw_t *hw, ocs_hw_sequence_t *seq)
2162 {
2163         ocs_hw_rtn_e   rc = OCS_HW_RTN_SUCCESS;
2164
2165         /* Check for auto xfer rdy dummy buffers and call the proper release function. */
2166         if (seq->header->rqindex == OCS_HW_RQ_INDEX_DUMMY_HDR) {
2167                 return ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(hw, seq);
2168         }
2169
2170         /*
2171          * Post the data buffer first. Because in RQ pair mode, ringing the
2172          * doorbell of the header ring will post the data buffer as well.
2173          */
2174         if (ocs_hw_rqpair_put(hw, seq)) {
2175                 ocs_log_err(hw->os, "error writing buffers\n");
2176                 return OCS_HW_RTN_ERROR;
2177         }
2178
2179         return rc;
2180 }
2181
2182 /**
2183  * @brief Find the RQ index of RQ_ID.
2184  *
2185  * @param hw Hardware context.
2186  * @param rq_id RQ ID to find.
2187  *
2188  * @return Returns the RQ index, or -1 if not found
2189  */
2190 static inline int32_t
2191 ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id)
2192 {
2193         return ocs_hw_queue_hash_find(hw->rq_hash, rq_id);
2194 }
2195
2196 /**
2197  * @ingroup devInitShutdown
2198  * @brief Allocate auto xfer rdy buffers.
2199  *
2200  * @par Description
2201  * Allocates the auto xfer rdy buffers and places them on the free list.
2202  *
2203  * @param hw Hardware context allocated by the caller.
2204  * @param num_buffers Number of buffers to allocate.
2205  *
2206  * @return Returns 0 on success, or a non-zero value on failure.
2207  */
2208 ocs_hw_rtn_e
2209 ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(ocs_hw_t *hw, uint32_t num_buffers)
2210 {
2211         ocs_hw_auto_xfer_rdy_buffer_t *buf;
2212         uint32_t i;
2213
2214         hw->auto_xfer_rdy_buf_pool = ocs_pool_alloc(hw->os, sizeof(ocs_hw_auto_xfer_rdy_buffer_t), num_buffers, FALSE);
2215         if (hw->auto_xfer_rdy_buf_pool == NULL) {
2216                 ocs_log_err(hw->os, "Failure to allocate auto xfer ready buffer pool\n");
2217                 return OCS_HW_RTN_NO_MEMORY;
2218         }
2219
2220         for (i = 0; i < num_buffers; i++) {
2221                 /* allocate the wrapper object */
2222                 buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i);
2223                 ocs_hw_assert(buf != NULL);
2224
2225                 /* allocate the auto xfer ready buffer */
2226                 if (ocs_dma_alloc(hw->os, &buf->payload.dma, hw->config.auto_xfer_rdy_size, OCS_MIN_DMA_ALIGNMENT)) {
2227                         ocs_log_err(hw->os, "DMA allocation failed\n");
2228                         ocs_free(hw->os, buf, sizeof(*buf));
2229                         return OCS_HW_RTN_NO_MEMORY;
2230                 }
2231
2232                 /* build a fake data header in big endian */
2233                 buf->hdr.info = FC_RCTL_INFO_SOL_DATA;
2234                 buf->hdr.r_ctl = FC_RCTL_FC4_DATA;
2235                 buf->hdr.type = FC_TYPE_FCP;
2236                 buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER |
2237                                             FC_FCTL_FIRST_SEQUENCE |
2238                                             FC_FCTL_LAST_SEQUENCE |
2239                                             FC_FCTL_END_SEQUENCE |
2240                                             FC_FCTL_SEQUENCE_INITIATIVE);
2241
2242                 /* build the fake header DMA object */
2243                 buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR;
2244                 buf->header.dma.virt = &buf->hdr;
2245                 buf->header.dma.alloc = buf;
2246                 buf->header.dma.size = sizeof(buf->hdr);
2247                 buf->header.dma.len = sizeof(buf->hdr);
2248
2249                 buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA;
2250         }
2251         return OCS_HW_RTN_SUCCESS;
2252 }
2253
2254 /**
2255  * @ingroup devInitShutdown
2256  * @brief Post Auto xfer rdy buffers to the XRIs posted with DNRX.
2257  *
2258  * @par Description
2259  * When new buffers are freed, check existing XRIs waiting for buffers.
2260  *
2261  * @param hw Hardware context allocated by the caller.
2262  */
2263 static void
2264 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(ocs_hw_t *hw)
2265 {
2266         ocs_hw_io_t *io;
2267         int32_t rc;
2268
2269         ocs_lock(&hw->io_lock);
2270
2271         while (!ocs_list_empty(&hw->io_port_dnrx)) {
2272                 io = ocs_list_remove_head(&hw->io_port_dnrx);
2273                 rc = ocs_hw_reque_xri(hw, io);
2274                 if(rc) {
2275                         break;
2276                 }
2277         }
2278
2279         ocs_unlock(&hw->io_lock);
2280 }
2281
2282 /**
2283  * @brief Called when the POST_SGL_PAGE command completes.
2284  *
2285  * @par Description
2286  * Free the mailbox command buffer.
2287  *
2288  * @param hw Hardware context.
2289  * @param status Status field from the mbox completion.
2290  * @param mqe Mailbox response structure.
2291  * @param arg Pointer to a callback function that signals the caller that the command is done.
2292  *
2293  * @return Returns 0.
2294  */
2295 static int32_t
2296 ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb(ocs_hw_t *hw, int32_t status, uint8_t *mqe, void  *arg)
2297 {
2298         if (status != 0) {
2299                 ocs_log_debug(hw->os, "Status 0x%x\n", status);
2300         }
2301
2302         ocs_free(hw->os, mqe, SLI4_BMBX_SIZE);
2303         return 0;
2304 }
2305
2306 /**
2307  * @brief Prepares an XRI to move to the chip.
2308  *
2309  * @par Description
2310  * Puts the data SGL into the SGL list for the IO object and possibly registers
2311  * an SGL list for the XRI. Since both the POST_XRI and POST_SGL_PAGES commands are
2312  * mailbox commands, we don't need to wait for completion before preceding.
2313  *
2314  * @param hw Hardware context allocated by the caller.
2315  * @param io Pointer to the IO object.
2316  *
2317  * @return Returns OCS_HW_RTN_SUCCESS for success, or an error code value for failure.
2318  */
2319 ocs_hw_rtn_e
2320 ocs_hw_rqpair_auto_xfer_rdy_move_to_port(ocs_hw_t *hw, ocs_hw_io_t *io)
2321 {
2322         /* We only need to preregister the SGL if it has not yet been done. */
2323         if (!sli_get_sgl_preregister(&hw->sli)) {
2324                 uint8_t *post_sgl;
2325                 ocs_dma_t *psgls = &io->def_sgl;
2326                 ocs_dma_t **sgls = &psgls;
2327
2328                 /* non-local buffer required for mailbox queue */
2329                 post_sgl = ocs_malloc(hw->os, SLI4_BMBX_SIZE, OCS_M_NOWAIT);
2330                 if (post_sgl == NULL) {
2331                         ocs_log_err(hw->os, "no buffer for command\n");
2332                         return OCS_HW_RTN_NO_MEMORY;
2333                 }
2334                 if (sli_cmd_fcoe_post_sgl_pages(&hw->sli, post_sgl, SLI4_BMBX_SIZE,
2335                                                 io->indicator, 1, sgls, NULL, NULL)) {
2336                         if (ocs_hw_command(hw, post_sgl, OCS_CMD_NOWAIT,
2337                                             ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb, NULL)) {
2338                                 ocs_free(hw->os, post_sgl, SLI4_BMBX_SIZE);
2339                                 ocs_log_err(hw->os, "SGL post failed\n");
2340                                 return OCS_HW_RTN_ERROR;
2341                         }
2342                 }
2343         }
2344
2345         ocs_lock(&hw->io_lock);
2346         if (ocs_hw_rqpair_auto_xfer_rdy_buffer_post(hw, io, 0) != 0) { /* DNRX set - no buffer */
2347                 ocs_unlock(&hw->io_lock);
2348                 return OCS_HW_RTN_ERROR;
2349         }
2350         ocs_unlock(&hw->io_lock);
2351         return OCS_HW_RTN_SUCCESS;
2352 }
2353
2354 /**
2355  * @brief Prepares an XRI to move back to the host.
2356  *
2357  * @par Description
2358  * Releases any attached buffer back to the pool.
2359  *
2360  * @param hw Hardware context allocated by the caller.
2361  * @param io Pointer to the IO object.
2362  */
2363 void
2364 ocs_hw_rqpair_auto_xfer_rdy_move_to_host(ocs_hw_t *hw, ocs_hw_io_t *io)
2365 {
2366         if (io->axr_buf != NULL) {
2367                 ocs_lock(&hw->io_lock);
2368                         /* check  list and remove if there */
2369                         if (ocs_list_on_list(&io->dnrx_link)) {
2370                                 ocs_list_remove(&hw->io_port_dnrx, io);
2371                                 io->auto_xfer_rdy_dnrx = 0;
2372
2373                                 /* release the count for waiting for a buffer */
2374                                 ocs_hw_io_free(hw, io);
2375                         }
2376
2377                         ocs_pool_put(hw->auto_xfer_rdy_buf_pool, io->axr_buf);
2378                         io->axr_buf = NULL;
2379                 ocs_unlock(&hw->io_lock);
2380
2381                 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw);
2382         }
2383         return;
2384 }
2385
2386 /**
2387  * @brief Posts an auto xfer rdy buffer to an IO.
2388  *
2389  * @par Description
2390  * Puts the data SGL into the SGL list for the IO object
2391  * @n @name
2392  * @b Note: io_lock must be held.
2393  *
2394  * @param hw Hardware context allocated by the caller.
2395  * @param io Pointer to the IO object.
2396  *
2397  * @return Returns the value of DNRX bit in the TRSP and ABORT WQEs.
2398  */
2399 uint8_t
2400 ocs_hw_rqpair_auto_xfer_rdy_buffer_post(ocs_hw_t *hw, ocs_hw_io_t *io, int reuse_buf)
2401 {
2402         ocs_hw_auto_xfer_rdy_buffer_t *buf;
2403         sli4_sge_t      *data;
2404
2405         if(!reuse_buf) {
2406                 buf = ocs_pool_get(hw->auto_xfer_rdy_buf_pool);
2407                 io->axr_buf = buf;
2408         }
2409
2410         data = io->def_sgl.virt;
2411         data[0].sge_type = SLI4_SGE_TYPE_SKIP;
2412         data[0].last = 0;
2413
2414         /*
2415          * Note: if we are doing DIF assists, then the SGE[1] must contain the
2416          * DI_SEED SGE. The host is responsible for programming:
2417          *   SGE Type (Word 2, bits 30:27)
2418          *   Replacement App Tag (Word 2 bits 15:0)
2419          *   App Tag (Word 3 bits 15:0)
2420          *   New Ref Tag (Word 3 bit 23)
2421          *   Metadata Enable (Word 3 bit 20)
2422          *   Auto-Increment RefTag (Word 3 bit 19)
2423          *   Block Size (Word 3 bits 18:16)
2424          * The following fields are managed by the SLI Port:
2425          *    Ref Tag Compare (Word 0)
2426          *    Replacement Ref Tag (Word 1) - In not the LBA
2427          *    NA (Word 2 bit 25)
2428          *    Opcode RX (Word 3 bits 27:24)
2429          *    Checksum Enable (Word 3 bit 22)
2430          *    RefTag Enable (Word 3 bit 21)
2431          *
2432          * The first two SGLs are cleared by ocs_hw_io_init_sges(), so assume eveything is cleared.
2433          */
2434         if (hw->config.auto_xfer_rdy_p_type) {
2435                 sli4_diseed_sge_t *diseed = (sli4_diseed_sge_t*)&data[1];
2436
2437                 diseed->sge_type = SLI4_SGE_TYPE_DISEED;
2438                 diseed->repl_app_tag = hw->config.auto_xfer_rdy_app_tag_value;
2439                 diseed->app_tag_cmp = hw->config.auto_xfer_rdy_app_tag_value;
2440                 diseed->check_app_tag = hw->config.auto_xfer_rdy_app_tag_valid;
2441                 diseed->auto_incr_ref_tag = TRUE; /* Always the LBA */
2442                 diseed->dif_blk_size = hw->config.auto_xfer_rdy_blk_size_chip;
2443         } else {
2444                 data[1].sge_type = SLI4_SGE_TYPE_SKIP;
2445                 data[1].last = 0;
2446         }
2447
2448         data[2].sge_type = SLI4_SGE_TYPE_DATA;
2449         data[2].buffer_address_high = ocs_addr32_hi(io->axr_buf->payload.dma.phys);
2450         data[2].buffer_address_low  = ocs_addr32_lo(io->axr_buf->payload.dma.phys);
2451         data[2].buffer_length = io->axr_buf->payload.dma.size;
2452         data[2].last = TRUE;
2453         data[3].sge_type = SLI4_SGE_TYPE_SKIP;
2454
2455         return 0;
2456 }
2457
2458 /**
2459  * @brief Return auto xfer ready buffers (while in RQ pair mode).
2460  *
2461  * @par Description
2462  * The header and payload buffers are returned to the auto xfer rdy pool.
2463  *
2464  * @param hw Hardware context.
2465  * @param seq Header/payload sequence buffers.
2466  *
2467  * @return Returns OCS_HW_RTN_SUCCESS for success, an error code value for failure.
2468  */
2469
2470 static ocs_hw_rtn_e
2471 ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq)
2472 {
2473         ocs_hw_auto_xfer_rdy_buffer_t *buf = seq->header->dma.alloc;
2474
2475         buf->data_cqe = 0;
2476         buf->cmd_cqe = 0;
2477         buf->fcfi = 0;
2478         buf->call_axr_cmd = 0;
2479         buf->call_axr_data = 0;
2480
2481         /* build a fake data header in big endian */
2482         buf->hdr.info = FC_RCTL_INFO_SOL_DATA;
2483         buf->hdr.r_ctl = FC_RCTL_FC4_DATA;
2484         buf->hdr.type = FC_TYPE_FCP;
2485         buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER |
2486                                         FC_FCTL_FIRST_SEQUENCE |
2487                                         FC_FCTL_LAST_SEQUENCE |
2488                                         FC_FCTL_END_SEQUENCE |
2489                                         FC_FCTL_SEQUENCE_INITIATIVE);
2490
2491         /* build the fake header DMA object */
2492         buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR;
2493         buf->header.dma.virt = &buf->hdr;
2494         buf->header.dma.alloc = buf;
2495         buf->header.dma.size = sizeof(buf->hdr);
2496         buf->header.dma.len = sizeof(buf->hdr);
2497         buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA;
2498
2499         ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw);
2500
2501         return OCS_HW_RTN_SUCCESS;
2502 }
2503
2504 /**
2505  * @ingroup devInitShutdown
2506  * @brief Free auto xfer rdy buffers.
2507  *
2508  * @par Description
2509  * Frees the auto xfer rdy buffers.
2510  *
2511  * @param hw Hardware context allocated by the caller.
2512  *
2513  * @return Returns 0 on success, or a non-zero value on failure.
2514  */
2515 static void
2516 ocs_hw_rqpair_auto_xfer_rdy_buffer_free(ocs_hw_t *hw)
2517 {
2518         ocs_hw_auto_xfer_rdy_buffer_t *buf;
2519         uint32_t i;
2520
2521         if (hw->auto_xfer_rdy_buf_pool != NULL) {
2522                 ocs_lock(&hw->io_lock);
2523                         for (i = 0; i < ocs_pool_get_count(hw->auto_xfer_rdy_buf_pool); i++) {
2524                                 buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i);
2525                                 if (buf != NULL) {
2526                                         ocs_dma_free(hw->os, &buf->payload.dma);
2527                                 }
2528                         }
2529                 ocs_unlock(&hw->io_lock);
2530
2531                 ocs_pool_free(hw->auto_xfer_rdy_buf_pool);
2532                 hw->auto_xfer_rdy_buf_pool = NULL;
2533         }
2534 }
2535
2536 /**
2537  * @ingroup devInitShutdown
2538  * @brief Configure the rq_pair function from ocs_hw_init().
2539  *
2540  * @par Description
2541  * Allocates the buffers to auto xfer rdy and posts initial XRIs for this feature.
2542  *
2543  * @param hw Hardware context allocated by the caller.
2544  *
2545  * @return Returns 0 on success, or a non-zero value on failure.
2546  */
2547 ocs_hw_rtn_e
2548 ocs_hw_rqpair_init(ocs_hw_t *hw)
2549 {
2550         ocs_hw_rtn_e    rc;
2551         uint32_t xris_posted;
2552
2553         ocs_log_debug(hw->os, "RQ Pair mode\n");
2554
2555         /*
2556          * If we get this far, the auto XFR_RDY feature was enabled successfully, otherwise ocs_hw_init() would
2557          * return with an error. So allocate the buffers based on the initial XRI pool required to support this
2558          * feature.
2559          */
2560         if (sli_get_auto_xfer_rdy_capable(&hw->sli) &&
2561             hw->config.auto_xfer_rdy_size > 0) {
2562                 if (hw->auto_xfer_rdy_buf_pool == NULL) {
2563                         /*
2564                          * Allocate one more buffer than XRIs so that when all the XRIs are in use, we still have
2565                          * one to post back for the case where the response phase is started in the context of
2566                          * the data completion.
2567                          */
2568                         rc = ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(hw, hw->config.auto_xfer_rdy_xri_cnt + 1);
2569                         if (rc != OCS_HW_RTN_SUCCESS) {
2570                                 return rc;
2571                         }
2572                 } else {
2573                         ocs_pool_reset(hw->auto_xfer_rdy_buf_pool);
2574                 }
2575
2576                 /* Post the auto XFR_RDY XRIs */
2577                 xris_posted = ocs_hw_xri_move_to_port_owned(hw, hw->config.auto_xfer_rdy_xri_cnt);
2578                 if (xris_posted != hw->config.auto_xfer_rdy_xri_cnt) {
2579                         ocs_log_err(hw->os, "post_xri failed, only posted %d XRIs\n", xris_posted);
2580                         return OCS_HW_RTN_ERROR;
2581                 }
2582         }
2583
2584         return 0;
2585 }
2586
2587 /**
2588  * @ingroup devInitShutdown
2589  * @brief Tear down the rq_pair function from ocs_hw_teardown().
2590  *
2591  * @par Description
2592  * Frees the buffers to auto xfer rdy.
2593  *
2594  * @param hw Hardware context allocated by the caller.
2595  */
2596 void
2597 ocs_hw_rqpair_teardown(ocs_hw_t *hw)
2598 {
2599         /* We need to free any auto xfer ready buffers */
2600         ocs_hw_rqpair_auto_xfer_rdy_buffer_free(hw);
2601 }