sys/dev/ixl/i40e_adminq.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #include "i40e_status.h"
  36 #include "i40e_type.h"
  37 #include "i40e_register.h"
  38 #include "i40e_adminq.h"
  39 #include "i40e_prototype.h"
  40
  41 /**
  42  * i40e_is_nvm_update_op - return TRUE if this is an NVM update operation
  43  * @desc: API request descriptor
  44  **/
  45 static INLINE bool i40e_is_nvm_update_op(struct i40e_aq_desc *desc)
  46 {
  47         return (desc->opcode == CPU_TO_LE16(i40e_aqc_opc_nvm_erase) ||
  48                 desc->opcode == CPU_TO_LE16(i40e_aqc_opc_nvm_update));
  49 }
  50
  51 /**
  52  *  i40e_adminq_init_regs - Initialize AdminQ registers
  53  *  @hw: pointer to the hardware structure
  54  *
  55  *  This assumes the alloc_asq and alloc_arq functions have already been called
  56  **/
  57 static void i40e_adminq_init_regs(struct i40e_hw *hw)
  58 {
  59         /* set head and tail registers in our local struct */
  60         if (i40e_is_vf(hw)) {
  61                 hw->aq.asq.tail = I40E_VF_ATQT1;
  62                 hw->aq.asq.head = I40E_VF_ATQH1;
  63                 hw->aq.asq.len  = I40E_VF_ATQLEN1;
  64                 hw->aq.asq.bal  = I40E_VF_ATQBAL1;
  65                 hw->aq.asq.bah  = I40E_VF_ATQBAH1;
  66                 hw->aq.arq.tail = I40E_VF_ARQT1;
  67                 hw->aq.arq.head = I40E_VF_ARQH1;
  68                 hw->aq.arq.len  = I40E_VF_ARQLEN1;
  69                 hw->aq.arq.bal  = I40E_VF_ARQBAL1;
  70                 hw->aq.arq.bah  = I40E_VF_ARQBAH1;
  71         } else {
  72                 hw->aq.asq.tail = I40E_PF_ATQT;
  73                 hw->aq.asq.head = I40E_PF_ATQH;
  74                 hw->aq.asq.len  = I40E_PF_ATQLEN;
  75                 hw->aq.asq.bal  = I40E_PF_ATQBAL;
  76                 hw->aq.asq.bah  = I40E_PF_ATQBAH;
  77                 hw->aq.arq.tail = I40E_PF_ARQT;
  78                 hw->aq.arq.head = I40E_PF_ARQH;
  79                 hw->aq.arq.len  = I40E_PF_ARQLEN;
  80                 hw->aq.arq.bal  = I40E_PF_ARQBAL;
  81                 hw->aq.arq.bah  = I40E_PF_ARQBAH;
  82         }
  83 }
  84
  85 /**
  86  *  i40e_alloc_adminq_asq_ring - Allocate Admin Queue send rings
  87  *  @hw: pointer to the hardware structure
  88  **/
  89 enum i40e_status_code i40e_alloc_adminq_asq_ring(struct i40e_hw *hw)
  90 {
  91         enum i40e_status_code ret_code;
  92
  93         ret_code = i40e_allocate_dma_mem(hw, &hw->aq.asq.desc_buf,
  94                                          i40e_mem_atq_ring,
  95                                          (hw->aq.num_asq_entries *
  96                                          sizeof(struct i40e_aq_desc)),
  97                                          I40E_ADMINQ_DESC_ALIGNMENT);
  98         if (ret_code)
  99                 return ret_code;
 100
 101         ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.cmd_buf,
 102                                           (hw->aq.num_asq_entries *
 103                                           sizeof(struct i40e_asq_cmd_details)));
 104         if (ret_code) {
 105                 i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);
 106                 return ret_code;
 107         }
 108
 109         return ret_code;
 110 }
 111
 112 /**
 113  *  i40e_alloc_adminq_arq_ring - Allocate Admin Queue receive rings
 114  *  @hw: pointer to the hardware structure
 115  **/
 116 enum i40e_status_code i40e_alloc_adminq_arq_ring(struct i40e_hw *hw)
 117 {
 118         enum i40e_status_code ret_code;
 119
 120         ret_code = i40e_allocate_dma_mem(hw, &hw->aq.arq.desc_buf,
 121                                          i40e_mem_arq_ring,
 122                                          (hw->aq.num_arq_entries *
 123                                          sizeof(struct i40e_aq_desc)),
 124                                          I40E_ADMINQ_DESC_ALIGNMENT);
 125
 126         return ret_code;
 127 }
 128
 129 /**
 130  *  i40e_free_adminq_asq - Free Admin Queue send rings
 131  *  @hw: pointer to the hardware structure
 132  *
 133  *  This assumes the posted send buffers have already been cleaned
 134  *  and de-allocated
 135  **/
 136 void i40e_free_adminq_asq(struct i40e_hw *hw)
 137 {
 138         i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);
 139 }
 140
 141 /**
 142  *  i40e_free_adminq_arq - Free Admin Queue receive rings
 143  *  @hw: pointer to the hardware structure
 144  *
 145  *  This assumes the posted receive buffers have already been cleaned
 146  *  and de-allocated
 147  **/
 148 void i40e_free_adminq_arq(struct i40e_hw *hw)
 149 {
 150         i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf);
 151 }
 152
 153 /**
 154  *  i40e_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue
 155  *  @hw: pointer to the hardware structure
 156  **/
 157 static enum i40e_status_code i40e_alloc_arq_bufs(struct i40e_hw *hw)
 158 {
 159         enum i40e_status_code ret_code;
 160         struct i40e_aq_desc *desc;
 161         struct i40e_dma_mem *bi;
 162         int i;
 163
 164         /* We'll be allocating the buffer info memory first, then we can
 165          * allocate the mapped buffers for the event processing
 166          */
 167
 168         /* buffer_info structures do not need alignment */
 169         ret_code = i40e_allocate_virt_mem(hw, &hw->aq.arq.dma_head,
 170                 (hw->aq.num_arq_entries * sizeof(struct i40e_dma_mem)));
 171         if (ret_code)
 172                 goto alloc_arq_bufs;
 173         hw->aq.arq.r.arq_bi = (struct i40e_dma_mem *)hw->aq.arq.dma_head.va;
 174
 175         /* allocate the mapped buffers */
 176         for (i = 0; i < hw->aq.num_arq_entries; i++) {
 177                 bi = &hw->aq.arq.r.arq_bi[i];
 178                 ret_code = i40e_allocate_dma_mem(hw, bi,
 179                                                  i40e_mem_arq_buf,
 180                                                  hw->aq.arq_buf_size,
 181                                                  I40E_ADMINQ_DESC_ALIGNMENT);
 182                 if (ret_code)
 183                         goto unwind_alloc_arq_bufs;
 184
 185                 /* now configure the descriptors for use */
 186                 desc = I40E_ADMINQ_DESC(hw->aq.arq, i);
 187
 188                 desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF);
 189                 if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF)
 190                         desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB);
 191                 desc->opcode = 0;
 192                 /* This is in accordance with Admin queue design, there is no
 193                  * register for buffer size configuration
 194                  */
 195                 desc->datalen = CPU_TO_LE16((u16)bi->size);
 196                 desc->retval = 0;
 197                 desc->cookie_high = 0;
 198                 desc->cookie_low = 0;
 199                 desc->params.external.addr_high =
 200                         CPU_TO_LE32(I40E_HI_DWORD(bi->pa));
 201                 desc->params.external.addr_low =
 202                         CPU_TO_LE32(I40E_LO_DWORD(bi->pa));
 203                 desc->params.external.param0 = 0;
 204                 desc->params.external.param1 = 0;
 205         }
 206
 207 alloc_arq_bufs:
 208         return ret_code;
 209
 210 unwind_alloc_arq_bufs:
 211         /* don't try to free the one that failed... */
 212         i--;
 213         for (; i >= 0; i--)
 214                 i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
 215         i40e_free_virt_mem(hw, &hw->aq.arq.dma_head);
 216
 217         return ret_code;
 218 }
 219
 220 /**
 221  *  i40e_alloc_asq_bufs - Allocate empty buffer structs for the send queue
 222  *  @hw: pointer to the hardware structure
 223  **/
 224 static enum i40e_status_code i40e_alloc_asq_bufs(struct i40e_hw *hw)
 225 {
 226         enum i40e_status_code ret_code;
 227         struct i40e_dma_mem *bi;
 228         int i;
 229
 230         /* No mapped memory needed yet, just the buffer info structures */
 231         ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.dma_head,
 232                 (hw->aq.num_asq_entries * sizeof(struct i40e_dma_mem)));
 233         if (ret_code)
 234                 goto alloc_asq_bufs;
 235         hw->aq.asq.r.asq_bi = (struct i40e_dma_mem *)hw->aq.asq.dma_head.va;
 236
 237         /* allocate the mapped buffers */
 238         for (i = 0; i < hw->aq.num_asq_entries; i++) {
 239                 bi = &hw->aq.asq.r.asq_bi[i];
 240                 ret_code = i40e_allocate_dma_mem(hw, bi,
 241                                                  i40e_mem_asq_buf,
 242                                                  hw->aq.asq_buf_size,
 243                                                  I40E_ADMINQ_DESC_ALIGNMENT);
 244                 if (ret_code)
 245                         goto unwind_alloc_asq_bufs;
 246         }
 247 alloc_asq_bufs:
 248         return ret_code;
 249
 250 unwind_alloc_asq_bufs:
 251         /* don't try to free the one that failed... */
 252         i--;
 253         for (; i >= 0; i--)
 254                 i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
 255         i40e_free_virt_mem(hw, &hw->aq.asq.dma_head);
 256
 257         return ret_code;
 258 }
 259
 260 /**
 261  *  i40e_free_arq_bufs - Free receive queue buffer info elements
 262  *  @hw: pointer to the hardware structure
 263  **/
 264 static void i40e_free_arq_bufs(struct i40e_hw *hw)
 265 {
 266         int i;
 267
 268         /* free descriptors */
 269         for (i = 0; i < hw->aq.num_arq_entries; i++)
 270                 i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
 271
 272         /* free the descriptor memory */
 273         i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf);
 274
 275         /* free the dma header */
 276         i40e_free_virt_mem(hw, &hw->aq.arq.dma_head);
 277 }
 278
 279 /**
 280  *  i40e_free_asq_bufs - Free send queue buffer info elements
 281  *  @hw: pointer to the hardware structure
 282  **/
 283 static void i40e_free_asq_bufs(struct i40e_hw *hw)
 284 {
 285         int i;
 286
 287         /* only unmap if the address is non-NULL */
 288         for (i = 0; i < hw->aq.num_asq_entries; i++)
 289                 if (hw->aq.asq.r.asq_bi[i].pa)
 290                         i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
 291
 292         /* free the buffer info list */
 293         i40e_free_virt_mem(hw, &hw->aq.asq.cmd_buf);
 294
 295         /* free the descriptor memory */
 296         i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);
 297
 298         /* free the dma header */
 299         i40e_free_virt_mem(hw, &hw->aq.asq.dma_head);
 300 }
 301
 302 /**
 303  *  i40e_config_asq_regs - configure ASQ registers
 304  *  @hw: pointer to the hardware structure
 305  *
 306  *  Configure base address and length registers for the transmit queue
 307  **/
 308 static enum i40e_status_code i40e_config_asq_regs(struct i40e_hw *hw)
 309 {
 310         enum i40e_status_code ret_code = I40E_SUCCESS;
 311         u32 reg = 0;
 312
 313         /* Clear Head and Tail */
 314         wr32(hw, hw->aq.asq.head, 0);
 315         wr32(hw, hw->aq.asq.tail, 0);
 316
 317         /* set starting point */
 318         if (!i40e_is_vf(hw))
 319                 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries |
 320                                           I40E_PF_ATQLEN_ATQENABLE_MASK));
 321         if (i40e_is_vf(hw))
 322                 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries |
 323                                           I40E_VF_ATQLEN1_ATQENABLE_MASK));
 324         wr32(hw, hw->aq.asq.bal, I40E_LO_DWORD(hw->aq.asq.desc_buf.pa));
 325         wr32(hw, hw->aq.asq.bah, I40E_HI_DWORD(hw->aq.asq.desc_buf.pa));
 326
 327         /* Check one register to verify that config was applied */
 328         reg = rd32(hw, hw->aq.asq.bal);
 329         if (reg != I40E_LO_DWORD(hw->aq.asq.desc_buf.pa))
 330                 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR;
 331
 332         return ret_code;
 333 }
 334
 335 /**
 336  *  i40e_config_arq_regs - ARQ register configuration
 337  *  @hw: pointer to the hardware structure
 338  *
 339  * Configure base address and length registers for the receive (event queue)
 340  **/
 341 static enum i40e_status_code i40e_config_arq_regs(struct i40e_hw *hw)
 342 {
 343         enum i40e_status_code ret_code = I40E_SUCCESS;
 344         u32 reg = 0;
 345
 346         /* Clear Head and Tail */
 347         wr32(hw, hw->aq.arq.head, 0);
 348         wr32(hw, hw->aq.arq.tail, 0);
 349
 350         /* set starting point */
 351         if (!i40e_is_vf(hw))
 352                 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries |
 353                                           I40E_PF_ARQLEN_ARQENABLE_MASK));
 354         if (i40e_is_vf(hw))
 355                 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries |
 356                                           I40E_VF_ARQLEN1_ARQENABLE_MASK));
 357         wr32(hw, hw->aq.arq.bal, I40E_LO_DWORD(hw->aq.arq.desc_buf.pa));
 358         wr32(hw, hw->aq.arq.bah, I40E_HI_DWORD(hw->aq.arq.desc_buf.pa));
 359
 360         /* Update tail in the HW to post pre-allocated buffers */
 361         wr32(hw, hw->aq.arq.tail, hw->aq.num_arq_entries - 1);
 362
 363         /* Check one register to verify that config was applied */
 364         reg = rd32(hw, hw->aq.arq.bal);
 365         if (reg != I40E_LO_DWORD(hw->aq.arq.desc_buf.pa))
 366                 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR;
 367
 368         return ret_code;
 369 }
 370
 371 /**
 372  *  i40e_init_asq - main initialization routine for ASQ
 373  *  @hw: pointer to the hardware structure
 374  *
 375  *  This is the main initialization routine for the Admin Send Queue
 376  *  Prior to calling this function, drivers *MUST* set the following fields
 377  *  in the hw->aq structure:
 378  *     - hw->aq.num_asq_entries
 379  *     - hw->aq.arq_buf_size
 380  *
 381  *  Do *NOT* hold the lock when calling this as the memory allocation routines
 382  *  called are not going to be atomic context safe
 383  **/
 384 enum i40e_status_code i40e_init_asq(struct i40e_hw *hw)
 385 {
 386         enum i40e_status_code ret_code = I40E_SUCCESS;
 387
 388         if (hw->aq.asq.count > 0) {
 389                 /* queue already initialized */
 390                 ret_code = I40E_ERR_NOT_READY;
 391                 goto init_adminq_exit;
 392         }
 393
 394         /* verify input for valid configuration */
 395         if ((hw->aq.num_asq_entries == 0) ||
 396             (hw->aq.asq_buf_size == 0)) {
 397                 ret_code = I40E_ERR_CONFIG;
 398                 goto init_adminq_exit;
 399         }
 400
 401         hw->aq.asq.next_to_use = 0;
 402         hw->aq.asq.next_to_clean = 0;
 403         hw->aq.asq.count = hw->aq.num_asq_entries;
 404
 405         /* allocate the ring memory */
 406         ret_code = i40e_alloc_adminq_asq_ring(hw);
 407         if (ret_code != I40E_SUCCESS)
 408                 goto init_adminq_exit;
 409
 410         /* allocate buffers in the rings */
 411         ret_code = i40e_alloc_asq_bufs(hw);
 412         if (ret_code != I40E_SUCCESS)
 413                 goto init_adminq_free_rings;
 414
 415         /* initialize base registers */
 416         ret_code = i40e_config_asq_regs(hw);
 417         if (ret_code != I40E_SUCCESS)
 418                 goto init_adminq_free_rings;
 419
 420         /* success! */
 421         goto init_adminq_exit;
 422
 423 init_adminq_free_rings:
 424         i40e_free_adminq_asq(hw);
 425
 426 init_adminq_exit:
 427         return ret_code;
 428 }
 429
 430 /**
 431  *  i40e_init_arq - initialize ARQ
 432  *  @hw: pointer to the hardware structure
 433  *
 434  *  The main initialization routine for the Admin Receive (Event) Queue.
 435  *  Prior to calling this function, drivers *MUST* set the following fields
 436  *  in the hw->aq structure:
 437  *     - hw->aq.num_asq_entries
 438  *     - hw->aq.arq_buf_size
 439  *
 440  *  Do *NOT* hold the lock when calling this as the memory allocation routines
 441  *  called are not going to be atomic context safe
 442  **/
 443 enum i40e_status_code i40e_init_arq(struct i40e_hw *hw)
 444 {
 445         enum i40e_status_code ret_code = I40E_SUCCESS;
 446
 447         if (hw->aq.arq.count > 0) {
 448                 /* queue already initialized */
 449                 ret_code = I40E_ERR_NOT_READY;
 450                 goto init_adminq_exit;
 451         }
 452
 453         /* verify input for valid configuration */
 454         if ((hw->aq.num_arq_entries == 0) ||
 455             (hw->aq.arq_buf_size == 0)) {
 456                 ret_code = I40E_ERR_CONFIG;
 457                 goto init_adminq_exit;
 458         }
 459
 460         hw->aq.arq.next_to_use = 0;
 461         hw->aq.arq.next_to_clean = 0;
 462         hw->aq.arq.count = hw->aq.num_arq_entries;
 463
 464         /* allocate the ring memory */
 465         ret_code = i40e_alloc_adminq_arq_ring(hw);
 466         if (ret_code != I40E_SUCCESS)
 467                 goto init_adminq_exit;
 468
 469         /* allocate buffers in the rings */
 470         ret_code = i40e_alloc_arq_bufs(hw);
 471         if (ret_code != I40E_SUCCESS)
 472                 goto init_adminq_free_rings;
 473
 474         /* initialize base registers */
 475         ret_code = i40e_config_arq_regs(hw);
 476         if (ret_code != I40E_SUCCESS)
 477                 goto init_adminq_free_rings;
 478
 479         /* success! */
 480         goto init_adminq_exit;
 481
 482 init_adminq_free_rings:
 483         i40e_free_adminq_arq(hw);
 484
 485 init_adminq_exit:
 486         return ret_code;
 487 }
 488
 489 /**
 490  *  i40e_shutdown_asq - shutdown the ASQ
 491  *  @hw: pointer to the hardware structure
 492  *
 493  *  The main shutdown routine for the Admin Send Queue
 494  **/
 495 enum i40e_status_code i40e_shutdown_asq(struct i40e_hw *hw)
 496 {
 497         enum i40e_status_code ret_code = I40E_SUCCESS;
 498
 499         if (hw->aq.asq.count == 0)
 500                 return I40E_ERR_NOT_READY;
 501
 502         /* Stop firmware AdminQ processing */
 503         wr32(hw, hw->aq.asq.head, 0);
 504         wr32(hw, hw->aq.asq.tail, 0);
 505         wr32(hw, hw->aq.asq.len, 0);
 506         wr32(hw, hw->aq.asq.bal, 0);
 507         wr32(hw, hw->aq.asq.bah, 0);
 508
 509         /* make sure spinlock is available */
 510         i40e_acquire_spinlock(&hw->aq.asq_spinlock);
 511
 512         hw->aq.asq.count = 0; /* to indicate uninitialized queue */
 513
 514         /* free ring buffers */
 515         i40e_free_asq_bufs(hw);
 516
 517         i40e_release_spinlock(&hw->aq.asq_spinlock);
 518
 519         return ret_code;
 520 }
 521
 522 /**
 523  *  i40e_shutdown_arq - shutdown ARQ
 524  *  @hw: pointer to the hardware structure
 525  *
 526  *  The main shutdown routine for the Admin Receive Queue
 527  **/
 528 enum i40e_status_code i40e_shutdown_arq(struct i40e_hw *hw)
 529 {
 530         enum i40e_status_code ret_code = I40E_SUCCESS;
 531
 532         if (hw->aq.arq.count == 0)
 533                 return I40E_ERR_NOT_READY;
 534
 535         /* Stop firmware AdminQ processing */
 536         wr32(hw, hw->aq.arq.head, 0);
 537         wr32(hw, hw->aq.arq.tail, 0);
 538         wr32(hw, hw->aq.arq.len, 0);
 539         wr32(hw, hw->aq.arq.bal, 0);
 540         wr32(hw, hw->aq.arq.bah, 0);
 541
 542         /* make sure spinlock is available */
 543         i40e_acquire_spinlock(&hw->aq.arq_spinlock);
 544
 545         hw->aq.arq.count = 0; /* to indicate uninitialized queue */
 546
 547         /* free ring buffers */
 548         i40e_free_arq_bufs(hw);
 549
 550         i40e_release_spinlock(&hw->aq.arq_spinlock);
 551
 552         return ret_code;
 553 }
 554
 555 /**
 556  *  i40e_init_adminq - main initialization routine for Admin Queue
 557  *  @hw: pointer to the hardware structure
 558  *
 559  *  Prior to calling this function, drivers *MUST* set the following fields
 560  *  in the hw->aq structure:
 561  *     - hw->aq.num_asq_entries
 562  *     - hw->aq.num_arq_entries
 563  *     - hw->aq.arq_buf_size
 564  *     - hw->aq.asq_buf_size
 565  **/
 566 enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw)
 567 {
 568         enum i40e_status_code ret_code;
 569         u16 eetrack_lo, eetrack_hi;
 570         u16 cfg_ptr, oem_hi, oem_lo;
 571         int retry = 0;
 572         /* verify input for valid configuration */
 573         if ((hw->aq.num_arq_entries == 0) ||
 574             (hw->aq.num_asq_entries == 0) ||
 575             (hw->aq.arq_buf_size == 0) ||
 576             (hw->aq.asq_buf_size == 0)) {
 577                 ret_code = I40E_ERR_CONFIG;
 578                 goto init_adminq_exit;
 579         }
 580
 581         /* initialize spin locks */
 582         i40e_init_spinlock(&hw->aq.asq_spinlock);
 583         i40e_init_spinlock(&hw->aq.arq_spinlock);
 584
 585         /* Set up register offsets */
 586         i40e_adminq_init_regs(hw);
 587
 588         /* setup ASQ command write back timeout */
 589         hw->aq.asq_cmd_timeout = I40E_ASQ_CMD_TIMEOUT;
 590
 591         /* allocate the ASQ */
 592         ret_code = i40e_init_asq(hw);
 593         if (ret_code != I40E_SUCCESS)
 594                 goto init_adminq_destroy_spinlocks;
 595
 596         /* allocate the ARQ */
 597         ret_code = i40e_init_arq(hw);
 598         if (ret_code != I40E_SUCCESS)
 599                 goto init_adminq_free_asq;
 600
 601         /* VF has no need of firmware */
 602         if (i40e_is_vf(hw))
 603                 goto init_adminq_exit;
 604         /* There are some cases where the firmware may not be quite ready
 605          * for AdminQ operations, so we retry the AdminQ setup a few times
 606          * if we see timeouts in this first AQ call.
 607          */
 608         do {
 609                 ret_code = i40e_aq_get_firmware_version(hw,
 610                                                         &hw->aq.fw_maj_ver,
 611                                                         &hw->aq.fw_min_ver,
 612                                                         &hw->aq.fw_build,
 613                                                         &hw->aq.api_maj_ver,
 614                                                         &hw->aq.api_min_ver,
 615                                                         NULL);
 616                 if (ret_code != I40E_ERR_ADMIN_QUEUE_TIMEOUT)
 617                         break;
 618                 retry++;
 619                 i40e_msec_delay(100);
 620                 i40e_resume_aq(hw);
 621         } while (retry < 10);
 622         if (ret_code != I40E_SUCCESS)
 623                 goto init_adminq_free_arq;
 624
 625         /* get the NVM version info */
 626         i40e_read_nvm_word(hw, I40E_SR_NVM_DEV_STARTER_VERSION,
 627                            &hw->nvm.version);
 628         i40e_read_nvm_word(hw, I40E_SR_NVM_EETRACK_LO, &eetrack_lo);
 629         i40e_read_nvm_word(hw, I40E_SR_NVM_EETRACK_HI, &eetrack_hi);
 630         hw->nvm.eetrack = (eetrack_hi << 16) | eetrack_lo;
 631         i40e_read_nvm_word(hw, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr);
 632         i40e_read_nvm_word(hw, (cfg_ptr + I40E_NVM_OEM_VER_OFF),
 633                            &oem_hi);
 634         i40e_read_nvm_word(hw, (cfg_ptr + (I40E_NVM_OEM_VER_OFF + 1)),
 635                            &oem_lo);
 636         hw->nvm.oem_ver = ((u32)oem_hi << 16) | oem_lo;
 637
 638         if (hw->aq.api_maj_ver > I40E_FW_API_VERSION_MAJOR) {
 639                 ret_code = I40E_ERR_FIRMWARE_API_VERSION;
 640                 goto init_adminq_free_arq;
 641         }
 642
 643         /* pre-emptive resource lock release */
 644         i40e_aq_release_resource(hw, I40E_NVM_RESOURCE_ID, 0, NULL);
 645         hw->aq.nvm_release_on_done = FALSE;
 646         hw->nvmupd_state = I40E_NVMUPD_STATE_INIT;
 647
 648         ret_code = i40e_aq_set_hmc_resource_profile(hw,
 649                                                     I40E_HMC_PROFILE_DEFAULT,
 650                                                     0,
 651                                                     NULL);
 652         ret_code = I40E_SUCCESS;
 653
 654         /* success! */
 655         goto init_adminq_exit;
 656
 657 init_adminq_free_arq:
 658         i40e_shutdown_arq(hw);
 659 init_adminq_free_asq:
 660         i40e_shutdown_asq(hw);
 661 init_adminq_destroy_spinlocks:
 662         i40e_destroy_spinlock(&hw->aq.asq_spinlock);
 663         i40e_destroy_spinlock(&hw->aq.arq_spinlock);
 664
 665 init_adminq_exit:
 666         return ret_code;
 667 }
 668
 669 /**
 670  *  i40e_shutdown_adminq - shutdown routine for the Admin Queue
 671  *  @hw: pointer to the hardware structure
 672  **/
 673 enum i40e_status_code i40e_shutdown_adminq(struct i40e_hw *hw)
 674 {
 675         enum i40e_status_code ret_code = I40E_SUCCESS;
 676
 677         if (i40e_check_asq_alive(hw))
 678                 i40e_aq_queue_shutdown(hw, TRUE);
 679
 680         i40e_shutdown_asq(hw);
 681         i40e_shutdown_arq(hw);
 682
 683         /* destroy the spinlocks */
 684         i40e_destroy_spinlock(&hw->aq.asq_spinlock);
 685         i40e_destroy_spinlock(&hw->aq.arq_spinlock);
 686
 687         if (hw->nvm_buff.va)
 688                 i40e_free_virt_mem(hw, &hw->nvm_buff);
 689
 690         return ret_code;
 691 }
 692
 693 /**
 694  *  i40e_clean_asq - cleans Admin send queue
 695  *  @hw: pointer to the hardware structure
 696  *
 697  *  returns the number of free desc
 698  **/
 699 u16 i40e_clean_asq(struct i40e_hw *hw)
 700 {
 701         struct i40e_adminq_ring *asq = &(hw->aq.asq);
 702         struct i40e_asq_cmd_details *details;
 703         u16 ntc = asq->next_to_clean;
 704         struct i40e_aq_desc desc_cb;
 705         struct i40e_aq_desc *desc;
 706
 707         desc = I40E_ADMINQ_DESC(*asq, ntc);
 708         details = I40E_ADMINQ_DETAILS(*asq, ntc);
 709
 710         while (rd32(hw, hw->aq.asq.head) != ntc) {
 711                 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
 712                            "ntc %d head %d.\n", ntc, rd32(hw, hw->aq.asq.head));
 713
 714                 if (details->callback) {
 715                         I40E_ADMINQ_CALLBACK cb_func =
 716                                         (I40E_ADMINQ_CALLBACK)details->callback;
 717                         i40e_memcpy(&desc_cb, desc, sizeof(struct i40e_aq_desc),
 718                                     I40E_DMA_TO_DMA);
 719                         cb_func(hw, &desc_cb);
 720                 }
 721                 i40e_memset(desc, 0, sizeof(*desc), I40E_DMA_MEM);
 722                 i40e_memset(details, 0, sizeof(*details), I40E_NONDMA_MEM);
 723                 ntc++;
 724                 if (ntc == asq->count)
 725                         ntc = 0;
 726                 desc = I40E_ADMINQ_DESC(*asq, ntc);
 727                 details = I40E_ADMINQ_DETAILS(*asq, ntc);
 728         }
 729
 730         asq->next_to_clean = ntc;
 731
 732         return I40E_DESC_UNUSED(asq);
 733 }
 734
 735 /**
 736  *  i40e_asq_done - check if FW has processed the Admin Send Queue
 737  *  @hw: pointer to the hw struct
 738  *
 739  *  Returns TRUE if the firmware has processed all descriptors on the
 740  *  admin send queue. Returns FALSE if there are still requests pending.
 741  **/
 742 bool i40e_asq_done(struct i40e_hw *hw)
 743 {
 744         /* AQ designers suggest use of head for better
 745          * timing reliability than DD bit
 746          */
 747         return rd32(hw, hw->aq.asq.head) == hw->aq.asq.next_to_use;
 748
 749 }
 750
 751 /**
 752  *  i40e_asq_send_command - send command to Admin Queue
 753  *  @hw: pointer to the hw struct
 754  *  @desc: prefilled descriptor describing the command (non DMA mem)
 755  *  @buff: buffer to use for indirect commands
 756  *  @buff_size: size of buffer for indirect commands
 757  *  @cmd_details: pointer to command details structure
 758  *
 759  *  This is the main send command driver routine for the Admin Queue send
 760  *  queue.  It runs the queue, cleans the queue, etc
 761  **/
 762 enum i40e_status_code i40e_asq_send_command(struct i40e_hw *hw,
 763                                 struct i40e_aq_desc *desc,
 764                                 void *buff, /* can be NULL */
 765                                 u16  buff_size,
 766                                 struct i40e_asq_cmd_details *cmd_details)
 767 {
 768         enum i40e_status_code status = I40E_SUCCESS;
 769         struct i40e_dma_mem *dma_buff = NULL;
 770         struct i40e_asq_cmd_details *details;
 771         struct i40e_aq_desc *desc_on_ring;
 772         bool cmd_completed = FALSE;
 773         u16  retval = 0;
 774         u32  val = 0;
 775
 776         hw->aq.asq_last_status = I40E_AQ_RC_OK;
 777
 778         val = rd32(hw, hw->aq.asq.head);
 779         if (val >= hw->aq.num_asq_entries) {
 780                 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
 781                            "AQTX: head overrun at %d\n", val);
 782                 status = I40E_ERR_QUEUE_EMPTY;
 783                 goto asq_send_command_exit;
 784         }
 785
 786         if (hw->aq.asq.count == 0) {
 787                 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
 788                            "AQTX: Admin queue not initialized.\n");
 789                 status = I40E_ERR_QUEUE_EMPTY;
 790                 goto asq_send_command_exit;
 791         }
 792
 793         details = I40E_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use);
 794         if (cmd_details) {
 795                 i40e_memcpy(details,
 796                             cmd_details,
 797                             sizeof(struct i40e_asq_cmd_details),
 798                             I40E_NONDMA_TO_NONDMA);
 799
 800                 /* If the cmd_details are defined copy the cookie.  The
 801                  * CPU_TO_LE32 is not needed here because the data is ignored
 802                  * by the FW, only used by the driver
 803                  */
 804                 if (details->cookie) {
 805                         desc->cookie_high =
 806                                 CPU_TO_LE32(I40E_HI_DWORD(details->cookie));
 807                         desc->cookie_low =
 808                                 CPU_TO_LE32(I40E_LO_DWORD(details->cookie));
 809                 }
 810         } else {
 811                 i40e_memset(details, 0,
 812                             sizeof(struct i40e_asq_cmd_details),
 813                             I40E_NONDMA_MEM);
 814         }
 815
 816         /* clear requested flags and then set additional flags if defined */
 817         desc->flags &= ~CPU_TO_LE16(details->flags_dis);
 818         desc->flags |= CPU_TO_LE16(details->flags_ena);
 819
 820         i40e_acquire_spinlock(&hw->aq.asq_spinlock);
 821
 822         if (buff_size > hw->aq.asq_buf_size) {
 823                 i40e_debug(hw,
 824                            I40E_DEBUG_AQ_MESSAGE,
 825                            "AQTX: Invalid buffer size: %d.\n",
 826                            buff_size);
 827                 status = I40E_ERR_INVALID_SIZE;
 828                 goto asq_send_command_error;
 829         }
 830
 831         if (details->postpone && !details->async) {
 832                 i40e_debug(hw,
 833                            I40E_DEBUG_AQ_MESSAGE,
 834                            "AQTX: Async flag not set along with postpone flag");
 835                 status = I40E_ERR_PARAM;
 836                 goto asq_send_command_error;
 837         }
 838
 839         /* call clean and check queue available function to reclaim the
 840          * descriptors that were processed by FW, the function returns the
 841          * number of desc available
 842          */
 843         /* the clean function called here could be called in a separate thread
 844          * in case of asynchronous completions
 845          */
 846         if (i40e_clean_asq(hw) == 0) {
 847                 i40e_debug(hw,
 848                            I40E_DEBUG_AQ_MESSAGE,
 849                            "AQTX: Error queue is full.\n");
 850                 status = I40E_ERR_ADMIN_QUEUE_FULL;
 851                 goto asq_send_command_error;
 852         }
 853
 854         /* initialize the temp desc pointer with the right desc */
 855         desc_on_ring = I40E_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use);
 856
 857         /* if the desc is available copy the temp desc to the right place */
 858         i40e_memcpy(desc_on_ring, desc, sizeof(struct i40e_aq_desc),
 859                     I40E_NONDMA_TO_DMA);
 860
 861         /* if buff is not NULL assume indirect command */
 862         if (buff != NULL) {
 863                 dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]);
 864                 /* copy the user buff into the respective DMA buff */
 865                 i40e_memcpy(dma_buff->va, buff, buff_size,
 866                             I40E_NONDMA_TO_DMA);
 867                 desc_on_ring->datalen = CPU_TO_LE16(buff_size);
 868
 869                 /* Update the address values in the desc with the pa value
 870                  * for respective buffer
 871                  */
 872                 desc_on_ring->params.external.addr_high =
 873                                 CPU_TO_LE32(I40E_HI_DWORD(dma_buff->pa));
 874                 desc_on_ring->params.external.addr_low =
 875                                 CPU_TO_LE32(I40E_LO_DWORD(dma_buff->pa));
 876         }
 877
 878         /* bump the tail */
 879         i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer:\n");
 880         i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc_on_ring,
 881                       buff, buff_size);
 882         (hw->aq.asq.next_to_use)++;
 883         if (hw->aq.asq.next_to_use == hw->aq.asq.count)
 884                 hw->aq.asq.next_to_use = 0;
 885         if (!details->postpone)
 886                 wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use);
 887
 888         /* if cmd_details are not defined or async flag is not set,
 889          * we need to wait for desc write back
 890          */
 891         if (!details->async && !details->postpone) {
 892                 u32 total_delay = 0;
 893
 894                 do {
 895                         /* AQ designers suggest use of head for better
 896                          * timing reliability than DD bit
 897                          */
 898                         if (i40e_asq_done(hw))
 899                                 break;
 900                         /* ugh! delay while spin_lock */
 901                         i40e_msec_delay(1);
 902                         total_delay++;
 903                 } while (total_delay < hw->aq.asq_cmd_timeout);
 904         }
 905
 906         /* if ready, copy the desc back to temp */
 907         if (i40e_asq_done(hw)) {
 908                 i40e_memcpy(desc, desc_on_ring, sizeof(struct i40e_aq_desc),
 909                             I40E_DMA_TO_NONDMA);
 910                 if (buff != NULL)
 911                         i40e_memcpy(buff, dma_buff->va, buff_size,
 912                                     I40E_DMA_TO_NONDMA);
 913                 retval = LE16_TO_CPU(desc->retval);
 914                 if (retval != 0) {
 915                         i40e_debug(hw,
 916                                    I40E_DEBUG_AQ_MESSAGE,
 917                                    "AQTX: Command completed with error 0x%X.\n",
 918                                    retval);
 919
 920                         /* strip off FW internal code */
 921                         retval &= 0xff;
 922                 }
 923                 cmd_completed = TRUE;
 924                 if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_OK)
 925                         status = I40E_SUCCESS;
 926                 else
 927                         status = I40E_ERR_ADMIN_QUEUE_ERROR;
 928                 hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval;
 929         }
 930
 931         i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
 932                    "AQTX: desc and buffer writeback:\n");
 933         i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, buff, buff_size);
 934
 935         /* save writeback aq if requested */
 936         if (details->wb_desc)
 937                 i40e_memcpy(details->wb_desc, desc_on_ring,
 938                             sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA);
 939
 940         /* update the error if time out occurred */
 941         if ((!cmd_completed) &&
 942             (!details->async && !details->postpone)) {
 943                 i40e_debug(hw,
 944                            I40E_DEBUG_AQ_MESSAGE,
 945                            "AQTX: Writeback timeout.\n");
 946                 status = I40E_ERR_ADMIN_QUEUE_TIMEOUT;
 947         }
 948
 949 asq_send_command_error:
 950         i40e_release_spinlock(&hw->aq.asq_spinlock);
 951 asq_send_command_exit:
 952         return status;
 953 }
 954
 955 /**
 956  *  i40e_fill_default_direct_cmd_desc - AQ descriptor helper function
 957  *  @desc:     pointer to the temp descriptor (non DMA mem)
 958  *  @opcode:   the opcode can be used to decide which flags to turn off or on
 959  *
 960  *  Fill the desc with default values
 961  **/
 962 void i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc *desc,
 963                                        u16 opcode)
 964 {
 965         /* zero out the desc */
 966         i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc),
 967                     I40E_NONDMA_MEM);
 968         desc->opcode = CPU_TO_LE16(opcode);
 969         desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_SI);
 970 }
 971
 972 /**
 973  *  i40e_clean_arq_element
 974  *  @hw: pointer to the hw struct
 975  *  @e: event info from the receive descriptor, includes any buffers
 976  *  @pending: number of events that could be left to process
 977  *
 978  *  This function cleans one Admin Receive Queue element and returns
 979  *  the contents through e.  It can also return how many events are
 980  *  left to process through 'pending'
 981  **/
 982 enum i40e_status_code i40e_clean_arq_element(struct i40e_hw *hw,
 983                                              struct i40e_arq_event_info *e,
 984                                              u16 *pending)
 985 {
 986         enum i40e_status_code ret_code = I40E_SUCCESS;
 987         u16 ntc = hw->aq.arq.next_to_clean;
 988         struct i40e_aq_desc *desc;
 989         struct i40e_dma_mem *bi;
 990         u16 desc_idx;
 991         u16 datalen;
 992         u16 flags;
 993         u16 ntu;
 994
 995         /* take the lock before we start messing with the ring */
 996         i40e_acquire_spinlock(&hw->aq.arq_spinlock);
 997
 998         /* set next_to_use to head */
 999         if (!i40e_is_vf(hw))
1000                 ntu = (rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK);
1001         if (i40e_is_vf(hw))
1002                 ntu = (rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK);
1003         if (ntu == ntc) {
1004                 /* nothing to do - shouldn't need to update ring's values */
1005                 ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK;
1006                 goto clean_arq_element_out;
1007         }
1008
1009         /* now clean the next descriptor */
1010         desc = I40E_ADMINQ_DESC(hw->aq.arq, ntc);
1011         desc_idx = ntc;
1012
1013         flags = LE16_TO_CPU(desc->flags);
1014         if (flags & I40E_AQ_FLAG_ERR) {
1015                 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR;
1016                 hw->aq.arq_last_status =
1017                         (enum i40e_admin_queue_err)LE16_TO_CPU(desc->retval);
1018                 i40e_debug(hw,
1019                            I40E_DEBUG_AQ_MESSAGE,
1020                            "AQRX: Event received with error 0x%X.\n",
1021                            hw->aq.arq_last_status);
1022         }
1023
1024         i40e_memcpy(&e->desc, desc, sizeof(struct i40e_aq_desc),
1025                     I40E_DMA_TO_NONDMA);
1026         datalen = LE16_TO_CPU(desc->datalen);
1027         e->msg_len = min(datalen, e->buf_len);
1028         if (e->msg_buf != NULL && (e->msg_len != 0))
1029                 i40e_memcpy(e->msg_buf,
1030                             hw->aq.arq.r.arq_bi[desc_idx].va,
1031                             e->msg_len, I40E_DMA_TO_NONDMA);
1032
1033         i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: desc and buffer:\n");
1034         i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf,
1035                       hw->aq.arq_buf_size);
1036
1037         /* Restore the original datalen and buffer address in the desc,
1038          * FW updates datalen to indicate the event message
1039          * size
1040          */
1041         bi = &hw->aq.arq.r.arq_bi[ntc];
1042         i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), I40E_DMA_MEM);
1043
1044         desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF);
1045         if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF)
1046                 desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB);
1047         desc->datalen = CPU_TO_LE16((u16)bi->size);
1048         desc->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(bi->pa));
1049         desc->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(bi->pa));
1050
1051         /* set tail = the last cleaned desc index. */
1052         wr32(hw, hw->aq.arq.tail, ntc);
1053         /* ntc is updated to tail + 1 */
1054         ntc++;
1055         if (ntc == hw->aq.num_arq_entries)
1056                 ntc = 0;
1057         hw->aq.arq.next_to_clean = ntc;
1058         hw->aq.arq.next_to_use = ntu;
1059
1060 clean_arq_element_out:
1061         /* Set pending if needed, unlock and return */
1062         if (pending != NULL)
1063                 *pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc);
1064         i40e_release_spinlock(&hw->aq.arq_spinlock);
1065
1066         if (i40e_is_nvm_update_op(&e->desc)) {
1067                 if (hw->aq.nvm_release_on_done) {
1068                         i40e_release_nvm(hw);
1069                         hw->aq.nvm_release_on_done = FALSE;
1070                 }
1071
1072                 switch (hw->nvmupd_state) {
1073                 case I40E_NVMUPD_STATE_INIT_WAIT:
1074                         hw->nvmupd_state = I40E_NVMUPD_STATE_INIT;
1075                         break;
1076
1077                 case I40E_NVMUPD_STATE_WRITE_WAIT:
1078                         hw->nvmupd_state = I40E_NVMUPD_STATE_WRITING;
1079                         break;
1080
1081                 default:
1082                         break;
1083                 }
1084         }
1085
1086         return ret_code;
1087 }
1088
1089 void i40e_resume_aq(struct i40e_hw *hw)
1090 {
1091         /* Registers are reset after PF reset */
1092         hw->aq.asq.next_to_use = 0;
1093         hw->aq.asq.next_to_clean = 0;
1094
1095         i40e_config_asq_regs(hw);
1096
1097         hw->aq.arq.next_to_use = 0;
1098         hw->aq.arq.next_to_clean = 0;
1099
1100         i40e_config_arq_regs(hw);
1101 }